Add domain traffic metrics and Grafana dashboard

.env.example

@@ -9,6 +9,7 @@ INSTALL_RUNTIME_DEPS=true
 INSTALL_SINGBOX=true
 PROXY_PORT=8080
 PROXY_BIND_IP=0.0.0.0
+SING_BOX_API_PORT=19090
 TPROXY_PORT=7895
 TPROXY_MARK=1
 TPROXY_TABLE=100

Dockerfile

@@ -44,6 +44,7 @@ RUN chmod +x /entrypoint.sh \
 ENV PORT=3456 \
     PROXY_PORT=8080 \
     PROXY_BIND_IP=0.0.0.0 \
+    SING_BOX_API_PORT=19090 \
     TPROXY_PORT=7895 \
     DATA_DIR=/var/lib/vpn-proxy \
     SING_BOX_CONFIG=/etc/sing-box/config.json \

README.md

@@ -272,7 +272,9 @@ scrape_configs:
       - targets: ["<gateway>:3456"]
 ```
 
-`harbor_traffic_bytes_total` содержит общий накопленный объём по источникам Gateway/Proxy. `harbor_device_traffic_bytes_total` содержит upload/download по стабильному `device_id`; пользовательское название и текущий IP находятся в `harbor_device_info`. В метрики не входит direct/физический трафик вне Harbor или quota провайдера.
+`harbor_traffic_bytes_total` содержит общий накопленный объём по источникам Gateway/Proxy. `harbor_device_traffic_bytes_total` содержит upload/download по стабильному `device_id`; пользовательское название и текущий IP находятся в `harbor_device_info`. `harbor_device_domain_traffic_bytes_total` добавляет наблюдённые домен, сервис, источник и направление для каждого устройства. Dashboard показывает top services за выбранный период и таблицу доменов с фильтром по устройствам.
+
+Domain counters снимаются с активных соединений sing-box раз в 2 секунды и хранятся в памяти dataplane до его перезапуска; историю и retention хранит Prometheus. YouTube и OpenAI / ChatGPT объединяются по известным связанным доменам в label `service`, остальные значения сохраняют домен как имя сервиса. Новые domain series сверх process limit складываются в `_other`. В метрики не входит физический трафик вне Harbor, устройства с policy Direct, ECH/IP-only соединения, соединения между двумя снимками и байты после последнего снимка перед закрытием или quota провайдера.
 
 Готовый dashboard: [`monitoring/grafana/harbor-gateway.json`](monitoring/grafana/harbor-gateway.json). При импорте Grafana попросит выбрать Prometheus data source. Та же конфигурация и dashboard доступны для копирования в Gateway drawer «Как использовать» → «Prometheus и Grafana».
 

monitoring/grafana/harbor-gateway.json

@@ -128,9 +128,17 @@
           "editorMode": "code",
           "expr": "time() - min(harbor_traffic_last_observed_timestamp_seconds)",
           "instant": true,
-          "legendFormat": "Возраст данных",
+          "legendFormat": "Общий трафик",
           "range": false,
           "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "time() - harbor_domain_traffic_last_observed_timestamp_seconds",
+          "instant": true,
+          "legendFormat": "Домены",
+          "range": false,
+          "refId": "B"
         }
       ],
       "title": "Свежесть счётчиков",
@@ -443,6 +451,140 @@
       ],
       "title": "Объём выбранных устройств",
       "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "decimals": 1,
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 10,
+        "x": 0,
+        "y": 32
+      },
+      "id": 8,
+      "options": {
+        "displayMode": "gradient",
+        "maxVizHeight": 300,
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto",
+        "valueMode": "color"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(15, sum by (service) (increase(harbor_device_domain_traffic_bytes_total[$__range])))",
+          "instant": true,
+          "legendFormat": "{{service}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Сервисы за выбранный период",
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "decimals": 1,
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 14,
+        "x": 10,
+        "y": 32
+      },
+      "id": 9,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "enablePagination": true,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(50, sum by (device_id, service, domain) (increase(harbor_device_domain_traffic_bytes_total[$__range])) * on (device_id) group_left (name, ip) max by (device_id, name, ip) (harbor_device_info{name=~\"$device\"}))",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{name}} · {{service}} · {{domain}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Домены выбранных устройств",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "device_id": true
+            },
+            "indexByName": {
+              "name": 0,
+              "ip": 1,
+              "service": 2,
+              "domain": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "Трафик за период",
+              "domain": "Домен",
+              "ip": "IP",
+              "name": "Устройство",
+              "service": "Сервис"
+            }
+          }
+        }
+      ],
+      "type": "table"
     }
   ],
   "refresh": "30s",
@@ -492,6 +634,6 @@
   "timezone": "browser",
   "title": "Harbor Gateway Traffic",
   "uid": "harbor-gateway-traffic",
-  "version": 1,
+  "version": 2,
   "weekStart": "monday"
 }

src/server/config.js

@@ -15,6 +15,7 @@ export const settings = {
   port: parsePort(process.env.PORT, 3456),
   proxyPort,
   diagnosticsProxyPort: parsePort(process.env.DIAGNOSTICS_PROXY_PORT, 18080),
+  singboxApiPort: parsePort(process.env.SING_BOX_API_PORT, 19090),
   tproxyPort: parsePort(process.env.TPROXY_PORT, 7895),
   tproxyMark: process.env.TPROXY_MARK || "1",
   tproxyChain: process.env.TPROXY_CHAIN || "VPN_PROXY_TPROXY",

src/server/dataplane.js

@@ -8,6 +8,10 @@ import { readNeighborSnapshot } from './adapters/neighbors.js';
 import { createDeviceTrafficService } from './services/deviceTrafficService.js';
 import { createDevicePolicyService } from './services/devicePolicyService.js';
 import { createConnectivityDiagnosticsService } from './services/connectivityDiagnosticsService.js';
+import {
+  createDomainTrafficService,
+  readSingboxConnections,
+} from './services/domainTrafficService.js';
 
 const socketPath = settings.dataplaneSocket;
 const runtime = createSingboxRuntime({
@@ -31,8 +35,13 @@ const devicePolicy = createDevicePolicyService({
 const connectivityDiagnostics = createConnectivityDiagnosticsService({
   proxyPort: settings.diagnosticsProxyPort,
 });
+const domainTraffic = createDomainTrafficService({
+  observe: () => readSingboxConnections(settings.singboxApiPort),
+  devices: () => traffic.snapshot().devices,
+});
 let ready = false;
 let trafficTimer = null;
+let domainTrafficTimer = null;
 const MAX_POLICY_BODY_BYTES = 256 * 1024;
 
 function readJson(req) {
@@ -83,6 +92,9 @@ const server = http.createServer(async (req, res) => {
     if (req.method === 'GET' && req.url === '/device-traffic') {
       return sendJson(res, 200, traffic.snapshot());
     }
+    if (req.method === 'GET' && req.url === '/domain-traffic') {
+      return sendJson(res, 200, domainTraffic.snapshot());
+    }
     if (req.method === 'GET' && req.url === '/device-policy') {
       return sendJson(res, 200, devicePolicy.snapshot());
     }
@@ -131,6 +143,16 @@ server.listen(socketPath, async () => {
       traffic.refresh().catch((error) => console.warn(`[dataplane] traffic counters не обновлены: ${error.message}`));
     }, 15_000);
     trafficTimer.unref();
+    setImmediate(() => {
+      domainTraffic.refresh()
+        .catch((error) => console.warn(`[dataplane] domain traffic не запущен: ${error.message}`));
+    });
+    // ponytail: snapshots can miss connections shorter than 2s; switch to an upstream close-event API if sing-box adds one.
+    domainTrafficTimer = setInterval(() => {
+      domainTraffic.refresh()
+        .catch((error) => console.warn(`[dataplane] domain traffic не обновлён: ${error.message}`));
+    }, 2_000);
+    domainTrafficTimer.unref();
     console.log(`[dataplane] control socket: ${socketPath}`);
   }
 });
@@ -141,6 +163,7 @@ async function shutdown() {
   shuttingDown = true;
   ready = false;
   if (trafficTimer) clearInterval(trafficTimer);
+  if (domainTrafficTimer) clearInterval(domainTrafficTimer);
   await runtime.shutdown();
   server.close(() => {
     fs.rmSync(socketPath, { force: true });

src/server/dataplaneClient.js

@@ -54,6 +54,7 @@ export function createDataplaneClient(socketPath, send = request) {
     refresh: () => update('/status', 'GET'),
     observeDevices: () => send(socketPath, '/devices', 'GET'),
     observeTraffic: () => send(socketPath, '/device-traffic', 'GET'),
+    observeDomainTraffic: () => send(socketPath, '/domain-traffic', 'GET'),
     observeDevicePolicy: () => send(socketPath, '/device-policy', 'GET'),
     applyDevicePolicies: (devices) => send(socketPath, '/device-policy', 'PUT', { devices }),
     runConnectivityDiagnostics: async (services = [], target = null) => {

src/server/index.js

@@ -127,6 +127,9 @@ const deviceInventory = settings.appMode === 'gateway'
       observeTraffic: remoteDataplane
         ? () => singboxRuntime.observeTraffic()
         : null,
+      observeDomainTraffic: remoteDataplane
+        ? () => singboxRuntime.observeDomainTraffic()
+        : null,
       observePolicy: remoteDataplane
         ? () => singboxRuntime.observeDevicePolicy()
         : () => localDevicePolicy.snapshot(),
@@ -864,7 +867,7 @@ const server = http.createServer(async (req, res) => {
     const requestUrl = new URL(req.url, `http://localhost:${settings.port}`);
     if (requestUrl.pathname === '/metrics') {
       if (!deviceInventory || req.method !== 'GET') throw new HarborError('ENDPOINT_NOT_FOUND');
-      return sendPrometheusMetrics(res, deviceInventory.snapshot());
+      return sendPrometheusMetrics(res, deviceInventory.metricsSnapshot());
     }
     return requestUrl.pathname.startsWith('/api/')
       ? await handleApi(req, res)

src/server/prometheusMetrics.js

@@ -99,6 +99,44 @@ export function renderPrometheusMetrics(snapshot) {
     }
   }
 
+  const domainTraffic = snapshot?.domainTraffic;
+  const domainSeries = Array.isArray(domainTraffic?.series) ? domainTraffic.series : [];
+  if (domainSeries.length) {
+    lines.push(
+      '# HELP harbor_device_domain_traffic_bytes_total Traffic observed by sing-box for a device and domain.',
+      '# TYPE harbor_device_domain_traffic_bytes_total counter',
+    );
+    for (const series of domainSeries) {
+      for (const [direction, value] of [
+        ['download', series.downloadBytes],
+        ['upload', series.uploadBytes],
+      ]) {
+        metric(lines, 'harbor_device_domain_traffic_bytes_total', {
+          device_id: series.deviceId,
+          domain: series.domain,
+          service: series.service,
+          source: series.source,
+          direction,
+        }, counter(value));
+      }
+    }
+  }
+  const domainObservedAt = timestamp(domainTraffic?.observedAt);
+  if (domainObservedAt) {
+    lines.push(
+      '# HELP harbor_domain_traffic_last_observed_timestamp_seconds Unix timestamp of the last successful sing-box connection observation.',
+      '# TYPE harbor_domain_traffic_last_observed_timestamp_seconds gauge',
+    );
+    lines.push(`harbor_domain_traffic_last_observed_timestamp_seconds ${domainObservedAt}`);
+  }
+  if (domainTraffic?.overflowConnections != null) {
+    lines.push(
+      '# HELP harbor_domain_traffic_overflow_connections_total Connections aggregated after the domain series limit was reached.',
+      '# TYPE harbor_domain_traffic_overflow_connections_total counter',
+    );
+    lines.push(`harbor_domain_traffic_overflow_connections_total ${counter(domainTraffic.overflowConnections)}`);
+  }
+
   return `${lines.join('\n')}\n`;
 }
 

src/server/services/deviceInventoryService.js

@@ -80,7 +80,7 @@ const DEFAULT_STATE = {
 };
 
 const normalizeMac = (value) => String(value || '').trim().toLowerCase();
-const deviceId = (mac) => `dev_${crypto.createHash('sha256').update(mac).digest('hex').slice(0, 16)}`;
+export const deviceId = (mac) => `dev_${crypto.createHash('sha256').update(mac).digest('hex').slice(0, 16)}`;
 const isPrivateMac = (mac) => (Number.parseInt(mac.slice(0, 2), 16) & 2) !== 0;
 const recordEntries = (value) => value && typeof value === 'object' && !Array.isArray(value)
   ? Object.entries(value)
@@ -414,6 +414,7 @@ export function createDeviceInventoryService({
   store,
   observe,
   observeTraffic = null,
+  observeDomainTraffic = null,
   observePolicy = null,
   applyPolicies = null,
   vendor = () => null,
@@ -425,6 +426,13 @@ export function createDeviceInventoryService({
   const trafficCursorByMac = new Map();
   let globalTrafficHistory = [];
   let globalTrafficCursor = null;
+  let domainTrafficSnapshot = {
+    epoch: null,
+    observedAt: null,
+    source: { error: null },
+    overflowConnections: '0',
+    series: [],
+  };
 
   function captureTrafficHistory(state) {
     const knownMacs = new Set(state.devices.map(({ mac }) => mac));
@@ -584,6 +592,10 @@ export function createDeviceInventoryService({
     };
   }
 
+  function metricsSnapshot() {
+    return { ...snapshot(), domainTraffic: domainTrafficSnapshot };
+  }
+
   function markPolicyEpoch(observed) {
     if (typeof observed?.epoch !== 'string' || !observed.epoch || !Array.isArray(observed.appliedIds)) return;
     store.update((stored) => {
@@ -695,7 +707,7 @@ export function createDeviceInventoryService({
   }
 
   async function performRefresh() {
-    const [result, trafficResult, policyResult] = await Promise.all([
+    const [result, trafficResult, policyResult, domainTrafficResult] = await Promise.all([
       Promise.resolve().then(() => observe()).catch((error) => ({
         observedAt: now().toISOString(),
         observations: [],
@@ -709,6 +721,10 @@ export function createDeviceInventoryService({
         ? Promise.resolve().then(() => observePolicy())
           .catch((error) => ({ transportError: error.message || String(error) }))
         : null,
+      observeDomainTraffic
+        ? Promise.resolve().then(() => observeDomainTraffic())
+          .catch((error) => ({ transportError: error.message || String(error) }))
+        : null,
     ]);
     const observedAt = result?.observedAt || now().toISOString();
     const observations = (Array.isArray(result?.observations) ? result.observations : [])
@@ -721,6 +737,14 @@ export function createDeviceInventoryService({
       identitiesByMac.get(mac).add(`${observation.ip}|${observation.interface || ''}`);
     }
     return serializePolicy(async () => {
+      if (domainTrafficResult?.transportError) {
+        domainTrafficSnapshot = {
+          ...domainTrafficSnapshot,
+          source: { error: domainTrafficResult.transportError },
+        };
+      } else if (domainTrafficResult) {
+        domainTrafficSnapshot = domainTrafficResult;
+      }
       const nextState = store.update((stored) => {
         const state = migrateDeviceInventoryState(stored);
         const byMac = new Map(state.devices.map((device) => [device.mac, device]));
@@ -1039,5 +1063,5 @@ export function createDeviceInventoryService({
     return serializePolicy(() => reconcileLocked(observed, true));
   }
 
-  return { snapshot, refresh, update, setPolicy, reconcilePolicies };
+  return { snapshot, metricsSnapshot, refresh, update, setPolicy, reconcilePolicies };
 }

src/server/services/domainTrafficService.js

@@ -0,0 +1,213 @@
+import crypto from 'node:crypto';
+import http from 'node:http';
+import net from 'node:net';
+import { domainToASCII } from 'node:url';
+import { deviceId } from './deviceInventoryService.js';
+
+const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
+const DEFAULT_MAX_SERIES = 4096;
+const SERVICE_DOMAINS = [
+  ['YouTube', ['youtube.com', 'youtube-nocookie.com', 'youtu.be', 'googlevideo.com', 'ytimg.com']],
+  ['OpenAI / ChatGPT', ['chatgpt.com', 'openai.com', 'oaistatic.com', 'oaiusercontent.com']],
+];
+
+const matchesDomain = (domain, suffix) => domain === suffix || domain.endsWith(`.${suffix}`);
+
+export function classifyDomain(value) {
+  let domain = domainToASCII(String(value || '').trim().replace(/\.$/, '')).toLowerCase();
+  if (domain.startsWith('www.')) domain = domain.slice(4);
+  const labels = domain.split('.');
+  if (!domain || domain.length > 253 || net.isIP(domain) || labels.length < 2
+    || labels.some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))) return null;
+  for (const [service, suffixes] of SERVICE_DOMAINS) {
+    const suffix = suffixes.find((candidate) => matchesDomain(domain, candidate));
+    if (suffix) return { domain: suffix, service };
+  }
+  return { domain, service: domain };
+}
+
+function sourceFor(type) {
+  if (type === 'tproxy/tproxy-in') return 'gateway';
+  if (type === 'mixed/mixed-in') return 'proxy';
+  return null;
+}
+
+function parseConnection(connection, devicesByIp) {
+  const id = String(connection?.id || '');
+  const metadata = connection?.metadata;
+  const source = sourceFor(String(metadata?.type || ''));
+  const domain = classifyDomain(metadata?.host);
+  const currentDeviceId = devicesByIp.get(String(metadata?.sourceIP || ''));
+  const upload = connection?.upload;
+  const download = connection?.download;
+  if (!id || !Number.isSafeInteger(upload) || upload < 0
+    || !Number.isSafeInteger(download) || download < 0) {
+    throw new Error('Sing-box вернул невалидный domain traffic counter');
+  }
+  if (!source || !domain || !currentDeviceId) return null;
+  return {
+    id,
+    deviceId: currentDeviceId,
+    ...domain,
+    source,
+    upload: BigInt(upload),
+    download: BigInt(download),
+  };
+}
+
+export function readSingboxConnections(port, timeoutMs = 1500) {
+  return new Promise((resolve, reject) => {
+    const request = http.get({ host: '127.0.0.1', port, path: '/connections' }, (response) => {
+      const chunks = [];
+      let size = 0;
+      let tooLarge = false;
+      response.on('data', (chunk) => {
+        if (tooLarge) return;
+        size += chunk.length;
+        if (size > MAX_RESPONSE_BYTES) {
+          tooLarge = true;
+          request.destroy(new Error('Sing-box connections response слишком большой'));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      response.on('end', () => {
+        if (tooLarge) return;
+        if ((response.statusCode || 500) >= 400) {
+          reject(new Error(`Sing-box connections HTTP ${response.statusCode}`));
+          return;
+        }
+        try {
+          resolve(JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'));
+        } catch (cause) {
+          reject(new Error('Sing-box вернул невалидный connections JSON', { cause }));
+        }
+      });
+    });
+    request.setTimeout(timeoutMs, () => request.destroy(new Error('Sing-box connections timeout')));
+    request.on('error', reject);
+  });
+}
+
+export function createDomainTrafficService({
+  observe,
+  devices,
+  now = () => new Date(),
+  maxSeries = DEFAULT_MAX_SERIES,
+}) {
+  if (!Number.isInteger(maxSeries) || maxSeries < 2) throw new Error('Domain traffic series limit должен быть не меньше 2');
+  const epoch = crypto.randomUUID();
+  const totals = new Map();
+  const normalSeriesLimit = maxSeries - 2;
+  let normalSeries = 0;
+  let previousConnections = new Map();
+  let overflowConnections = 0n;
+  let refreshPromise = null;
+  let current = {
+    epoch,
+    observedAt: null,
+    source: { error: null },
+    overflowConnections: '0',
+    series: [],
+  };
+
+  function buildSnapshot(error = null) {
+    return {
+      epoch,
+      observedAt: current.observedAt,
+      source: { error },
+      overflowConnections: overflowConnections.toString(),
+      series: [...totals.values()]
+        .map((entry) => ({
+          ...entry,
+          uploadBytes: entry.uploadBytes.toString(),
+          downloadBytes: entry.downloadBytes.toString(),
+        }))
+        .sort((left, right) => (
+          left.deviceId.localeCompare(right.deviceId)
+          || left.service.localeCompare(right.service)
+          || left.domain.localeCompare(right.domain)
+          || left.source.localeCompare(right.source)
+        )),
+    };
+  }
+
+  async function performRefresh() {
+    try {
+      const response = await observe();
+      if (!Array.isArray(response?.connections)) throw new Error('Sing-box не вернул connections array');
+      const devicesByIp = new Map();
+      const observedDevices = devices();
+      for (const device of Array.isArray(observedDevices) ? observedDevices : []) {
+        const ip = String(device?.ip || '');
+        const id = typeof device?.mac === 'string' ? deviceId(device.mac.toLowerCase()) : null;
+        if (!net.isIPv4(ip) || !id) continue;
+        devicesByIp.set(ip, devicesByIp.has(ip) ? null : id);
+      }
+      const parsed = response.connections
+        .map((connection) => parseConnection(connection, devicesByIp))
+        .filter(Boolean);
+      const activeConnections = new Map();
+      for (const connection of parsed) {
+        const requestedKey = `${connection.deviceId}\0${connection.domain}\0${connection.source}`;
+        const previous = previousConnections.get(connection.id);
+        let key = previous?.requestedKey === requestedKey ? previous.key : requestedKey;
+        let domain = connection.domain;
+        let service = connection.service;
+        if (key !== requestedKey) {
+          domain = '_other';
+          service = 'Другие домены';
+        } else if (!totals.has(key) && normalSeries >= normalSeriesLimit) {
+          overflowConnections += 1n;
+          domain = '_other';
+          service = 'Другие домены';
+          key = `_other\0${domain}\0${connection.source}`;
+        } else if (!totals.has(key)) {
+          normalSeries += 1;
+        }
+        const sameSeries = previous?.key === key;
+        const uploadDelta = sameSeries && connection.upload >= previous.upload
+          ? connection.upload - previous.upload
+          : connection.upload;
+        const downloadDelta = sameSeries && connection.download >= previous.download
+          ? connection.download - previous.download
+          : connection.download;
+        const total = totals.get(key) || {
+          deviceId: key === requestedKey ? connection.deviceId : '_other',
+          domain,
+          service,
+          source: connection.source,
+          uploadBytes: 0n,
+          downloadBytes: 0n,
+        };
+        total.uploadBytes += uploadDelta;
+        total.downloadBytes += downloadDelta;
+        totals.set(key, total);
+        activeConnections.set(connection.id, {
+          key,
+          requestedKey,
+          upload: connection.upload,
+          download: connection.download,
+        });
+      }
+      previousConnections = activeConnections;
+      current = { ...current, observedAt: now().toISOString() };
+      current = buildSnapshot();
+      return current;
+    } catch (error) {
+      current = buildSnapshot(error.message || String(error));
+      throw error;
+    }
+  }
+
+  function refresh() {
+    if (!refreshPromise) {
+      refreshPromise = performRefresh().finally(() => {
+        refreshPromise = null;
+      });
+    }
+    return refreshPromise;
+  }
+
+  return { snapshot: () => current, refresh };
+}

src/server/singbox.js

@@ -79,6 +79,9 @@ export function buildGatewayConfig(subscriptionConfig, selectedTag, {
     log: { level: settings.logLevel, timestamp: true },
     experimental: {
       cache_file: { enabled: true, path: settings.cachePath },
+      ...(!clientMode ? {
+        clash_api: { external_controller: `127.0.0.1:${settings.singboxApiPort}` },
+      } : {}),
     },
     dns: { independent_cache: true },
     inbounds,

src/shared/versions.js

@@ -1,7 +1,7 @@
 export const HARBOR_VERSIONS = Object.freeze({
-  macClient: '0.19.0',
+  macClient: '0.20.0',
-  gatewayClient: '0.20.0',
+  gatewayClient: '0.21.0',
-  gatewayBackend: '0.20.0',
+  gatewayBackend: '0.21.0',
 });
 
 export function parseVersion(value) {

src/web/instructions.js

@@ -88,11 +88,11 @@ export function instructionBlocks({ isGateway, host, port, controlHost }) {
       id: 'prometheus',
       label: 'Мониторинг',
       title: 'Prometheus и Grafana',
-      summary: 'Готовые traffic metrics и dashboard для Gateway и отдельных устройств.',
+      summary: 'Готовые traffic и domain metrics для Gateway и отдельных устройств.',
       paragraphs: [
         `Prometheus забирает накопленные Harbor counters с http://${controlHost}/metrics. Ручка читает готовый snapshot и не запускает новый сбор трафика.`,
         'Harbor обновляет traffic snapshot раз в 15 секунд, поэтому начальный scrape interval и refresh dashboard в 30 секунд не создают лишних одинаковых выборок.',
-        'Dashboard показывает общий объём и скорость, источники Gateway/Proxy, список устройств и выбранных клиентов по имени. В статистику входит только трафик, учтённый Harbor.',
+        'Dashboard показывает общий объём и скорость, top services за выбранный период и домены выбранных устройств. В статистику входит только трафик, учтённый Harbor.',
       ],
       steps: [
         'Добавьте блок ниже в prometheus.yml и перезагрузите Prometheus.',
@@ -105,7 +105,7 @@ export function instructionBlocks({ isGateway, host, port, controlHost }) {
         { id: 'prometheus-config', label: 'prometheus.yml', text: prometheusScrapeConfig(controlHost) },
         { id: 'grafana-dashboard', label: 'Grafana dashboard', text: grafanaDashboardJson },
       ],
-      note: 'Имя устройства берётся из заданного вами названия, затем из hostname или IP. Переименование не сбрасывает traffic series: счётчик привязан к стабильному device ID.',
+      note: 'Domain counters снимаются с активных соединений sing-box раз в 2 секунды. Историю хранит Prometheus; соединения между снимками, устройства с policy Direct и трафик без распознанного домена в domain series не входят.',
     }] : []),
   ];
 }

test/server/dataplane-client.test.js

@@ -24,6 +24,8 @@ test('control uses the dataplane socket protocol', async () => {
   assert.equal(devices.running, true);
   const traffic = await client.observeTraffic();
   assert.equal(traffic.running, true);
+  const domainTraffic = await client.observeDomainTraffic();
+  assert.equal(domainTraffic.running, true);
   await client.observeDevicePolicy();
   await client.applyDevicePolicies([{ id: 'dev_0011223344556677' }]);
   await client.runConnectivityDiagnostics(
@@ -38,18 +40,19 @@ test('control uses the dataplane socket protocol', async () => {
     'POST /apply /run/dataplane.sock',
     'GET /devices /run/dataplane.sock',
     'GET /device-traffic /run/dataplane.sock',
+    'GET /domain-traffic /run/dataplane.sock',
     'GET /device-policy /run/dataplane.sock',
     'PUT /device-policy /run/dataplane.sock',
     'POST /diagnostics/connectivity /run/dataplane.sock',
     'POST /restart /run/dataplane.sock',
     'POST /stop /run/dataplane.sock',
   ]);
-  assert.deepEqual(requests[5].body, { devices: [{ id: 'dev_0011223344556677' }] });
+  assert.deepEqual(requests[6].body, { devices: [{ id: 'dev_0011223344556677' }] });
-  assert.deepEqual(requests[6].body, {
+  assert.deepEqual(requests[7].body, {
     services: [{ id: 'custom-test', url: 'https://example.com' }],
     target: 'site:custom-test',
   });
-  assert.equal(requests[6].timeoutMs, 25_000);
+  assert.equal(requests[7].timeoutMs, 25_000);
 });
 
 test('connectivity diagnostics expose a retryable domain error', async () => {

test/server/device-inventory.test.js

@@ -848,3 +848,19 @@ test('malformed proxy totals are backed up and an expired recovery marker is cle
   assert.deepEqual(store.read().traffic.proxy.rebaselineMacs, []);
   assert.equal(snapshot.source.traffic.proxy.error, null);
 });
+
+test('an old dataplane without domain traffic keeps inventory refresh and existing metrics available', async (t) => {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-domain-compat-'));
+  t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+  const store = createJsonStore({ filePath: path.join(directory, 'devices.json'), defaultValue: {} });
+  const service = createDeviceInventoryService({
+    store,
+    observe: () => ({ observedAt: '2026-08-08T10:00:00.000Z', observations: [], error: null }),
+    observeDomainTraffic: () => { throw new Error('Dataplane HTTP 404'); },
+  });
+
+  const snapshot = await service.refresh();
+  assert.equal(snapshot.devices.length, 0);
+  assert.equal(service.metricsSnapshot().domainTraffic.source.error, 'Dataplane HTTP 404');
+  assert.equal(service.metricsSnapshot().traffic.totalBytes, '0');
+});

test/server/domain-traffic.test.js

@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  classifyDomain,
+  createDomainTrafficService,
+} from '../../src/server/services/domainTrafficService.js';
+import { deviceId } from '../../src/server/services/deviceInventoryService.js';
+
+const mac = '00:11:22:33:44:55';
+const id = deviceId(mac);
+const device = { ip: '192.168.50.7', mac };
+const connection = (connectionId, type, host, upload, download, sourceIP = device.ip) => ({
+  id: connectionId,
+  metadata: { type, host, sourceIP },
+  upload,
+  download,
+});
+
+test('domain traffic accumulates connection deltas by device, service and source', async () => {
+  let observedAt = new Date('2026-08-08T10:00:00.000Z');
+  let response = { connections: [
+    connection('youtube', 'tproxy/tproxy-in', 'r1.googlevideo.com', 10, 100),
+    connection('chatgpt', 'mixed/mixed-in', 'www.chatgpt.com.', 20, 200),
+    connection('diagnostics', 'mixed/diagnostics-vpn-in', 'example.com', 30, 300),
+    connection('unknown-device', 'tproxy/tproxy-in', 'example.net', 40, 400, '192.168.50.99'),
+  ] };
+  const service = createDomainTrafficService({
+    observe: async () => response,
+    devices: () => [device],
+    now: () => observedAt,
+  });
+
+  await service.refresh();
+  response = { connections: [
+    connection('youtube', 'tproxy/tproxy-in', 'r1.googlevideo.com', 15, 130),
+    connection('chatgpt', 'mixed/mixed-in', 'www.chatgpt.com.', 22, 260),
+  ] };
+  observedAt = new Date('2026-08-08T10:00:02.000Z');
+  await service.refresh();
+  await service.refresh();
+
+  assert.deepEqual(service.snapshot().series, [
+    {
+      deviceId: id,
+      domain: 'chatgpt.com',
+      service: 'OpenAI / ChatGPT',
+      source: 'proxy',
+      uploadBytes: '22',
+      downloadBytes: '260',
+    },
+    {
+      deviceId: id,
+      domain: 'googlevideo.com',
+      service: 'YouTube',
+      source: 'gateway',
+      uploadBytes: '15',
+      downloadBytes: '130',
+    },
+  ]);
+  assert.equal(service.snapshot().observedAt, observedAt.toISOString());
+  assert.equal(service.snapshot().source.error, null);
+
+  response = { connections: [] };
+  await service.refresh();
+  assert.equal(service.snapshot().series[1].downloadBytes, '130');
+});
+
+test('domain traffic is bounded and keeps the last good snapshot on source failure', async () => {
+  let fail = false;
+  const service = createDomainTrafficService({
+    observe: async () => {
+      if (fail) throw new Error('Clash API unavailable');
+      return { connections: [
+        connection('first', 'tproxy/tproxy-in', 'one.example', 1, 10),
+        connection('second', 'tproxy/tproxy-in', 'two.example', 2, 20),
+      ] };
+    },
+    devices: () => [device],
+    maxSeries: 3,
+  });
+
+  await service.refresh();
+  await service.refresh();
+  assert.deepEqual(
+    Object.fromEntries(service.snapshot().series.map(({ domain, downloadBytes }) => [domain, downloadBytes])),
+    { 'one.example': '10', _other: '20' },
+  );
+  assert.equal(service.snapshot().overflowConnections, '1');
+  assert.equal(service.snapshot().series.find(({ domain }) => domain === '_other').deviceId, '_other');
+  assert.ok(service.snapshot().series.length <= 3);
+
+  fail = true;
+  await assert.rejects(service.refresh(), /Clash API unavailable/);
+  assert.equal(service.snapshot().series.find(({ domain }) => domain === 'one.example').downloadBytes, '10');
+  assert.equal(service.snapshot().source.error, 'Clash API unavailable');
+});
+
+test('domain classification normalizes known services and rejects IP or malformed labels', () => {
+  assert.deepEqual(classifyDomain('WWW.YouTube.com.'), { domain: 'youtube.com', service: 'YouTube' });
+  assert.deepEqual(classifyDomain('api.example.org'), { domain: 'api.example.org', service: 'api.example.org' });
+  assert.equal(classifyDomain('192.0.2.1'), null);
+  assert.equal(classifyDomain('broken_label.example'), null);
+});

test/server/prometheus-metrics.test.js

@@ -26,6 +26,18 @@ const snapshot = {
     proxyDownloadBytes: '0',
     proxyTrafficObservedAt: null,
   }],
+  domainTraffic: {
+    observedAt,
+    overflowConnections: '2',
+    series: [{
+      deviceId: 'dev_0011223344556677',
+      domain: 'chatgpt.com',
+      service: 'OpenAI / ChatGPT',
+      source: 'proxy',
+      uploadBytes: '12',
+      downloadBytes: '345',
+    }],
+  },
 };
 
 test('Prometheus exposition keeps exact counters, stable identity and escaped names', () => {
@@ -39,6 +51,10 @@ test('Prometheus exposition keeps exact counters, stable identity and escaped na
   assert.doesNotMatch(output, /harbor_device_traffic_bytes_total\{[^\n]*source="proxy"/);
   assert.doesNotMatch(output, /00:11:22:33:44:55/);
   assert.match(output, /harbor_device_traffic_last_observed_timestamp_seconds\{device_id="dev_0011223344556677",source="gateway"\} 1786183200/);
+  assert.match(output, /harbor_device_domain_traffic_bytes_total\{device_id="dev_0011223344556677",domain="chatgpt\.com",service="OpenAI \/ ChatGPT",source="proxy",direction="download"\} 345/);
+  assert.match(output, /harbor_domain_traffic_last_observed_timestamp_seconds 1786183200/);
+  assert.match(output, /harbor_domain_traffic_overflow_connections_total 2/);
+  assert.doesNotMatch(output, /harbor_device_domain_traffic_bytes_total\{[^\n]*name=/);
   assert.equal(output.endsWith('\n'), true);
 });
 

test/server/singbox-gateway-mode.test.js

@@ -46,5 +46,6 @@ test('gateway routes .ru domains directly and other traffic through the selected
     { inbound: ['tproxy-in'], outbound: 'test-vpn' },
     { inbound: ['mixed-in'], outbound: 'test-vpn' },
   ]);
+  assert.deepEqual(config.experimental.clash_api, { external_controller: '127.0.0.1:19090' });
   assert.equal(config.route.final, 'test-vpn');
 });

test/web/prometheus-instructions.test.js

@@ -32,7 +32,7 @@ test('metrics route reads the current snapshot before static fallback', () => {
   const staticFallback = server.indexOf(': serveStatic(req, res)');
 
   assert.ok(metricsRoute >= 0 && metricsRoute < staticFallback);
-  assert.match(server, /requestUrl\.pathname === '\/metrics'[\s\S]*deviceInventory\.snapshot\(\)/);
+  assert.match(server, /requestUrl\.pathname === '\/metrics'[\s\S]*deviceInventory\.metricsSnapshot\(\)/);
   assert.doesNotMatch(server.slice(metricsRoute, staticFallback), /deviceInventory\.refresh\(/);
 });
 
@@ -45,8 +45,12 @@ test('Grafana dashboard covers global and named per-device traffic', () => {
   assert.ok(titles.includes('Общая скорость за 5 минут'));
   assert.ok(titles.includes('Устройства по объёму'));
   assert.ok(titles.includes('Скорость выбранных устройств'));
+  assert.ok(titles.includes('Сервисы за выбранный период'));
+  assert.ok(titles.includes('Домены выбранных устройств'));
   assert.ok(expressions.some((expression) => expression.includes('harbor_traffic_bytes_total')));
   assert.ok(expressions.some((expression) => expression.includes('harbor_device_traffic_bytes_total')));
+  assert.ok(expressions.some((expression) => expression.includes('harbor_device_domain_traffic_bytes_total[$__range]')));
+  assert.ok(expressions.some((expression) => expression.includes('harbor_domain_traffic_last_observed_timestamp_seconds')));
   assert.ok(expressions.some((expression) => expression.includes('group_left (name, ip)')));
   assert.equal(dashboard.templating.list[0].query.query, 'label_values(harbor_device_info, name)');
 });