Add native traffic inspection to Harbor Connect and Gateway

2026-08-31 05:19:15 +03:00
parent 116686a138
commit 4d066cb879
62 changed files with 10975 additions and 220 deletions
@@ -3,6 +3,7 @@ import http from 'node:http';
 import net from 'node:net';
 import { domainToASCII } from 'node:url';
 import { deviceId } from './deviceInventoryService.js';
+import type { NativeTrafficProjectionBatch } from './liveTrafficService.js';

 const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
 const DEFAULT_MAX_SERIES = 4096;
@@ -17,6 +18,7 @@ const SERVICE_DOMAINS = [

 interface ParsedBaseConnection {
  id: string;
+  startedAt?: string;
  upload: bigint;
  download: bigint;
 }
@@ -38,6 +40,7 @@ type ParsedConnection =
    });

 interface PreviousConnection {
+  startedAt?: string;
  outcome: AttributionOutcome | 'classified';
  key?: string;
  requestedKey?: string;
@@ -67,7 +70,7 @@ interface RouteSeriesTotal {
 interface DomainTrafficSnapshot {
  epoch: string;
  observedAt: string | null;
-  source: { error: string | null };
+  source: { error: string | null; activeConnections: number };
  overflowConnections: string;
  attributionEvents: Record<AttributionOutcome, string>;
  tracked: Array<Omit<RouteSeriesTotal, 'deviceId' | 'uploadBytes' | 'downloadBytes'> & {
@@ -170,6 +173,46 @@ function parseConnection(value: unknown, devicesByIp: Map<string, string | null>
  };
 }

+function decimalCounter(value: unknown) {
+  if (typeof value !== 'string' || !/^\d+$/.test(value)) {
+    throw new Error('Sing-box вернул невалидный native traffic counter');
+  }
+  return BigInt(value);
+}
+
+function parseNativeConnection(value: unknown): ParsedConnection {
+  const connection = record(value);
+  const inbound = record(connection.inbound);
+  const origin = record(connection.origin);
+  const destination = record(connection.destination);
+  const route = record(connection.route);
+  const traffic = record(connection.traffic);
+  const parsed = {
+    id: String(connection.id || ''),
+    startedAt: typeof connection.startedAt === 'string' ? connection.startedAt : undefined,
+    upload: decimalCounter(traffic.uploadBytes),
+    download: decimalCounter(traffic.downloadBytes),
+  };
+  if (!parsed.id) throw new Error('Sing-box вернул native traffic без id');
+  const source = sourceFor(`${String(inbound.type || '')}/${String(inbound.tag || '')}`);
+  if (!source) return { ...parsed, outcome: 'unsupported_source' };
+  const outbound: TrafficRoute = route.kind === 'vpn' || route.kind === 'direct' ? route.kind : 'unknown';
+  const currentDeviceId = origin.kind === 'device' && typeof origin.id === 'string' && origin.id
+    ? origin.id
+    : null;
+  if (!currentDeviceId) return { ...parsed, outcome: 'unknown_device', source, outbound };
+  const classifiedDomain = classifyDomain(destination.domain);
+  const domain = classifiedDomain || UNKNOWN_DOMAIN;
+  return {
+    ...parsed,
+    outcome: classifiedDomain ? 'classified' : 'unresolved_host',
+    deviceId: currentDeviceId,
+    ...domain,
+    source,
+    outbound,
+  };
+}
+
 export function readSingboxConnections(port: number, timeoutMs = 1500): Promise<unknown> {
  return new Promise((resolve, reject) => {
    const request = http.get({ host: '127.0.0.1', port, path: '/connections' }, (response) => {
@@ -223,6 +266,9 @@ export function createDomainTrafficService({
  const normalSeriesLimit = maxSeries - 2;
  let normalSeries = 0;
  let previousConnections = new Map<string, PreviousConnection>();
+  const settledNativeConnections = new Map<string, PreviousConnection>();
+  let nativeEpoch: string | null = null;
+  let activeConnections = 0;
  let overflowConnections = 0n;
  const attributionEvents: Record<AttributionOutcome, bigint> = {
    unresolved_host: 0n,
@@ -237,7 +283,7 @@ export function createDomainTrafficService({
  let current: DomainTrafficSnapshot = {
    epoch,
    observedAt: null,
-    source: { error: null },
+    source: { error: null, activeConnections: 0 },
    overflowConnections: '0',
    attributionEvents: { unresolved_host: '0', unknown_device: '0', unsupported_source: '0' },
    tracked: [],
@@ -249,7 +295,7 @@ export function createDomainTrafficService({
    return {
      epoch,
      observedAt: current.observedAt,
-      source: { error },
+      source: { error, activeConnections },
      overflowConnections: overflowConnections.toString(),
      attributionEvents: Object.fromEntries(
        ATTRIBUTION_OUTCOMES.map((outcome) => [outcome, attributionEvents[outcome].toString()]),
@@ -290,6 +336,153 @@ export function createDomainTrafficService({
    };
  }

+  function applyParsedConnections({
+    connections,
+    reset,
+    closedIds = [],
+    observed,
+    deviceLabels,
+    sourceActiveConnections,
+  }: {
+    connections: ParsedConnection[];
+    reset: boolean;
+    closedIds?: string[];
+    observed: Date;
+    deviceLabels: Map<string, string>;
+    sourceActiveConnections?: number;
+  }) {
+    const nextConnections = reset
+      ? new Map<string, PreviousConnection>()
+      : new Map(previousConnections);
+    const activityEntries: ActivityEntry[] = [];
+    for (const connection of connections) {
+      const settled = settledNativeConnections.get(connection.id);
+      const previous = previousConnections.get(connection.id)
+        ?? (connection.startedAt && settled?.startedAt === connection.startedAt ? settled : undefined);
+      if (previous === settled) settledNativeConnections.delete(connection.id);
+      if (connection.outcome !== 'classified' && previous?.outcome !== connection.outcome) {
+        attributionEvents[connection.outcome] += 1n;
+      }
+      if (connection.outcome !== 'unsupported_source') {
+        const uploadDelta = previous?.trackedUpload != null && connection.upload >= previous.trackedUpload
+          ? connection.upload - previous.trackedUpload
+          : connection.upload;
+        const downloadDelta = previous?.trackedDownload != null && connection.download >= previous.trackedDownload
+          ? connection.download - previous.trackedDownload
+          : connection.download;
+        const trackedKey = `${connection.source}\0${connection.outbound}`;
+        const tracked = trackedTotals.get(trackedKey) || {
+          source: connection.source,
+          outbound: connection.outbound,
+          uploadBytes: 0n,
+          downloadBytes: 0n,
+        };
+        tracked.uploadBytes += uploadDelta;
+        tracked.downloadBytes += downloadDelta;
+        trackedTotals.set(trackedKey, tracked);
+        if (activityEnabled && connection.outbound === 'vpn' && uploadDelta + downloadDelta > 0n) {
+          activityEntries.push({
+            device: 'deviceId' in connection
+              ? deviceLabels.get(connection.deviceId) || 'Устройство'
+              : 'Неизвестное устройство',
+            service: 'service' in connection ? connection.service : 'Не распознано',
+            upload: uploadDelta,
+            download: downloadDelta,
+          });
+        }
+      }
+      if (connection.outcome === 'unknown_device' || connection.outcome === 'unsupported_source') {
+        nextConnections.set(connection.id, {
+          startedAt: connection.startedAt,
+          outcome: connection.outcome,
+          countedUpload: previous?.countedUpload ?? null,
+          countedDownload: previous?.countedDownload ?? null,
+          trackedUpload: connection.outcome === 'unknown_device'
+            ? connection.upload
+            : previous?.trackedUpload ?? null,
+          trackedDownload: connection.outcome === 'unknown_device'
+            ? connection.download
+            : previous?.trackedDownload ?? null,
+        });
+        continue;
+      }
+      if (!('deviceId' in connection)) throw new Error('Sing-box вернул невалидную attribution запись');
+      const requestedKey = `${connection.deviceId}\0${connection.domain}\0${connection.source}`;
+      let key = previous?.requestedKey === requestedKey && previous.key ? previous.key : requestedKey;
+      let domain = connection.domain;
+      let service = connection.service;
+      if (key !== requestedKey) {
+        domain = '_other';
+        service = 'Другие домены';
+      } else if (!totals.has(key) && normalSeries >= normalSeriesLimit) {
+        overflowConnections += 1n;
+        domain = '_other';
+        service = 'Другие домены';
+        key = `_other\0${domain}\0${connection.source}`;
+      } else if (!totals.has(key)) {
+        normalSeries += 1;
+      }
+      const uploadDelta = previous?.countedUpload != null && connection.upload >= previous.countedUpload
+        ? connection.upload - previous.countedUpload
+        : connection.upload;
+      const downloadDelta = previous?.countedDownload != null && connection.download >= previous.countedDownload
+        ? connection.download - previous.countedDownload
+        : connection.download;
+      const routeKey = `${connection.deviceId}\0${connection.source}\0${connection.outbound}`;
+      const routeTotal = routeTotals.get(routeKey) || {
+        deviceId: connection.deviceId,
+        source: connection.source,
+        outbound: connection.outbound,
+        uploadBytes: 0n,
+        downloadBytes: 0n,
+      };
+      routeTotal.uploadBytes += uploadDelta;
+      routeTotal.downloadBytes += downloadDelta;
+      routeTotals.set(routeKey, routeTotal);
+      const total = totals.get(key) || {
+        deviceId: key === requestedKey ? connection.deviceId : '_other',
+        domain,
+        service,
+        source: connection.source,
+        uploadBytes: 0n,
+        downloadBytes: 0n,
+      };
+      total.uploadBytes += uploadDelta;
+      total.downloadBytes += downloadDelta;
+      totals.set(key, total);
+      nextConnections.set(connection.id, {
+        startedAt: connection.startedAt,
+        outcome: connection.outcome,
+        key,
+        requestedKey,
+        countedUpload: connection.upload,
+        countedDownload: connection.download,
+        trackedUpload: connection.upload,
+        trackedDownload: connection.download,
+      });
+    }
+    for (const id of closedIds) {
+      const baseline = nextConnections.get(id);
+      if (baseline?.startedAt) {
+        settledNativeConnections.delete(id);
+        settledNativeConnections.set(id, baseline);
+        while (settledNativeConnections.size > 2_048) {
+          settledNativeConnections.delete(settledNativeConnections.keys().next().value as string);
+        }
+      }
+      nextConnections.delete(id);
+    }
+    previousConnections = nextConnections;
+    activeConnections = sourceActiveConnections ?? nextConnections.size;
+    if (activityEnabled) {
+      activitySamples.push({ at: observed.getTime(), entries: activityEntries });
+      activitySamples = activitySamples.filter(({ at }) => at >= observed.getTime() - 10_000);
+    }
+    current = { ...current, observedAt: observed.toISOString() };
+    current = buildSnapshot();
+    return current;
+  }
+
  async function performRefresh() {
    try {
      const response = record(await observe());
@@ -306,118 +499,46 @@ export function createDomainTrafficService({
        deviceLabels.set(id, publicDeviceLabel(device));
      }
      const connections = response.connections.map((connection) => parseConnection(connection, devicesByIp));
-      const activeConnections = new Map<string, PreviousConnection>();
-      const activityEntries: ActivityEntry[] = [];
-      for (const connection of connections) {
-        const previous = previousConnections.get(connection.id);
-        if (connection.outcome !== 'classified' && previous?.outcome !== connection.outcome) {
-          attributionEvents[connection.outcome] += 1n;
+      nativeEpoch = null;
+      return applyParsedConnections({
+        connections,
+        reset: true,
+        observed: now(),
+        deviceLabels,
+        sourceActiveConnections: response.connections.length,
+      });
+    } catch (error) {
+      current = buildSnapshot(error instanceof Error ? error.message : String(error));
+      throw error;
+    }
+  }
+
+  function ingestNative(batch: NativeTrafficProjectionBatch) {
+    try {
+      const observed = new Date(batch.observedAt);
+      if (!batch.epoch || Number.isNaN(observed.getTime())) throw new Error('Sing-box вернул невалидный native traffic batch');
+      const deviceLabels = new Map<string, string>();
+      for (const value of batch.connections) {
+        const connection = record(value);
+        const origin = record(connection.origin);
+        if (origin.kind === 'device' && typeof origin.id === 'string' && origin.id) {
+          deviceLabels.set(origin.id, publicDeviceLabel({ alias: origin.label }));
        }
-        if (connection.outcome !== 'unsupported_source') {
-          const uploadDelta = previous?.trackedUpload != null && connection.upload >= previous.trackedUpload
-            ? connection.upload - previous.trackedUpload
-            : connection.upload;
-          const downloadDelta = previous?.trackedDownload != null && connection.download >= previous.trackedDownload
-            ? connection.download - previous.trackedDownload
-            : connection.download;
-          const trackedKey = `${connection.source}\0${connection.outbound}`;
-          const tracked = trackedTotals.get(trackedKey) || {
-            source: connection.source,
-            outbound: connection.outbound,
-            uploadBytes: 0n,
-            downloadBytes: 0n,
-          };
-          tracked.uploadBytes += uploadDelta;
-          tracked.downloadBytes += downloadDelta;
-          trackedTotals.set(trackedKey, tracked);
-          if (activityEnabled && connection.outbound === 'vpn' && uploadDelta + downloadDelta > 0n) {
-            activityEntries.push({
-              device: 'deviceId' in connection
-                ? deviceLabels.get(connection.deviceId) || 'Устройство'
-                : 'Неизвестное устройство',
-              service: 'service' in connection ? connection.service : 'Не распознано',
-              upload: uploadDelta,
-              download: downloadDelta,
-            });
-          }
-        }
-        if (connection.outcome === 'unknown_device' || connection.outcome === 'unsupported_source') {
-          activeConnections.set(connection.id, {
-            outcome: connection.outcome,
-            countedUpload: previous?.countedUpload ?? null,
-            countedDownload: previous?.countedDownload ?? null,
-            trackedUpload: connection.outcome === 'unknown_device'
-              ? connection.upload
-              : previous?.trackedUpload ?? null,
-            trackedDownload: connection.outcome === 'unknown_device'
-              ? connection.download
-              : previous?.trackedDownload ?? null,
-          });
-          continue;
-        }
-        if (!('deviceId' in connection)) throw new Error('Sing-box вернул невалидную attribution запись');
-        const requestedKey = `${connection.deviceId}\0${connection.domain}\0${connection.source}`;
-        let key = previous?.requestedKey === requestedKey && previous.key ? previous.key : requestedKey;
-        let domain = connection.domain;
-        let service = connection.service;
-        if (key !== requestedKey) {
-          domain = '_other';
-          service = 'Другие домены';
-        } else if (!totals.has(key) && normalSeries >= normalSeriesLimit) {
-          overflowConnections += 1n;
-          domain = '_other';
-          service = 'Другие домены';
-          key = `_other\0${domain}\0${connection.source}`;
-        } else if (!totals.has(key)) {
-          normalSeries += 1;
-        }
-        const uploadDelta = previous?.countedUpload != null && connection.upload >= previous.countedUpload
-          ? connection.upload - previous.countedUpload
-          : connection.upload;
-        const downloadDelta = previous?.countedDownload != null && connection.download >= previous.countedDownload
-          ? connection.download - previous.countedDownload
-          : connection.download;
-        const routeKey = `${connection.deviceId}\0${connection.source}\0${connection.outbound}`;
-        const routeTotal = routeTotals.get(routeKey) || {
-          deviceId: connection.deviceId,
-          source: connection.source,
-          outbound: connection.outbound,
-          uploadBytes: 0n,
-          downloadBytes: 0n,
-        };
-        routeTotal.uploadBytes += uploadDelta;
-        routeTotal.downloadBytes += downloadDelta;
-        routeTotals.set(routeKey, routeTotal);
-        const total = totals.get(key) || {
-          deviceId: key === requestedKey ? connection.deviceId : '_other',
-          domain,
-          service,
-          source: connection.source,
-          uploadBytes: 0n,
-          downloadBytes: 0n,
-        };
-        total.uploadBytes += uploadDelta;
-        total.downloadBytes += downloadDelta;
-        totals.set(key, total);
-        activeConnections.set(connection.id, {
-          outcome: connection.outcome,
-          key,
-          requestedKey,
-          countedUpload: connection.upload,
-          countedDownload: connection.download,
-          trackedUpload: connection.upload,
-          trackedDownload: connection.download,
-        });
      }
-      previousConnections = activeConnections;
-      const observed = now();
-      if (activityEnabled) {
-        activitySamples.push({ at: observed.getTime(), entries: activityEntries });
-        activitySamples = activitySamples.filter(({ at }) => at >= observed.getTime() - 10_000);
+      if (nativeEpoch !== batch.epoch) {
+        nativeEpoch = batch.epoch;
+        previousConnections = new Map();
+        settledNativeConnections.clear();
      }
-      current = { ...current, observedAt: observed.toISOString() };
-      current = buildSnapshot();
-      return current;
+      const connections = batch.connections.map(parseNativeConnection);
+      const result = applyParsedConnections({
+        connections,
+        reset: batch.reset,
+        closedIds: batch.closedIds,
+        observed,
+        deviceLabels,
+      });
+      return result;
    } catch (error) {
      current = buildSnapshot(error instanceof Error ? error.message : String(error));
      throw error;
@@ -492,5 +613,5 @@ export function createDomainTrafficService({
    };
  }

-  return { snapshot: () => current, refresh, enableActivity, disableActivity, activitySnapshot };
+  return { snapshot: () => current, refresh, ingestNative, enableActivity, disableActivity, activitySnapshot };
 }