Files
harbor-net/src/server/services/domainTrafficService.ts
T

618 lines
23 KiB
TypeScript

import crypto from 'node:crypto';
import http from 'node:http';
import net from 'node:net';
import { domainToASCII } from 'node:url';
import { deviceId } from './deviceInventoryService.js';
import type { NativeTrafficProjectionBatch } from './liveTrafficService.js';
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
const DEFAULT_MAX_SERIES = 4096;
const UNKNOWN_DOMAIN = { domain: '_unknown', service: 'Не распознано' };
const ATTRIBUTION_OUTCOMES = ['unresolved_host', 'unknown_device', 'unsupported_source'] as const;
type AttributionOutcome = typeof ATTRIBUTION_OUTCOMES[number];
type TrafficRoute = 'vpn' | 'direct' | 'unknown';
const SERVICE_DOMAINS = [
['YouTube', ['youtube.com', 'youtube-nocookie.com', 'youtu.be', 'googlevideo.com', 'ytimg.com']],
['OpenAI / ChatGPT', ['chatgpt.com', 'openai.com', 'oaistatic.com', 'oaiusercontent.com']],
] as const;
interface ParsedBaseConnection {
id: string;
startedAt?: string;
upload: bigint;
download: bigint;
}
type ParsedConnection =
| (ParsedBaseConnection & { outcome: 'unsupported_source' })
| (ParsedBaseConnection & {
outcome: 'unknown_device';
source: 'gateway' | 'proxy';
outbound: TrafficRoute;
})
| (ParsedBaseConnection & {
outcome: 'classified' | 'unresolved_host';
deviceId: string;
domain: string;
service: string;
source: 'gateway' | 'proxy';
outbound: TrafficRoute;
});
interface PreviousConnection {
startedAt?: string;
outcome: AttributionOutcome | 'classified';
key?: string;
requestedKey?: string;
countedUpload: bigint | null;
countedDownload: bigint | null;
trackedUpload: bigint | null;
trackedDownload: bigint | null;
}
interface DomainSeriesTotal {
deviceId: string;
domain: string;
service: string;
source: string;
uploadBytes: bigint;
downloadBytes: bigint;
}
interface RouteSeriesTotal {
deviceId?: string;
source: 'gateway' | 'proxy';
outbound: TrafficRoute;
uploadBytes: bigint;
downloadBytes: bigint;
}
interface DomainTrafficSnapshot {
epoch: string;
observedAt: string | null;
source: { error: string | null; activeConnections: number };
overflowConnections: string;
attributionEvents: Record<AttributionOutcome, string>;
tracked: Array<Omit<RouteSeriesTotal, 'deviceId' | 'uploadBytes' | 'downloadBytes'> & {
uploadBytes: string;
downloadBytes: string;
}>;
routes: Array<Omit<RouteSeriesTotal, 'uploadBytes' | 'downloadBytes'> & {
uploadBytes: string;
downloadBytes: string;
}>;
series: Array<Omit<DomainSeriesTotal, 'uploadBytes' | 'downloadBytes'> & {
uploadBytes: string;
downloadBytes: string;
}>;
}
interface ActivityEntry {
device: string;
service: string;
upload: bigint;
download: bigint;
}
interface ActivitySample {
at: number;
entries: ActivityEntry[];
}
function record(value: unknown): Record<string, unknown> {
return value && typeof value === 'object' && !Array.isArray(value)
? value as Record<string, unknown>
: {};
}
function publicDeviceLabel(value: unknown) {
const device = record(value);
for (const candidate of [device.alias, device.hostname]) {
const label = typeof candidate === 'string' ? candidate.trim() : '';
if (label && label.length <= 64 && !/[\/?#@\\]/.test(label) && !net.isIP(label)) return label;
}
return 'Устройство';
}
const matchesDomain = (domain: string, suffix: string) => domain === suffix || domain.endsWith(`.${suffix}`);
export function classifyDomain(value: unknown): { domain: string; service: string } | null {
let domain = domainToASCII(String(value || '').trim().replace(/\.$/, '')).toLowerCase();
if (domain.startsWith('www.')) domain = domain.slice(4);
const labels = domain.split('.');
if (!domain || domain.length > 253 || net.isIP(domain) || labels.length < 2
|| labels.some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))) return null;
for (const [service, suffixes] of SERVICE_DOMAINS) {
const suffix = suffixes.find((candidate) => matchesDomain(domain, candidate));
if (suffix) return { domain: suffix, service };
}
return { domain, service: domain };
}
function sourceFor(type: string): 'gateway' | 'proxy' | null {
if (type === 'tproxy/tproxy-in') return 'gateway';
if (type === 'mixed/mixed-in') return 'proxy';
return null;
}
function routeFor(value: unknown): TrafficRoute {
if (!Array.isArray(value) || !value.length
|| value.some((entry) => typeof entry !== 'string' || !entry.trim())) return 'unknown';
return value[0].trim() === 'direct' ? 'direct' : 'vpn';
}
function parseConnection(value: unknown, devicesByIp: Map<string, string | null>): ParsedConnection {
const connection = record(value);
const id = String(connection.id || '');
const metadata = record(connection.metadata);
const upload = connection.upload;
const download = connection.download;
if (!id || typeof upload !== 'number' || !Number.isSafeInteger(upload) || upload < 0
|| typeof download !== 'number' || !Number.isSafeInteger(download) || download < 0) {
throw new Error('Sing-box вернул невалидный domain traffic counter');
}
const parsed = {
id,
upload: BigInt(upload),
download: BigInt(download),
};
const source = sourceFor(String(metadata.type || ''));
if (!source) return { ...parsed, outcome: 'unsupported_source' };
const outbound = routeFor(connection.chains);
const currentDeviceId = devicesByIp.get(String(metadata.sourceIP || ''));
if (!currentDeviceId) return { ...parsed, outcome: 'unknown_device', source, outbound };
const classifiedDomain = classifyDomain(metadata.host);
const domain = classifiedDomain || UNKNOWN_DOMAIN;
return {
...parsed,
outcome: classifiedDomain ? 'classified' : 'unresolved_host',
deviceId: currentDeviceId,
...domain,
source,
outbound,
};
}
function decimalCounter(value: unknown) {
if (typeof value !== 'string' || !/^\d+$/.test(value)) {
throw new Error('Sing-box вернул невалидный native traffic counter');
}
return BigInt(value);
}
function parseNativeConnection(value: unknown): ParsedConnection {
const connection = record(value);
const inbound = record(connection.inbound);
const origin = record(connection.origin);
const destination = record(connection.destination);
const route = record(connection.route);
const traffic = record(connection.traffic);
const parsed = {
id: String(connection.id || ''),
startedAt: typeof connection.startedAt === 'string' ? connection.startedAt : undefined,
upload: decimalCounter(traffic.uploadBytes),
download: decimalCounter(traffic.downloadBytes),
};
if (!parsed.id) throw new Error('Sing-box вернул native traffic без id');
const source = sourceFor(`${String(inbound.type || '')}/${String(inbound.tag || '')}`);
if (!source) return { ...parsed, outcome: 'unsupported_source' };
const outbound: TrafficRoute = route.kind === 'vpn' || route.kind === 'direct' ? route.kind : 'unknown';
const currentDeviceId = origin.kind === 'device' && typeof origin.id === 'string' && origin.id
? origin.id
: null;
if (!currentDeviceId) return { ...parsed, outcome: 'unknown_device', source, outbound };
const classifiedDomain = classifyDomain(destination.domain);
const domain = classifiedDomain || UNKNOWN_DOMAIN;
return {
...parsed,
outcome: classifiedDomain ? 'classified' : 'unresolved_host',
deviceId: currentDeviceId,
...domain,
source,
outbound,
};
}
export function readSingboxConnections(port: number, timeoutMs = 1500): Promise<unknown> {
return new Promise((resolve, reject) => {
const request = http.get({ host: '127.0.0.1', port, path: '/connections' }, (response) => {
const chunks: Buffer[] = [];
let size = 0;
let tooLarge = false;
response.on('data', (chunk: Buffer) => {
if (tooLarge) return;
size += chunk.length;
if (size > MAX_RESPONSE_BYTES) {
tooLarge = true;
request.destroy(new Error('Sing-box connections response слишком большой'));
return;
}
chunks.push(chunk);
});
response.on('end', () => {
if (tooLarge) return;
if ((response.statusCode || 500) >= 400) {
reject(new Error(`Sing-box connections HTTP ${response.statusCode}`));
return;
}
try {
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'));
} catch (cause) {
reject(new Error('Sing-box вернул невалидный connections JSON', { cause }));
}
});
});
request.setTimeout(timeoutMs, () => request.destroy(new Error('Sing-box connections timeout')));
request.on('error', reject);
});
}
export function createDomainTrafficService({
observe,
devices,
now = () => new Date(),
maxSeries = DEFAULT_MAX_SERIES,
}: {
observe: () => Promise<unknown> | unknown;
devices: () => unknown;
now?: () => Date;
maxSeries?: number;
}) {
if (!Number.isInteger(maxSeries) || maxSeries < 2) throw new Error('Domain traffic series limit должен быть не меньше 2');
const epoch = crypto.randomUUID();
const totals = new Map<string, DomainSeriesTotal>();
const routeTotals = new Map<string, RouteSeriesTotal>();
const trackedTotals = new Map<string, RouteSeriesTotal>();
const normalSeriesLimit = maxSeries - 2;
let normalSeries = 0;
let previousConnections = new Map<string, PreviousConnection>();
const settledNativeConnections = new Map<string, PreviousConnection>();
let nativeEpoch: string | null = null;
let activeConnections = 0;
let overflowConnections = 0n;
const attributionEvents: Record<AttributionOutcome, bigint> = {
unresolved_host: 0n,
unknown_device: 0n,
unsupported_source: 0n,
};
let refreshPromise: Promise<DomainTrafficSnapshot> | null = null;
let activityEnabled = false;
let activityStartedAt = 0;
let activitySamples: ActivitySample[] = [];
let quietSince: string | null = null;
let current: DomainTrafficSnapshot = {
epoch,
observedAt: null,
source: { error: null, activeConnections: 0 },
overflowConnections: '0',
attributionEvents: { unresolved_host: '0', unknown_device: '0', unsupported_source: '0' },
tracked: [],
routes: [],
series: [],
};
function buildSnapshot(error: string | null = null): DomainTrafficSnapshot {
return {
epoch,
observedAt: current.observedAt,
source: { error, activeConnections },
overflowConnections: overflowConnections.toString(),
attributionEvents: Object.fromEntries(
ATTRIBUTION_OUTCOMES.map((outcome) => [outcome, attributionEvents[outcome].toString()]),
) as Record<AttributionOutcome, string>,
tracked: [...trackedTotals.values()]
.map((entry) => ({
source: entry.source,
outbound: entry.outbound,
uploadBytes: entry.uploadBytes.toString(),
downloadBytes: entry.downloadBytes.toString(),
}))
.sort((left, right) => (
left.source.localeCompare(right.source) || left.outbound.localeCompare(right.outbound)
)),
routes: [...routeTotals.values()]
.map((entry) => ({
...entry,
uploadBytes: entry.uploadBytes.toString(),
downloadBytes: entry.downloadBytes.toString(),
}))
.sort((left, right) => (
String(left.deviceId).localeCompare(String(right.deviceId))
|| left.source.localeCompare(right.source)
|| left.outbound.localeCompare(right.outbound)
)),
series: [...totals.values()]
.map((entry) => ({
...entry,
uploadBytes: entry.uploadBytes.toString(),
downloadBytes: entry.downloadBytes.toString(),
}))
.sort((left, right) => (
left.deviceId.localeCompare(right.deviceId)
|| left.service.localeCompare(right.service)
|| left.domain.localeCompare(right.domain)
|| left.source.localeCompare(right.source)
)),
};
}
function applyParsedConnections({
connections,
reset,
closedIds = [],
observed,
deviceLabels,
sourceActiveConnections,
}: {
connections: ParsedConnection[];
reset: boolean;
closedIds?: string[];
observed: Date;
deviceLabels: Map<string, string>;
sourceActiveConnections?: number;
}) {
const nextConnections = reset
? new Map<string, PreviousConnection>()
: new Map(previousConnections);
const activityEntries: ActivityEntry[] = [];
for (const connection of connections) {
const settled = settledNativeConnections.get(connection.id);
const previous = previousConnections.get(connection.id)
?? (connection.startedAt && settled?.startedAt === connection.startedAt ? settled : undefined);
if (previous === settled) settledNativeConnections.delete(connection.id);
if (connection.outcome !== 'classified' && previous?.outcome !== connection.outcome) {
attributionEvents[connection.outcome] += 1n;
}
if (connection.outcome !== 'unsupported_source') {
const uploadDelta = previous?.trackedUpload != null && connection.upload >= previous.trackedUpload
? connection.upload - previous.trackedUpload
: connection.upload;
const downloadDelta = previous?.trackedDownload != null && connection.download >= previous.trackedDownload
? connection.download - previous.trackedDownload
: connection.download;
const trackedKey = `${connection.source}\0${connection.outbound}`;
const tracked = trackedTotals.get(trackedKey) || {
source: connection.source,
outbound: connection.outbound,
uploadBytes: 0n,
downloadBytes: 0n,
};
tracked.uploadBytes += uploadDelta;
tracked.downloadBytes += downloadDelta;
trackedTotals.set(trackedKey, tracked);
if (activityEnabled && connection.outbound === 'vpn' && uploadDelta + downloadDelta > 0n) {
activityEntries.push({
device: 'deviceId' in connection
? deviceLabels.get(connection.deviceId) || 'Устройство'
: 'Неизвестное устройство',
service: 'service' in connection ? connection.service : 'Не распознано',
upload: uploadDelta,
download: downloadDelta,
});
}
}
if (connection.outcome === 'unknown_device' || connection.outcome === 'unsupported_source') {
nextConnections.set(connection.id, {
startedAt: connection.startedAt,
outcome: connection.outcome,
countedUpload: previous?.countedUpload ?? null,
countedDownload: previous?.countedDownload ?? null,
trackedUpload: connection.outcome === 'unknown_device'
? connection.upload
: previous?.trackedUpload ?? null,
trackedDownload: connection.outcome === 'unknown_device'
? connection.download
: previous?.trackedDownload ?? null,
});
continue;
}
if (!('deviceId' in connection)) throw new Error('Sing-box вернул невалидную attribution запись');
const requestedKey = `${connection.deviceId}\0${connection.domain}\0${connection.source}`;
let key = previous?.requestedKey === requestedKey && previous.key ? previous.key : requestedKey;
let domain = connection.domain;
let service = connection.service;
if (key !== requestedKey) {
domain = '_other';
service = 'Другие домены';
} else if (!totals.has(key) && normalSeries >= normalSeriesLimit) {
overflowConnections += 1n;
domain = '_other';
service = 'Другие домены';
key = `_other\0${domain}\0${connection.source}`;
} else if (!totals.has(key)) {
normalSeries += 1;
}
const uploadDelta = previous?.countedUpload != null && connection.upload >= previous.countedUpload
? connection.upload - previous.countedUpload
: connection.upload;
const downloadDelta = previous?.countedDownload != null && connection.download >= previous.countedDownload
? connection.download - previous.countedDownload
: connection.download;
const routeKey = `${connection.deviceId}\0${connection.source}\0${connection.outbound}`;
const routeTotal = routeTotals.get(routeKey) || {
deviceId: connection.deviceId,
source: connection.source,
outbound: connection.outbound,
uploadBytes: 0n,
downloadBytes: 0n,
};
routeTotal.uploadBytes += uploadDelta;
routeTotal.downloadBytes += downloadDelta;
routeTotals.set(routeKey, routeTotal);
const total = totals.get(key) || {
deviceId: key === requestedKey ? connection.deviceId : '_other',
domain,
service,
source: connection.source,
uploadBytes: 0n,
downloadBytes: 0n,
};
total.uploadBytes += uploadDelta;
total.downloadBytes += downloadDelta;
totals.set(key, total);
nextConnections.set(connection.id, {
startedAt: connection.startedAt,
outcome: connection.outcome,
key,
requestedKey,
countedUpload: connection.upload,
countedDownload: connection.download,
trackedUpload: connection.upload,
trackedDownload: connection.download,
});
}
for (const id of closedIds) {
const baseline = nextConnections.get(id);
if (baseline?.startedAt) {
settledNativeConnections.delete(id);
settledNativeConnections.set(id, baseline);
while (settledNativeConnections.size > 2_048) {
settledNativeConnections.delete(settledNativeConnections.keys().next().value as string);
}
}
nextConnections.delete(id);
}
previousConnections = nextConnections;
activeConnections = sourceActiveConnections ?? nextConnections.size;
if (activityEnabled) {
activitySamples.push({ at: observed.getTime(), entries: activityEntries });
activitySamples = activitySamples.filter(({ at }) => at >= observed.getTime() - 10_000);
}
current = { ...current, observedAt: observed.toISOString() };
current = buildSnapshot();
return current;
}
async function performRefresh() {
try {
const response = record(await observe());
if (!Array.isArray(response.connections)) throw new Error('Sing-box не вернул connections array');
const devicesByIp = new Map<string, string | null>();
const deviceLabels = new Map<string, string>();
const observedDevices = devices();
for (const value of Array.isArray(observedDevices) ? observedDevices : []) {
const device = record(value);
const ip = String(device.ip || '');
const id = typeof device.mac === 'string' ? deviceId(device.mac.toLowerCase()) : null;
if (!net.isIPv4(ip) || !id) continue;
devicesByIp.set(ip, devicesByIp.has(ip) ? null : id);
deviceLabels.set(id, publicDeviceLabel(device));
}
const connections = response.connections.map((connection) => parseConnection(connection, devicesByIp));
nativeEpoch = null;
return applyParsedConnections({
connections,
reset: true,
observed: now(),
deviceLabels,
sourceActiveConnections: response.connections.length,
});
} catch (error) {
current = buildSnapshot(error instanceof Error ? error.message : String(error));
throw error;
}
}
function ingestNative(batch: NativeTrafficProjectionBatch) {
try {
const observed = new Date(batch.observedAt);
if (!batch.epoch || Number.isNaN(observed.getTime())) throw new Error('Sing-box вернул невалидный native traffic batch');
const deviceLabels = new Map<string, string>();
for (const value of batch.connections) {
const connection = record(value);
const origin = record(connection.origin);
if (origin.kind === 'device' && typeof origin.id === 'string' && origin.id) {
deviceLabels.set(origin.id, publicDeviceLabel({ alias: origin.label }));
}
}
if (nativeEpoch !== batch.epoch) {
nativeEpoch = batch.epoch;
previousConnections = new Map();
settledNativeConnections.clear();
}
const connections = batch.connections.map(parseNativeConnection);
const result = applyParsedConnections({
connections,
reset: batch.reset,
closedIds: batch.closedIds,
observed,
deviceLabels,
});
return result;
} catch (error) {
current = buildSnapshot(error instanceof Error ? error.message : String(error));
throw error;
}
}
function refresh() {
if (!refreshPromise) {
refreshPromise = performRefresh().finally(() => {
refreshPromise = null;
});
}
return refreshPromise;
}
function enableActivity() {
if (activityEnabled) return;
activityEnabled = true;
activityStartedAt = now().getTime();
activitySamples = [];
quietSince = null;
}
function disableActivity() {
activityEnabled = false;
activityStartedAt = 0;
activitySamples = [];
quietSince = null;
}
function activitySnapshot(thresholdBytesPerSecond: unknown = 0) {
if (!activityEnabled || !current.observedAt) return null;
const observedAt = Date.parse(current.observedAt);
const threshold = Math.max(0, Number(thresholdBytesPerSecond) || 0);
const divisorMs = Math.max(1_000, Math.min(10_000, observedAt - activityStartedAt || 1_000));
const totals = new Map<string, ActivityEntry>();
let bytes = 0n;
for (const sample of activitySamples) {
for (const entry of sample.entries) {
bytes += entry.upload + entry.download;
const key = `${entry.device}\0${entry.service}`;
const total = totals.get(key) || { ...entry, upload: 0n, download: 0n };
total.upload += entry.upload;
total.download += entry.download;
totals.set(key, total);
}
}
const totalBytesPerSecond = Number(bytes * 1_000n / BigInt(divisorMs));
const active = totalBytesPerSecond > threshold;
quietSince = active ? null : quietSince || current.observedAt;
const latest = activitySamples.at(-1);
return {
state: active ? 'active' : 'quiet',
observedAt: current.observedAt,
windowMs: 10_000,
thresholdBytesPerSecond: threshold,
totalBytesPerSecond,
transmittingConnections: latest?.entries.length || 0,
quietSince,
blockers: [...totals.values()]
.map((entry) => ({
device: entry.device,
service: entry.service,
uploadBytesPerSecond: Number(entry.upload * 1_000n / BigInt(divisorMs)),
downloadBytesPerSecond: Number(entry.download * 1_000n / BigInt(divisorMs)),
}))
.sort((left, right) => (
right.uploadBytesPerSecond + right.downloadBytesPerSecond
- left.uploadBytesPerSecond - left.downloadBytesPerSecond
))
.slice(0, 3),
};
}
return { snapshot: () => current, refresh, ingestNative, enableActivity, disableActivity, activitySnapshot };
}