352 lines
15 KiB
TypeScript
352 lines
15 KiB
TypeScript
import type { ServerResponse } from 'node:http';
|
|
|
|
const COUNTER_PATTERN = /^\d+$/;
|
|
const SIGNED_DECIMAL_PATTERN = /^-?\d+$/;
|
|
const COLLECTOR_MODES = new Set(['snapshot', 'shadow', 'native']);
|
|
const COLLECTOR_WRITERS = new Set(['snapshot', 'native']);
|
|
const APPLIED_POLICY_VALUES: Readonly<Record<string, string>> = Object.freeze({ direct: '0', vpn: '1' });
|
|
const COLLECTOR_STATES = new Set([
|
|
'connecting', 'live', 'degraded', 'stale', 'stopped', 'incompatible', 'disabled',
|
|
]);
|
|
|
|
const labelValue = (value: unknown) => String(value ?? '')
|
|
.replaceAll('\\', '\\\\')
|
|
.replaceAll('\n', '\\n')
|
|
.replaceAll('"', '\\"');
|
|
|
|
const labels = (values: Record<string, unknown>) => Object.entries(values)
|
|
.map(([key, value]) => `${key}="${labelValue(value)}"`)
|
|
.join(',');
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
? value as Record<string, unknown>
|
|
: {};
|
|
}
|
|
|
|
function counter(value: unknown) {
|
|
const decimal = String(value ?? '');
|
|
if (!COUNTER_PATTERN.test(decimal)) throw new Error(`Invalid Prometheus counter: ${decimal}`);
|
|
return decimal;
|
|
}
|
|
|
|
function signedGauge(value: unknown) {
|
|
const decimal = String(value ?? '');
|
|
if (!SIGNED_DECIMAL_PATTERN.test(decimal)) throw new Error(`Invalid Prometheus gauge: ${decimal}`);
|
|
return decimal;
|
|
}
|
|
|
|
function safeInteger(value: unknown, { signed = false } = {}) {
|
|
if (!Number.isSafeInteger(value) || (!signed && Number(value) < 0)) {
|
|
throw new Error(`Invalid Prometheus gauge: ${String(value)}`);
|
|
}
|
|
return String(value);
|
|
}
|
|
|
|
function timestamp(value: unknown) {
|
|
const milliseconds = Date.parse(String(value ?? ''));
|
|
return Number.isFinite(milliseconds) ? String(milliseconds / 1000) : null;
|
|
}
|
|
|
|
function metric(
|
|
lines: string[],
|
|
name: string,
|
|
metricLabels: Record<string, unknown>,
|
|
value: unknown,
|
|
) {
|
|
lines.push(`${name}{${labels(metricLabels)}} ${value}`);
|
|
}
|
|
|
|
export function renderPrometheusMetrics(value: unknown) {
|
|
const snapshot = record(value);
|
|
const traffic = record(snapshot.traffic);
|
|
const lines = [
|
|
'# HELP harbor_traffic_bytes_total Total traffic accounted by Harbor.',
|
|
'# TYPE harbor_traffic_bytes_total counter',
|
|
];
|
|
metric(lines, 'harbor_traffic_bytes_total', { source: 'gateway' }, counter(traffic.gatewayBytes));
|
|
metric(lines, 'harbor_traffic_bytes_total', { source: 'proxy' }, counter(traffic.proxyBytes));
|
|
|
|
const globalFreshness = [
|
|
['gateway', traffic.gatewayObservedAt],
|
|
['proxy', traffic.proxyObservedAt],
|
|
].map(([source, observedAt]) => [source, timestamp(observedAt)] as const).filter(([, observedAt]) => observedAt);
|
|
if (globalFreshness.length) {
|
|
lines.push(
|
|
'# HELP harbor_traffic_last_observed_timestamp_seconds Unix timestamp of the last successful Harbor traffic observation.',
|
|
'# TYPE harbor_traffic_last_observed_timestamp_seconds gauge',
|
|
);
|
|
for (const [source, observedAt] of globalFreshness) {
|
|
metric(lines, 'harbor_traffic_last_observed_timestamp_seconds', { source }, observedAt);
|
|
}
|
|
}
|
|
|
|
const devices = Array.isArray(snapshot.devices) ? snapshot.devices.map(record) : [];
|
|
if (devices.length) {
|
|
lines.push(
|
|
'# HELP harbor_device_info Current Harbor device identity metadata.',
|
|
'# TYPE harbor_device_info gauge',
|
|
);
|
|
for (const device of devices) {
|
|
metric(lines, 'harbor_device_info', {
|
|
device_id: device.id,
|
|
name: device.alias || device.hostname || device.ip || device.id,
|
|
ip: device.ip || '',
|
|
}, '1');
|
|
}
|
|
|
|
lines.push(
|
|
'# HELP harbor_device_applied_policy Current applied device policy: 0 Direct, 1 VPN.',
|
|
'# TYPE harbor_device_applied_policy gauge',
|
|
);
|
|
for (const device of devices) {
|
|
const appliedPolicy = String(device.appliedPolicy || '');
|
|
if (!Object.hasOwn(APPLIED_POLICY_VALUES, appliedPolicy)) {
|
|
throw new Error('Invalid applied device policy');
|
|
}
|
|
metric(lines, 'harbor_device_applied_policy', {
|
|
device_id: device.id,
|
|
}, APPLIED_POLICY_VALUES[appliedPolicy]);
|
|
}
|
|
}
|
|
|
|
const deviceTraffic = devices.flatMap((device) => [
|
|
timestamp(device.trafficObservedAt)
|
|
? { device, source: 'gateway', observedAt: timestamp(device.trafficObservedAt), uploadBytes: device.uploadBytes, downloadBytes: device.downloadBytes }
|
|
: null,
|
|
timestamp(device.proxyTrafficObservedAt)
|
|
? { device, source: 'proxy', observedAt: timestamp(device.proxyTrafficObservedAt), uploadBytes: device.proxyUploadBytes, downloadBytes: device.proxyDownloadBytes }
|
|
: null,
|
|
].filter((entry): entry is NonNullable<typeof entry> => entry !== null));
|
|
if (deviceTraffic.length) {
|
|
lines.push(
|
|
'# HELP harbor_device_traffic_bytes_total Total traffic accounted by Harbor for a device.',
|
|
'# TYPE harbor_device_traffic_bytes_total counter',
|
|
);
|
|
for (const { device, source, uploadBytes, downloadBytes } of deviceTraffic) {
|
|
metric(lines, 'harbor_device_traffic_bytes_total', {
|
|
device_id: device.id,
|
|
source,
|
|
direction: 'download',
|
|
}, counter(downloadBytes));
|
|
metric(lines, 'harbor_device_traffic_bytes_total', {
|
|
device_id: device.id,
|
|
source,
|
|
direction: 'upload',
|
|
}, counter(uploadBytes));
|
|
}
|
|
|
|
lines.push(
|
|
'# HELP harbor_device_traffic_last_observed_timestamp_seconds Unix timestamp of the last successful device traffic observation.',
|
|
'# TYPE harbor_device_traffic_last_observed_timestamp_seconds gauge',
|
|
);
|
|
for (const { device, source, observedAt } of deviceTraffic) {
|
|
metric(lines, 'harbor_device_traffic_last_observed_timestamp_seconds', {
|
|
device_id: device.id,
|
|
source,
|
|
}, observedAt);
|
|
}
|
|
}
|
|
|
|
const directTraffic = record(snapshot.directTraffic);
|
|
const directSeries = Array.isArray(directTraffic.series) ? directTraffic.series.map(record) : [];
|
|
const directObservedAt = timestamp(directTraffic.observedAt);
|
|
if (directObservedAt) {
|
|
lines.push(
|
|
'# HELP harbor_direct_ipv4_packet_bytes_total IPv4 L3 packet bytes forwarded directly instead of entering sing-box; includes IP headers and retransmissions.',
|
|
'# TYPE harbor_direct_ipv4_packet_bytes_total counter',
|
|
);
|
|
metric(lines, 'harbor_direct_ipv4_packet_bytes_total', { direction: 'download' }, counter(directTraffic.downloadBytes));
|
|
metric(lines, 'harbor_direct_ipv4_packet_bytes_total', { direction: 'upload' }, counter(directTraffic.uploadBytes));
|
|
}
|
|
if (directSeries.length) {
|
|
lines.push(
|
|
'# HELP harbor_device_direct_ipv4_packet_bytes_total Attributed IPv4 L3 packet bytes forwarded directly instead of entering sing-box.',
|
|
'# TYPE harbor_device_direct_ipv4_packet_bytes_total counter',
|
|
);
|
|
for (const series of directSeries) {
|
|
for (const [direction, amount] of [
|
|
['download', series.downloadBytes],
|
|
['upload', series.uploadBytes],
|
|
]) {
|
|
metric(lines, 'harbor_device_direct_ipv4_packet_bytes_total', {
|
|
device_id: series.deviceId,
|
|
direction,
|
|
}, counter(amount));
|
|
}
|
|
}
|
|
}
|
|
if (directObservedAt) {
|
|
lines.push(
|
|
'# HELP harbor_direct_ipv4_packet_last_observed_timestamp_seconds Unix timestamp of the last successful direct IPv4 packet observation.',
|
|
'# TYPE harbor_direct_ipv4_packet_last_observed_timestamp_seconds gauge',
|
|
);
|
|
lines.push(`harbor_direct_ipv4_packet_last_observed_timestamp_seconds ${directObservedAt}`);
|
|
}
|
|
|
|
const domainTraffic = record(snapshot.domainTraffic);
|
|
const collectorSource = record(domainTraffic.source);
|
|
const hasCollectorDiagnostics = ['mode', 'writer', 'native', 'shadow']
|
|
.some((field) => Object.hasOwn(collectorSource, field));
|
|
if (hasCollectorDiagnostics) {
|
|
const source = collectorSource;
|
|
const mode = String(source.mode || '');
|
|
const writer = String(source.writer || '');
|
|
if (!COLLECTOR_MODES.has(mode) || !COLLECTOR_WRITERS.has(writer)) {
|
|
throw new Error('Invalid traffic collector labels');
|
|
}
|
|
lines.push(
|
|
'# HELP harbor_traffic_collector_info Current Gateway traffic collector mode and canonical writer.',
|
|
'# TYPE harbor_traffic_collector_info gauge',
|
|
);
|
|
metric(lines, 'harbor_traffic_collector_info', { mode, writer }, '1');
|
|
|
|
if (source.native !== null) {
|
|
const native = record(source.native);
|
|
const state = String(native.state || '');
|
|
if (!COLLECTOR_STATES.has(state)) throw new Error('Invalid traffic collector state');
|
|
lines.push(
|
|
'# HELP harbor_traffic_collector_state Current native traffic collector state.',
|
|
'# TYPE harbor_traffic_collector_state gauge',
|
|
);
|
|
metric(lines, 'harbor_traffic_collector_state', { state }, '1');
|
|
lines.push(
|
|
'# HELP harbor_traffic_collector_unattributed_bytes Native traffic bytes not attributed to a lifecycle connection.',
|
|
'# TYPE harbor_traffic_collector_unattributed_bytes gauge',
|
|
);
|
|
metric(lines, 'harbor_traffic_collector_unattributed_bytes', { direction: 'download' }, counter(native.unattributedDownloadBytes));
|
|
metric(lines, 'harbor_traffic_collector_unattributed_bytes', { direction: 'upload' }, counter(native.unattributedUploadBytes));
|
|
}
|
|
|
|
if (source.shadow !== null) {
|
|
const shadow = record(source.shadow);
|
|
lines.push(
|
|
'# HELP harbor_traffic_shadow_active_difference Native active connections minus snapshot active connections.',
|
|
'# TYPE harbor_traffic_shadow_active_difference gauge',
|
|
`harbor_traffic_shadow_active_difference ${safeInteger(shadow.activeDifference, { signed: true })}`,
|
|
'# HELP harbor_traffic_shadow_difference_bytes Native traffic bytes minus snapshot traffic bytes.',
|
|
'# TYPE harbor_traffic_shadow_difference_bytes gauge',
|
|
);
|
|
metric(lines, 'harbor_traffic_shadow_difference_bytes', { direction: 'download' }, signedGauge(shadow.downloadDifferenceBytes));
|
|
metric(lines, 'harbor_traffic_shadow_difference_bytes', { direction: 'upload' }, signedGauge(shadow.uploadDifferenceBytes));
|
|
lines.push(
|
|
'# HELP harbor_traffic_shadow_route_mismatches Route aggregate keys that differ between native and snapshot projections.',
|
|
'# TYPE harbor_traffic_shadow_route_mismatches gauge',
|
|
`harbor_traffic_shadow_route_mismatches ${safeInteger(shadow.routeMismatches)}`,
|
|
'# HELP harbor_traffic_shadow_device_mismatches Device aggregate keys that differ between native and snapshot projections.',
|
|
'# TYPE harbor_traffic_shadow_device_mismatches gauge',
|
|
`harbor_traffic_shadow_device_mismatches ${safeInteger(shadow.deviceMismatches)}`,
|
|
);
|
|
}
|
|
}
|
|
const trackedSeries = Array.isArray(domainTraffic.tracked) ? domainTraffic.tracked.map(record) : [];
|
|
if (trackedSeries.length) {
|
|
lines.push(
|
|
'# HELP harbor_singbox_tracked_bytes_total Bytes observed by the configured sing-box traffic collector; excludes IP and tunnel overhead.',
|
|
'# TYPE harbor_singbox_tracked_bytes_total counter',
|
|
);
|
|
for (const series of trackedSeries) {
|
|
const source = String(series.source || '');
|
|
const outbound = String(series.outbound || '');
|
|
if (!['gateway', 'proxy'].includes(source) || !['vpn', 'direct', 'unknown'].includes(outbound)) {
|
|
throw new Error('Invalid sing-box outbound labels');
|
|
}
|
|
for (const [direction, amount] of [
|
|
['download', series.downloadBytes],
|
|
['upload', series.uploadBytes],
|
|
]) {
|
|
metric(lines, 'harbor_singbox_tracked_bytes_total', { source, outbound, direction }, counter(amount));
|
|
}
|
|
}
|
|
}
|
|
const routeSeries = Array.isArray(domainTraffic.routes) ? domainTraffic.routes.map(record) : [];
|
|
if (routeSeries.length) {
|
|
lines.push(
|
|
'# HELP harbor_device_singbox_tracked_bytes_total Attributed bytes observed by the sing-box TCP/UDP tracker for a selected outbound.',
|
|
'# TYPE harbor_device_singbox_tracked_bytes_total counter',
|
|
);
|
|
for (const series of routeSeries) {
|
|
const source = String(series.source || '');
|
|
const outbound = String(series.outbound || '');
|
|
if (!['gateway', 'proxy'].includes(source) || !['vpn', 'direct', 'unknown'].includes(outbound)) {
|
|
throw new Error('Invalid sing-box outbound labels');
|
|
}
|
|
for (const [direction, amount] of [
|
|
['download', series.downloadBytes],
|
|
['upload', series.uploadBytes],
|
|
]) {
|
|
metric(lines, 'harbor_device_singbox_tracked_bytes_total', {
|
|
device_id: series.deviceId,
|
|
source,
|
|
outbound,
|
|
direction,
|
|
}, counter(amount));
|
|
}
|
|
}
|
|
}
|
|
const domainSeries = Array.isArray(domainTraffic.series) ? domainTraffic.series.map(record) : [];
|
|
if (domainSeries.length) {
|
|
lines.push(
|
|
'# HELP harbor_device_domain_traffic_bytes_total Traffic observed by sing-box for a device and domain.',
|
|
'# TYPE harbor_device_domain_traffic_bytes_total counter',
|
|
);
|
|
for (const series of domainSeries) {
|
|
for (const [direction, value] of [
|
|
['download', series.downloadBytes],
|
|
['upload', series.uploadBytes],
|
|
]) {
|
|
metric(lines, 'harbor_device_domain_traffic_bytes_total', {
|
|
device_id: series.deviceId,
|
|
domain: series.domain,
|
|
service: series.service,
|
|
source: series.source,
|
|
direction,
|
|
}, counter(value));
|
|
}
|
|
}
|
|
}
|
|
const domainObservedAt = timestamp(domainTraffic.observedAt);
|
|
if (domainObservedAt) {
|
|
lines.push(
|
|
'# HELP harbor_domain_traffic_last_observed_timestamp_seconds Unix timestamp of the last successful sing-box connection observation.',
|
|
'# TYPE harbor_domain_traffic_last_observed_timestamp_seconds gauge',
|
|
);
|
|
lines.push(`harbor_domain_traffic_last_observed_timestamp_seconds ${domainObservedAt}`);
|
|
lines.push(
|
|
'# HELP harbor_singbox_traffic_last_observed_timestamp_seconds Unix timestamp of the last successful sing-box traffic observation.',
|
|
'# TYPE harbor_singbox_traffic_last_observed_timestamp_seconds gauge',
|
|
`harbor_singbox_traffic_last_observed_timestamp_seconds ${domainObservedAt}`,
|
|
);
|
|
}
|
|
if (domainTraffic.overflowConnections != null) {
|
|
lines.push(
|
|
'# HELP harbor_domain_traffic_overflow_connections_total Connections aggregated after the domain series limit was reached.',
|
|
'# TYPE harbor_domain_traffic_overflow_connections_total counter',
|
|
);
|
|
lines.push(`harbor_domain_traffic_overflow_connections_total ${counter(domainTraffic.overflowConnections)}`);
|
|
}
|
|
const attributionEvents = record(domainTraffic.attributionEvents);
|
|
if (domainTraffic.attributionEvents) {
|
|
lines.push(
|
|
'# HELP harbor_domain_traffic_attribution_events_total Connections with incomplete Harbor domain attribution.',
|
|
'# TYPE harbor_domain_traffic_attribution_events_total counter',
|
|
);
|
|
for (const outcome of ['unresolved_host', 'unknown_device', 'unsupported_source']) {
|
|
metric(
|
|
lines,
|
|
'harbor_domain_traffic_attribution_events_total',
|
|
{ outcome },
|
|
counter(attributionEvents[outcome]),
|
|
);
|
|
}
|
|
}
|
|
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
export function sendPrometheusMetrics(res: ServerResponse, snapshot: unknown) {
|
|
const body = renderPrometheusMetrics(snapshot);
|
|
res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4; charset=utf-8' });
|
|
res.end(body);
|
|
}
|