Migrate Harbor state and traffic history to SQLite
This commit is contained in:
@@ -14,7 +14,7 @@ on:
|
|||||||
env:
|
env:
|
||||||
DEPLOY_PATH: /opt/vpn-proxy
|
DEPLOY_PATH: /opt/vpn-proxy
|
||||||
BASE_IMAGE: vpn-proxy-runtime-base:bookworm-slim
|
BASE_IMAGE: vpn-proxy-runtime-base:bookworm-slim
|
||||||
NODE_BUILD_IMAGE: mirror.gcr.io/library/node:20.19-bookworm
|
NODE_BUILD_IMAGE: mirror.gcr.io/library/node:24.21.0-bookworm
|
||||||
RUNTIME_BASE_SOURCE_IMAGE: mirror.gcr.io/library/debian:bookworm-slim
|
RUNTIME_BASE_SOURCE_IMAGE: mirror.gcr.io/library/debian:bookworm-slim
|
||||||
APT_MIRROR: http://mirror.yandex.ru/debian
|
APT_MIRROR: http://mirror.yandex.ru/debian
|
||||||
APT_SECURITY_MIRROR: http://mirror.yandex.ru/debian-security
|
APT_SECURITY_MIRROR: http://mirror.yandex.ru/debian-security
|
||||||
@@ -100,7 +100,7 @@ jobs:
|
|||||||
echo "Restart scope: ${RESTART_SCOPE}"
|
echo "Restart scope: ${RESTART_SCOPE}"
|
||||||
echo "affected_components=${AFFECTED_COMPONENTS}" >> "$GITHUB_OUTPUT"
|
echo "affected_components=${AFFECTED_COMPONENTS}" >> "$GITHUB_OUTPUT"
|
||||||
echo "restart_scope=${RESTART_SCOPE}" >> "$GITHUB_OUTPUT"
|
echo "restart_scope=${RESTART_SCOPE}" >> "$GITHUB_OUTPUT"
|
||||||
if command -v npm >/dev/null 2>&1; then
|
if command -v npm >/dev/null 2>&1 && node scripts/check-sqlite-runtime.mjs; then
|
||||||
npm ci --no-audit --no-fund
|
npm ci --no-audit --no-fund
|
||||||
npm run typecheck
|
npm run typecheck
|
||||||
npm run check:boundaries
|
npm run check:boundaries
|
||||||
@@ -108,7 +108,7 @@ jobs:
|
|||||||
npm run build:production
|
npm run build:production
|
||||||
else
|
else
|
||||||
if ! docker run --rm "${{ env.NODE_BUILD_IMAGE }}" sh -lc 'command -v npm >/dev/null && command -v git >/dev/null && test -x /bin/bash'; then
|
if ! docker run --rm "${{ env.NODE_BUILD_IMAGE }}" sh -lc 'command -v npm >/dev/null && command -v git >/dev/null && test -x /bin/bash'; then
|
||||||
echo "Cannot validate change: host npm and the Node 20.19 build toolchain are unavailable." >&2
|
echo "Cannot validate change: the pinned Node 24.21.0 build toolchain is unavailable." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "Host npm not found; validating inside ${{ env.NODE_BUILD_IMAGE }}"
|
echo "Host npm not found; validating inside ${{ env.NODE_BUILD_IMAGE }}"
|
||||||
@@ -147,6 +147,7 @@ jobs:
|
|||||||
APT_MIRROR="${{ env.APT_MIRROR }}" \
|
APT_MIRROR="${{ env.APT_MIRROR }}" \
|
||||||
APT_SECURITY_MIRROR="${{ env.APT_SECURITY_MIRROR }}" \
|
APT_SECURITY_MIRROR="${{ env.APT_SECURITY_MIRROR }}" \
|
||||||
SINGBOX_VERSION="${{ env.SINGBOX_VERSION }}" \
|
SINGBOX_VERSION="${{ env.SINGBOX_VERSION }}" \
|
||||||
|
NODE_BUILD_IMAGE="${{ env.NODE_BUILD_IMAGE }}" \
|
||||||
./scripts/build-runtime-base.sh
|
./scripts/build-runtime-base.sh
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
24.21.0
|
||||||
+9
-2
@@ -1,8 +1,10 @@
|
|||||||
ARG NODE_BUILD_IMAGE=node:20.19-alpine
|
ARG NODE_BUILD_IMAGE=node:24.21.0-bookworm
|
||||||
ARG BASE_IMAGE=debian:bookworm-slim
|
ARG BASE_IMAGE=debian:bookworm-slim
|
||||||
|
|
||||||
FROM ${NODE_BUILD_IMAGE} AS build
|
FROM ${NODE_BUILD_IMAGE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
COPY scripts/check-sqlite-runtime.mjs ./scripts/check-sqlite-runtime.mjs
|
||||||
|
RUN node scripts/check-sqlite-runtime.mjs
|
||||||
COPY package.json package-lock.json ./
|
COPY package.json package-lock.json ./
|
||||||
RUN npm ci
|
RUN npm ci
|
||||||
COPY index.html vite.config.ts tsconfig*.json ./
|
COPY index.html vite.config.ts tsconfig*.json ./
|
||||||
@@ -13,13 +15,14 @@ COPY monitoring/grafana/harbor-gateway.json ./monitoring/grafana/harbor-gateway.
|
|||||||
RUN npm run build:production
|
RUN npm run build:production
|
||||||
|
|
||||||
FROM ${BASE_IMAGE}
|
FROM ${BASE_IMAGE}
|
||||||
|
COPY --from=build /usr/local /usr/local
|
||||||
ARG SINGBOX_VERSION=1.14.0-rc.5
|
ARG SINGBOX_VERSION=1.14.0-rc.5
|
||||||
ARG INSTALL_RUNTIME_DEPS=true
|
ARG INSTALL_RUNTIME_DEPS=true
|
||||||
ARG INSTALL_SINGBOX=true
|
ARG INSTALL_SINGBOX=true
|
||||||
|
|
||||||
RUN if [ "${INSTALL_RUNTIME_DEPS}" = "true" ]; then \
|
RUN if [ "${INSTALL_RUNTIME_DEPS}" = "true" ]; then \
|
||||||
apt-get update \
|
apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates curl iptables iproute2 ieee-data nodejs dumb-init \
|
&& apt-get install -y --no-install-recommends ca-certificates curl iptables iproute2 ieee-data dumb-init \
|
||||||
&& rm -rf /var/lib/apt/lists/*; \
|
&& rm -rf /var/lib/apt/lists/*; \
|
||||||
else \
|
else \
|
||||||
command -v dumb-init >/dev/null \
|
command -v dumb-init >/dev/null \
|
||||||
@@ -49,6 +52,10 @@ COPY --from=build /src/dist /app/dist
|
|||||||
COPY --from=build /src/node_modules/@bufbuild/protobuf /app/node_modules/@bufbuild/protobuf
|
COPY --from=build /src/node_modules/@bufbuild/protobuf /app/node_modules/@bufbuild/protobuf
|
||||||
COPY --from=build /src/node_modules/@connectrpc/connect /app/node_modules/@connectrpc/connect
|
COPY --from=build /src/node_modules/@connectrpc/connect /app/node_modules/@connectrpc/connect
|
||||||
COPY --from=build /src/node_modules/@connectrpc/connect-node /app/node_modules/@connectrpc/connect-node
|
COPY --from=build /src/node_modules/@connectrpc/connect-node /app/node_modules/@connectrpc/connect-node
|
||||||
|
COPY --from=build /src/node_modules/tldts /app/node_modules/tldts
|
||||||
|
COPY --from=build /src/node_modules/tldts-core /app/node_modules/tldts-core
|
||||||
|
COPY scripts/check-sqlite-runtime.mjs /app/scripts/check-sqlite-runtime.mjs
|
||||||
|
RUN node /app/scripts/check-sqlite-runtime.mjs
|
||||||
COPY package.json /app/package.json
|
COPY package.json /app/package.json
|
||||||
COPY entrypoint.sh /entrypoint.sh
|
COPY entrypoint.sh /entrypoint.sh
|
||||||
|
|
||||||
|
|||||||
+9
-2
@@ -1,8 +1,10 @@
|
|||||||
ARG NODE_BUILD_IMAGE=node:20.19-alpine
|
ARG NODE_BUILD_IMAGE=node:24.21.0-bookworm
|
||||||
ARG RUNTIME_IMAGE=debian:bookworm-slim
|
ARG RUNTIME_IMAGE=debian:bookworm-slim
|
||||||
|
|
||||||
FROM ${NODE_BUILD_IMAGE} AS build
|
FROM ${NODE_BUILD_IMAGE} AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
COPY scripts/check-sqlite-runtime.mjs ./scripts/check-sqlite-runtime.mjs
|
||||||
|
RUN node scripts/check-sqlite-runtime.mjs
|
||||||
COPY package.json package-lock.json ./
|
COPY package.json package-lock.json ./
|
||||||
RUN npm ci
|
RUN npm ci
|
||||||
COPY index.html vite.config.ts tsconfig*.json ./
|
COPY index.html vite.config.ts tsconfig*.json ./
|
||||||
@@ -13,10 +15,11 @@ COPY monitoring/grafana/harbor-gateway.json ./monitoring/grafana/harbor-gateway.
|
|||||||
RUN npm run build:production
|
RUN npm run build:production
|
||||||
|
|
||||||
FROM ${RUNTIME_IMAGE}
|
FROM ${RUNTIME_IMAGE}
|
||||||
|
COPY --from=build /usr/local /usr/local
|
||||||
ARG SINGBOX_VERSION=1.14.0-rc.5
|
ARG SINGBOX_VERSION=1.14.0-rc.5
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates curl dumb-init nodejs tar \
|
&& apt-get install -y --no-install-recommends ca-certificates curl dumb-init tar \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN set -eux; \
|
RUN set -eux; \
|
||||||
@@ -37,6 +40,10 @@ COPY --from=build /src/dist /app/dist
|
|||||||
COPY --from=build /src/node_modules/@bufbuild/protobuf /app/node_modules/@bufbuild/protobuf
|
COPY --from=build /src/node_modules/@bufbuild/protobuf /app/node_modules/@bufbuild/protobuf
|
||||||
COPY --from=build /src/node_modules/@connectrpc/connect /app/node_modules/@connectrpc/connect
|
COPY --from=build /src/node_modules/@connectrpc/connect /app/node_modules/@connectrpc/connect
|
||||||
COPY --from=build /src/node_modules/@connectrpc/connect-node /app/node_modules/@connectrpc/connect-node
|
COPY --from=build /src/node_modules/@connectrpc/connect-node /app/node_modules/@connectrpc/connect-node
|
||||||
|
COPY --from=build /src/node_modules/tldts /app/node_modules/tldts
|
||||||
|
COPY --from=build /src/node_modules/tldts-core /app/node_modules/tldts-core
|
||||||
|
COPY scripts/check-sqlite-runtime.mjs /app/scripts/check-sqlite-runtime.mjs
|
||||||
|
RUN node /app/scripts/check-sqlite-runtime.mjs
|
||||||
COPY package.json /app/package.json
|
COPY package.json /app/package.json
|
||||||
COPY entrypoint.client.sh /entrypoint.client.sh
|
COPY entrypoint.client.sh /entrypoint.client.sh
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
ARG BASE_IMAGE=mirror.gcr.io/library/debian:bookworm-slim
|
ARG BASE_IMAGE=mirror.gcr.io/library/debian:bookworm-slim
|
||||||
|
ARG NODE_BUILD_IMAGE=node:24.21.0-bookworm
|
||||||
|
FROM ${NODE_BUILD_IMAGE} AS node-runtime
|
||||||
FROM ${BASE_IMAGE}
|
FROM ${BASE_IMAGE}
|
||||||
|
COPY --from=node-runtime /usr/local /usr/local
|
||||||
|
COPY scripts/check-sqlite-runtime.mjs /opt/harbor/check-sqlite-runtime.mjs
|
||||||
|
RUN node /opt/harbor/check-sqlite-runtime.mjs
|
||||||
ARG SINGBOX_VERSION=1.14.0-rc.5
|
ARG SINGBOX_VERSION=1.14.0-rc.5
|
||||||
ARG APT_MIRROR=http://mirror.yandex.ru/debian
|
ARG APT_MIRROR=http://mirror.yandex.ru/debian
|
||||||
ARG APT_SECURITY_MIRROR=http://mirror.yandex.ru/debian-security
|
ARG APT_SECURITY_MIRROR=http://mirror.yandex.ru/debian-security
|
||||||
@@ -32,7 +37,7 @@ RUN export http_proxy="${http_proxy:-${HTTP_PROXY:-}}" \
|
|||||||
-o Acquire::http::Timeout=20 \
|
-o Acquire::http::Timeout=20 \
|
||||||
-o Acquire::https::Timeout=20 \
|
-o Acquire::https::Timeout=20 \
|
||||||
-o Acquire::ForceIPv4=true \
|
-o Acquire::ForceIPv4=true \
|
||||||
install -y --no-install-recommends ca-certificates curl iptables ipset iproute2 ieee-data nodejs npm dumb-init \
|
install -y --no-install-recommends ca-certificates curl iptables ipset iproute2 ieee-data dumb-init \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN set -eux; \
|
RUN set -eux; \
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ http://АДРЕС-GATEWAY:3456
|
|||||||
|
|
||||||
Откройте «Устройства» в правой панели Gateway — подписка для просмотра списка не требуется. Harbor раз в 15 секунд читает локальную таблицу соседей и показывает каждое устройство одной компактной строкой: заданное название, hostname или IP, последний контакт, выбранный график трафика и иконку применённого маршрута. По умолчанию график показывает приблизительный выход `VPN`/`Direct`; переключатель `Вход` возвращает накопленную разбивку `Gateway`/`Прокси`. Наведите курсор на имя или переведите на него фокус, чтобы открыть IP, MAC и доступный hostname; нажатие на значение копирует его. Hostname определяется через локальное обратное разрешение имён и может отсутствовать, если сеть его не публикует. Технические interface и manufacturer продолжают храниться для идентификации, но не занимают место в строке. Список разделён на «Закреплённые», «Остальные» и «Фоновые»: последняя группа сохраняется между перезапусками, показывает только identity/presence и кнопку возврата без графика, traffic и route controls. Название, закрепление, фоновое положение и накопленные totals сохраняются в volume Gateway, пока устройство остаётся в inventory.
|
Откройте «Устройства» в правой панели Gateway — подписка для просмотра списка не требуется. Harbor раз в 15 секунд читает локальную таблицу соседей и показывает каждое устройство одной компактной строкой: заданное название, hostname или IP, последний контакт, выбранный график трафика и иконку применённого маршрута. По умолчанию график показывает приблизительный выход `VPN`/`Direct`; переключатель `Вход` возвращает накопленную разбивку `Gateway`/`Прокси`. Наведите курсор на имя или переведите на него фокус, чтобы открыть IP, MAC и доступный hostname; нажатие на значение копирует его. Hostname определяется через локальное обратное разрешение имён и может отсутствовать, если сеть его не публикует. Технические interface и manufacturer продолжают храниться для идентификации, но не занимают место в строке. Список разделён на «Закреплённые», «Остальные» и «Фоновые»: последняя группа сохраняется между перезапусками, показывает только identity/presence и кнопку возврата без графика, traffic и route controls. Название, закрепление, фоновое положение и накопленные totals сохраняются в volume Gateway, пока устройство остаётся в inventory.
|
||||||
|
|
||||||
Левая панель списка ищет по имени, hostname, IP, MAC и тегам, фильтрует новые, закреплённые, фоновые или устройства без тегов и позволяет выбрать несколько тегов по правилу «хотя бы один». Каталог тегов общий для Gateway: в нём можно создать до 32 тегов и назначить устройству до 8. Назначения сохраняются вместе с `devices.json`, но маршруты не меняют. После удаления устройства по 30-дневному retention его назначения удаляются, сам каталог остаётся; вернувшееся позже устройство появляется без тегов. Если Mac-клиент подключён к старой версии Gateway, список продолжает работать, а управление тегами скрывается до обновления Gateway.
|
Левая панель списка ищет по имени, hostname, IP, MAC и тегам, фильтрует новые, закреплённые, фоновые или устройства без тегов и позволяет выбрать несколько тегов по правилу «хотя бы один». Каталог тегов общий для Gateway: в нём можно создать до 32 тегов и назначить устройству до 8. Назначения сохраняются в документе устройств внутри `harbor.sqlite`, но маршруты не меняют. После удаления устройства по 30-дневному retention его назначения удаляются, сам каталог остаётся; вернувшееся позже устройство появляется без тегов. Если Mac-клиент подключён к старой версии Gateway, список продолжает работать, а управление тегами скрывается до обновления Gateway.
|
||||||
|
|
||||||
Красная кнопка `Сбросить данные` после отдельного подтверждения обнуляет вход и выход всех устройств и начинает считать их заново. Общий график скорости на Home и уже сохранённая история Prometheus/Grafana не очищаются: входной counter выглядит для Prometheus как стандартный reset, а для выхода Harbor сохраняет только baseline отображения и не изменяет raw dataplane counters.
|
Красная кнопка `Сбросить данные` после отдельного подтверждения обнуляет вход и выход всех устройств и начинает считать их заново. Общий график скорости на Home и уже сохранённая история Prometheus/Grafana не очищаются: входной counter выглядит для Prometheus как стандартный reset, а для выхода Harbor сохраняет только baseline отображения и не изменяет raw dataplane counters.
|
||||||
|
|
||||||
@@ -288,8 +288,18 @@ SING_BOX_TRAFFIC_SOURCE=snapshot \
|
|||||||
docker compose -f docker-compose.gateway.yml up -d --build
|
docker compose -f docker-compose.gateway.yml up -d --build
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Локальная история трафика
|
||||||
|
|
||||||
|
Harbor полностью работает без Prometheus. В существующем drawer «Трафик» режимы `Сейчас / История` разделяют текущие соединения и локальные суммы. История поддерживает `24 часа / 7 дней / 30 дней / 90 дней`, поиск, маршрут и устройство на Gateway; строки раскрываются как сервис → домен → полное имя → IP. Например, `www.yandex.ru` и `mail.yandex.com` остаются разными именами внутри группы «Яндекс». IP без наблюдённого домена не выдаётся за распознанный сайт.
|
||||||
|
|
||||||
|
`traffic.sqlite` хранит рабочие данные за 90 дней: завершённые минуты за последние 7 дней, далее часы. Текущая история отстаёт не более чем на минуту при исправном сборе; API сообщает фактически доступный период, детализацию и пропуски. История начинается с включения нового native-сбора. Данные по доменам относятся только к соединениям, наблюдаемым sing-box, и не восстанавливают ранее накопленные общие счётчики.
|
||||||
|
|
||||||
|
Запись и запросы выполняются в отдельном рабочем потоке. Ошибка базы или переполнение ограниченной очереди отмечает историю как неполную, но не останавливает VPN или экспорт метрик. Для защиты от повторного учёта сохраняются позиции счётчиков: активные — пока нужны их исходные значения, закрытые — до 90 дней либо смены процесса sing-box. Размер зависит не только от доменов, но и от числа соединений; это не база фиксированного размера. Освобождённые страницы переиспользуются без обязательного немедленного уменьшения файла.
|
||||||
|
|
||||||
## Prometheus и Grafana
|
## Prometheus и Grafana
|
||||||
|
|
||||||
|
Prometheus необязателен. Он хранит только экспортируемые метрики по политике своего владельца, а не копию всей SQLite. Полную доменную/IP-детализацию внешнего архива этот релиз не обещает. Нет синхронизации баз, автоматического восполнения пропущенных scrape, восстановления SQLite из Prometheus или переключения интерфейса на него. Очистка локальной истории не удаляет внешнюю; отсутствие Prometheus не продлевает локальные 90 дней.
|
||||||
|
|
||||||
Gateway публикует уже накопленные Harbor traffic counters по адресу `http://<gateway>:3456/metrics`. Scrape не запускает дополнительный netfilter read и не меняет сохранённое состояние. Harbor обновляет snapshot раз в 15 секунд, поэтому рекомендуемый начальный scrape interval и refresh dashboard — 30 секунд:
|
Gateway публикует уже накопленные Harbor traffic counters по адресу `http://<gateway>:3456/metrics`. Scrape не запускает дополнительный netfilter read и не меняет сохранённое состояние. Harbor обновляет snapshot раз в 15 секунд, поэтому рекомендуемый начальный scrape interval и refresh dashboard — 30 секунд:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@@ -308,7 +318,7 @@ scrape_configs:
|
|||||||
|
|
||||||
Dashboard начинает со скорости скачивания и отправки в конце выбранного периода, общего трафика и фактического VPN / Direct за этот период. Под обзором полоса `Применённый режим` показывает applied policy устройства, а график `Фактический VPN / Direct` независимо показывает маршрут наблюдённых байтов. Поэтому компьютер с policy Direct, браузер которого использует Harbor Proxy, остаётся Direct на полосе режима, но его proxy-соединения учитываются в VPN. Единый фильтр `Устройства` управляет режимом, скоростью, общим трафиком, сервисами, доменами и технической детализацией. Таблица «Все устройства за период» намеренно остаётся общей и выбирает устройство в том же фильтре. Блок «Куда уходит трафик» показывает основные назначения и Top-15 доменов без пагинации. Свёрнутая техническая детализация показывает `source × outbound`, включая `proxy · vpn`, и раздельные Direct-пути через sing-box и Linux мимо sing-box. Автообновление настроено на 30 секунд; индикатор предупреждает после 60 секунд и считает данные устаревшими после 120 секунд.
|
Dashboard начинает со скорости скачивания и отправки в конце выбранного периода, общего трафика и фактического VPN / Direct за этот период. Под обзором полоса `Применённый режим` показывает applied policy устройства, а график `Фактический VPN / Direct` независимо показывает маршрут наблюдённых байтов. Поэтому компьютер с policy Direct, браузер которого использует Harbor Proxy, остаётся Direct на полосе режима, но его proxy-соединения учитываются в VPN. Единый фильтр `Устройства` управляет режимом, скоростью, общим трафиком, сервисами, доменами и технической детализацией. Таблица «Все устройства за период» намеренно остаётся общей и выбирает устройство в том же фильтре. Блок «Куда уходит трафик» показывает основные назначения и Top-15 доменов без пагинации. Свёрнутая техническая детализация показывает `source × outbound`, включая `proxy · vpn`, и раздельные Direct-пути через sing-box и Linux мимо sing-box. Автообновление настроено на 30 секунд; индикатор предупреждает после 60 секунд и считает данные устаревшими после 120 секунд.
|
||||||
|
|
||||||
В `snapshot` и `shadow` domain и sing-box outbound counters снимаются с активных соединений раз в 2 секунды. В `native` dataplane получает полный lifecycle, включая короткие соединения и финальный хвост; данные всё равно хранятся в памяти только до перезапуска, а историю и retention хранит Prometheus. Перед routing sing-box до 1 секунды распознаёт HTTP Host, TLS SNI и QUIC Server Name. YouTube и OpenAI / ChatGPT объединяются по известным связанным доменам в label `service`, остальные значения сохраняют домен как имя сервиса. Если устройство и Harbor source известны, но hostname недоступен (например, ECH или IP-only), трафик попадает в `domain="_unknown",service="Не распознано"` и не теряется. Новые domain series сверх process limit складываются в `_other`.
|
В `snapshot` и `shadow` domain и sing-box outbound counters снимаются с активных соединений раз в 2 секунды. В `native` dataplane получает полный lifecycle, включая короткие соединения и финальный хвост; существующая проекция экспортируемых domain/outbound counters хранится в памяти до перезапуска, а необязательный Prometheus независимо сохраняет полученные метрики. Полный поток также поступает в отдельную локальную `traffic.sqlite`; её очистка не сбрасывает эту проекцию. Перед routing sing-box до 1 секунды распознаёт HTTP Host, TLS SNI и QUIC Server Name. YouTube и OpenAI / ChatGPT объединяются по известным связанным доменам в label `service`, остальные значения сохраняют домен как имя сервиса. Если устройство и Harbor source известны, но hostname недоступен (например, ECH или IP-only), трафик попадает в `domain="_unknown",service="Не распознано"` и не теряется. Новые domain series сверх process limit складываются в `_other`.
|
||||||
|
|
||||||
Состояние collector и сравнение `shadow` экспортируются отдельными bounded gauges `harbor_traffic_collector_*` и `harbor_traffic_shadow_*`. Они не содержат UUID, IP, домены или пользовательские имена и не заменяют canonical traffic counters.
|
Состояние collector и сравнение `shadow` экспортируются отдельными bounded gauges `harbor_traffic_collector_*` и `harbor_traffic_shadow_*`. Они не содержат UUID, IP, домены или пользовательские имена и не заменяют canonical traffic counters.
|
||||||
|
|
||||||
@@ -383,6 +393,8 @@ docker compose -f docker-compose.client.local.yml up -d --build
|
|||||||
|
|
||||||
### Команды npm
|
### Команды npm
|
||||||
|
|
||||||
|
Для сборки и backend закреплён Node **24.21.0** (`.node-version`); используется встроенная SQLite без ORM. `npm run check:runtime` проверяет точную Node-версию, движок SQLite не старше 3.51.3 и точность 64-битных счётчиков. Та же проверка выполняется в сборочных и конечных Docker-образах. При использовании fnm: `fnm use 24.21.0`.
|
||||||
|
|
||||||
| Команда | Назначение |
|
| Команда | Назначение |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `npm ci` | Установить точные версии зависимостей из `package-lock.json` |
|
| `npm ci` | Установить точные версии зависимостей из `package-lock.json` |
|
||||||
@@ -406,4 +418,8 @@ docker compose -f docker-compose.client.local.yml up -d --build
|
|||||||
|
|
||||||
Подписка, выбранный сервер и состояние подключения хранятся в именованных Docker volumes. Поэтому обычные команды `restart`, `down`, обновление проекта и повторная сборка не удаляют настройки.
|
Подписка, выбранный сервер и состояние подключения хранятся в именованных Docker volumes. Поэтому обычные команды `restart`, `down`, обновление проекта и повторная сборка не удаляют настройки.
|
||||||
|
|
||||||
|
На каждом Mac/Gateway свои `harbor.sqlite` (настройки, подписки, устройства, правила, накопленные счётчики и журнал) и `traffic.sqlite` (ограниченная история). Журнал сохраняет прежний предел 30 дней/10 000 событий; настройки не подчиняются retention истории. Секреты, hardware ID, генерируемый конфиг и кеш sing-box остаются файлами.
|
||||||
|
|
||||||
|
Первый запуск транзакционно импортирует прежние JSON, сохраняя IDs, revisions и исходные значения счётчиков. После успеха SQLite становится единственным рабочим хранилищем; исходные JSON остаются неизменными резервными копиями, без параллельной записи. Повреждение или неизвестная версия останавливает миграцию без обнуления. Старый бинарник не читает новые данные: простой downgrade вернул бы устаревшие JSON. Правила backup и восстановления описаны в [state recovery](docs/recovery/state-recovery.md).
|
||||||
|
|
||||||
Не публикуйте файл `.env`, ссылку подписки и содержимое Docker volumes. `.env` уже исключён из Git.
|
Не публикуйте файл `.env`, ссылку подписки и содержимое Docker volumes. `.env` уже исключён из Git.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Harbor application state v1
|
# Harbor application state v1
|
||||||
|
|
||||||
`GET /api/state` is the canonical Harbor domain snapshot. Successful mutations return the same snapshot as `state`. The persisted owner is `state.json` schema v8; React keeps only drafts, disclosure, focus, animation and transport freshness.
|
`GET /api/state` is the canonical Harbor domain snapshot. Successful mutations return the same snapshot as `state`. The persisted owner is the `state` JSON document in `harbor.sqlite` (currently document schema v10). React keeps only drafts, disclosure, focus, animation and transport freshness.
|
||||||
|
|
||||||
An abbreviated snapshot:
|
An abbreviated snapshot:
|
||||||
|
|
||||||
@@ -107,7 +107,7 @@ Enabling failover while VPN is stopped validates a temporary dual-channel candid
|
|||||||
|
|
||||||
The dual config keeps one stable inbound and a sing-box selector with `interrupt_exist_connections: false`. A switch changes the outbound for new connections only. Before an automatic switch, the existing `/connections` observer measures VPN byte deltas over a bounded 10-second window. Active or unknown traffic blocks the switch; the public snapshot contains only aggregate speed, connection count and at most three safe device/service labels.
|
The dual config keeps one stable inbound and a sing-box selector with `interrupt_exist_connections: false`. A switch changes the outbound for new connections only. Before an automatic switch, the existing `/connections` observer measures VPN byte deltas over a bounded 10-second window. Active or unknown traffic blocks the switch; the public snapshot contains only aggregate speed, connection count and at most three safe device/service labels.
|
||||||
|
|
||||||
Failover mutations use `PUT /api/failover`, `POST /api/failover/pause` and `POST /api/failover/switch`. Important user events are stored separately in `activity-journal.json` and read through `GET /api/activity-journal`. The journal is not a second state owner, contains no provider URLs or raw diagnostics, uses stable ID cursors and prunes entries after 30 days.
|
Failover mutations use `PUT /api/failover`, `POST /api/failover/pause` and `POST /api/failover/switch`. Important user events are separate rows in the `harbor.sqlite` journal table and read through `GET /api/activity-journal`. The journal is not a second state owner, contains no provider URLs or raw diagnostics, uses stable ID cursors and retains at most 30 days and 10,000 entries.
|
||||||
|
|
||||||
## Compatibility and migration
|
## Compatibility and migration
|
||||||
|
|
||||||
@@ -115,6 +115,16 @@ Schema v5 migrates the legacy singleton and `subscription-cache.json` into one p
|
|||||||
|
|
||||||
Schema v6 adds the routing-rule outbound. Rules read from schemas v0-v5 migrate to `outbound: "direct"` in their existing order and both desired/applied arrays are normalized together. A schema-v6 rule without a valid outbound is rejected rather than silently rewritten. Schema v7 adds canonical connectivity-diagnostics settings. Schema v8 adds a disabled failover policy, empty runtime history and no applied dual config, so upgrading does not start monitoring or change traffic.
|
Schema v6 adds the routing-rule outbound. Rules read from schemas v0-v5 migrate to `outbound: "direct"` in their existing order and both desired/applied arrays are normalized together. A schema-v6 rule without a valid outbound is rejected rather than silently rewritten. Schema v7 adds canonical connectivity-diagnostics settings. Schema v8 adds a disabled failover policy, empty runtime history and no applied dual config, so upgrading does not start monitoring or change traffic.
|
||||||
|
|
||||||
Migration atomically backs up the previous `state.json`. After the embedded profile is committed, Harbor also backs up and removes the legacy subscription cache so there is one persisted owner. Invalid legacy cache/config returns to a truthful stopped first-run state instead of starting stale generated config.
|
The initial SQLite migration imports settings, devices and journal in one transaction, including a legacy subscription cache when needed. Original JSON files remain unchanged as transition-time backups, with no parallel writes. Invalid input or a conflicting cache owner aborts migration without replacing state or starting stale configuration. Subsequent starts use only SQLite. Current document normalizers also preserve traffic-display settings and the device inventory's tag catalogue.
|
||||||
|
|
||||||
The old HTTP projection remains bounded for one release. Schema v8 persistence is not downgrade-compatible: stop Harbor and restore the `state.json.backup-v<fromVersion>-*` matching the rollback binary instead of deploying old code over v8 data. Rolling back before profiles still also requires the matching legacy subscription-cache backup.
|
The existing HTTP compatibility projection is unchanged. A pre-SQLite binary cannot read current persistence: restore a complete compatible backup or explicitly export current data before downgrading. Old JSON files do not contain post-migration changes. See [state recovery](../recovery/state-recovery.md).
|
||||||
|
|
||||||
|
## Local traffic history
|
||||||
|
|
||||||
|
`GET /api/traffic/history` reads only the local collector's `traffic.sqlite`, through a worker and, on a split Gateway, the existing control/dataplane socket. It accepts `range=24h|7d|30d|90d`, `level=service|domain|hostname|ip`, parent filters `service/domain/hostname`, `originId`, `route=all|vpn|direct|other`, `search`, `offset` and an optional `until` timestamp in milliseconds. Pages contain at most 100 groups. Bytes are decimal strings, preserving integers above JavaScript's safe-number range.
|
||||||
|
|
||||||
|
The response reports the requested/effective period, first available observation, minute/hour boundary, current collector state, gap count and partial coverage. `query.until` is the effective end of a complete bucket; use it for matching drilldown and pagination. Current history can lag by one minute. Data older than 7 days is hourly; retention and rollup can change available granularity between requests.
|
||||||
|
|
||||||
|
History starts with the new collector, not with previously accumulated device counters. Full observed hostnames and IPs remain distinct; service grouping is a local presentation classification, not proof of ownership of an IP. Unknown domains remain unknown. A history storage error reports unavailable/partial data without stopping VPN or exported metrics.
|
||||||
|
|
||||||
|
Prometheus is optional, independent and contains only exported metrics—not a copy of this database. There is no synchronization, automatic UI fallback, scrape backfill or restoration from Prometheus. Local cleanup does not delete external history or change its retention.
|
||||||
|
|||||||
@@ -1,64 +1,39 @@
|
|||||||
# Harbor state recovery
|
# Harbor state recovery
|
||||||
|
|
||||||
Harbor keeps the existing data directory and `state.json` path. The current persisted format is `schemaVersion: 8`: schema v2 introduced local route rules, v3 added rule enabled state, v4 added stable server IDs, v5 embeds the canonical `profiles[]` collection with desired/applied profile identity, v6 adds an explicit `vpn` or `direct` outbound to every route rule, v7 stores connectivity-diagnostics settings, and v8 adds Gateway failover state.
|
## Storage owners
|
||||||
|
|
||||||
## Atomic writes
|
Harbor uses the existing data directory. `harbor.sqlite` is the only working owner of settings, profiles, subscriptions, device inventory and accumulated device counters. Settings and devices are versioned JSON documents inside SQLite (currently state schema 10 and inventory schema 3); the journal is a separate indexed table with the existing 30-day/10,000-event limit.
|
||||||
|
|
||||||
Persistent files are written to a unique temporary file in the same directory, flushed with `fsync`, closed and atomically renamed over the target. A failure before rename leaves the previous target untouched and removes the temporary file.
|
`traffic.sqlite` is separate, replaceable working history: 90 days, completed minute buckets for the latest 7 days and hourly buckets before that. Removing history does not reset settings or exported counters. Prometheus is optional and independently retains only the metrics it scrapes; it cannot restore this database.
|
||||||
|
|
||||||
Profile/server switching prepares candidate config and runtime before the final state publication. If any later step fails, Harbor restores the previous config, runtime and canonical state.
|
Secrets, hardware identity, generated configuration and sing-box's own cache remain files.
|
||||||
|
|
||||||
## Migration to profiles
|
## Atomic writes and migration
|
||||||
|
|
||||||
On startup, a legacy state is normalized before the process starts. Harbor creates one profile named `Основной`, moves the provider URL/config and metadata into it, and preserves unambiguous desired/applied server identity. A legacy state explicitly marked stopped clears stale applied residue.
|
SQLite uses WAL, FULL synchronous commits and a five-second busy timeout. A document mutation runs its read and write in one transaction. Journal append/deduplication/pruning is transactional. Profile/server switching still prepares candidate configuration and runtime before publishing the canonical state.
|
||||||
|
|
||||||
Before replacing state Harbor saves the original beside it:
|
Before the first successful SQLite startup, Harbor imports `state.json`, `devices.json`, `activity-journal.json` and, when required by a pre-profile schema, `subscription-cache.json` in one transaction. Existing normalizers preserve revision, stable IDs, ordered rules and decimal-string counters. A null optional subscription cache is valid.
|
||||||
|
|
||||||
```text
|
The import marker commits with all imported records. An unsupported version, invalid input or conflicting cache owner aborts the entire import. Harbor does not erase settings, rename damaged originals, start stale configuration or silently return to first-run. Correct the reported original and retry only after making a backup.
|
||||||
state.json.backup-v4-2026-08-11T12-00-00-000Z
|
|
||||||
```
|
|
||||||
|
|
||||||
After a valid profile has been committed, the raw legacy cache is saved and removed as a second owner:
|
After a successful import, the original JSON files remain unchanged under their original names as transition-time backups. They are never read or written as current state again. Changing them does not change Harbor. A corrupt or unsupported SQLite database does not fall back to those stale JSON files.
|
||||||
|
|
||||||
```text
|
## Backup and recovery
|
||||||
subscription-cache.json.backup-v1-2026-08-11T12-00-00-000Z
|
|
||||||
```
|
|
||||||
|
|
||||||
An invalid legacy provider config is backed up but not started. Harbor removes stale generated config and returns to a stopped first-run state.
|
Stop both control and collector processes before offline recovery. On a Gateway this means the control and dataplane components; stop the Mac backend for Mac recovery.
|
||||||
|
|
||||||
## Migration to ordered VPN/Direct rules
|
1. Preserve the whole data directory, including any `-wal` and `-shm` files, before changing anything.
|
||||||
|
2. Restore a matching backup of `harbor.sqlite` and any necessary secret/config files. Do not mix a database with another backup's WAL.
|
||||||
|
3. Start the same compatible release and inspect `GET /api/state` before applying a profile.
|
||||||
|
|
||||||
When schemas v0-v5 are read, Harbor preserves the order of `routeRules` and `appliedRouteRules` and adds `outbound: "direct"` to legacy entries before atomically committing schema v6. The original file is preserved using its actual source version, for example:
|
For online backups use SQLite's backup API; copying only a live `.sqlite` file can omit committed WAL data. Offline copies after a clean stop are simpler.
|
||||||
|
|
||||||
```text
|
To discard only working traffic history, stop the collector and move its `traffic.sqlite` plus any associated `traffic.sqlite-wal` and `traffic.sqlite-shm` aside together. Leave `harbor.sqlite` untouched. A new collector database starts a new coverage period; there is no automatic Prometheus backfill. Do not unlink an open database.
|
||||||
state.json.backup-v5-2026-08-17T12-00-00-000Z
|
|
||||||
```
|
|
||||||
|
|
||||||
After migration, malformed schema-v6 rules are rejected; Harbor does not reinterpret a missing or unknown outbound as direct.
|
Deletion/retention makes pages reusable; it does not necessarily shrink the physical file immediately. Traffic retention and compaction run in the worker, outside connection processing.
|
||||||
|
|
||||||
## Migration to failover
|
## Downgrade
|
||||||
|
|
||||||
Schemas v0-v7 migrate to v8 with failover disabled, empty switch history and no applied dual config. Migration does not start probes, enable traffic accounting or change the single-channel runtime. The original state is preserved as `state.json.backup-v<fromVersion>-*` before the atomic replacement.
|
A pre-SQLite binary ignores `harbor.sqlite`. Merely starting it would revive old JSON settings and lose all changes since the cutover. Automatic downgrade is unsupported.
|
||||||
|
|
||||||
The separate `activity-journal.json` is created on the first important event. It uses the same atomic write and corrupt-file isolation mechanism as state, retains at most 30 days, and can be removed while Harbor is stopped without affecting subscriptions, routing or VPN startup.
|
Either restore a complete pre-upgrade backup deliberately, accepting the loss of subsequent changes, or first export current state into the exact schema required by the older binary. No automatic export/downgrade tool is provided. Preserve the SQLite backup in either case; do not overwrite current state with stale JSON as a recovery shortcut.
|
||||||
|
|
||||||
## Corrupt JSON
|
|
||||||
|
|
||||||
If `state.json` cannot be parsed, Harbor renames the exact damaged bytes to:
|
|
||||||
|
|
||||||
```text
|
|
||||||
state.json.corrupt-2026-08-11T12-00-00-000Z
|
|
||||||
```
|
|
||||||
|
|
||||||
It then creates a valid empty current-schema state and reports storage recovery. A corrupt legacy subscription cache is preserved with the same suffix and is never used to start stale config.
|
|
||||||
|
|
||||||
## Manual recovery and downgrade
|
|
||||||
|
|
||||||
Perform recovery while Harbor is stopped:
|
|
||||||
|
|
||||||
1. Copy the whole data directory.
|
|
||||||
2. Inspect the intended backup with `jq . <backup-file>`.
|
|
||||||
3. Restore only matching state/cache backups to their original filenames.
|
|
||||||
4. Start Harbor and verify `GET /api/state` before applying a profile.
|
|
||||||
|
|
||||||
A pre-v8 binary cannot interpret failover state. Restore `state.json.backup-v<fromVersion>-*` matching the rollback binary; deploying old code over schema v8 is not safe. A rollback to pre-v5 additionally requires the matching state and subscription-cache backups because that binary cannot interpret canonical profiles.
|
|
||||||
|
|||||||
Generated
+30
-27
@@ -14,6 +14,7 @@
|
|||||||
"@vitejs/plugin-react": "^5.0.0",
|
"@vitejs/plugin-react": "^5.0.0",
|
||||||
"react": "^19.0.0",
|
"react": "^19.0.0",
|
||||||
"react-dom": "^19.0.0",
|
"react-dom": "^19.0.0",
|
||||||
|
"tldts": "7.4.12",
|
||||||
"vite": "^7.0.0"
|
"vite": "^7.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -21,13 +22,15 @@
|
|||||||
"@bufbuild/buf": "1.47.2",
|
"@bufbuild/buf": "1.47.2",
|
||||||
"@bufbuild/protoc-gen-es": "2.6.0",
|
"@bufbuild/protoc-gen-es": "2.6.0",
|
||||||
"@csstools/selector-specificity": "6.0.0",
|
"@csstools/selector-specificity": "6.0.0",
|
||||||
"@types/node": "22.19.17",
|
"@types/node": "24.13.4",
|
||||||
"@types/node18": "npm:@types/node@18.19.130",
|
|
||||||
"@types/react": "^19.2.18",
|
"@types/react": "^19.2.18",
|
||||||
"@types/react-dom": "^19.2.4",
|
"@types/react-dom": "^19.2.4",
|
||||||
"postcss": "8.5.14",
|
"postcss": "8.5.14",
|
||||||
"postcss-selector-parser": "7.1.4",
|
"postcss-selector-parser": "7.1.4",
|
||||||
"typescript": "7.0.2"
|
"typescript": "7.0.2"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "24.21.x"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@babel/code-frame": {
|
"node_modules/@babel/code-frame": {
|
||||||
@@ -1379,33 +1382,15 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "22.19.17",
|
"version": "24.13.4",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.4.tgz",
|
||||||
"integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==",
|
"integrity": "sha512-YJ7EqCstVTzIr0fMr7qul/977en+pQHrfmuKIo6Zr9i75Be21dr3MovcfvGtyvi2HAUrRerWps5sMO9I7WaxDw==",
|
||||||
"devOptional": true,
|
"devOptional": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~6.21.0"
|
"undici-types": "~7.18.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/node18": {
|
|
||||||
"name": "@types/node",
|
|
||||||
"version": "18.19.130",
|
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
|
|
||||||
"integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"undici-types": "~5.26.4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@types/node18/node_modules/undici-types": {
|
|
||||||
"version": "5.26.5",
|
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
|
||||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT"
|
|
||||||
},
|
|
||||||
"node_modules/@types/react": {
|
"node_modules/@types/react": {
|
||||||
"version": "19.2.18",
|
"version": "19.2.18",
|
||||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz",
|
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz",
|
||||||
@@ -2246,6 +2231,24 @@
|
|||||||
"url": "https://github.com/sponsors/SuperchupuDev"
|
"url": "https://github.com/sponsors/SuperchupuDev"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tldts": {
|
||||||
|
"version": "7.4.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.12.tgz",
|
||||||
|
"integrity": "sha512-WylhSDKVeYnWXL3a+vKTaOxjnOeEGw938hImY8zoRWJjRRK/Jp1K+IihBzIONpUmW4e3WmXT6q5FW6vlESVZCA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"tldts-core": "^7.4.12"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"tldts": "bin/cli.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/tldts-core": {
|
||||||
|
"version": "7.4.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.12.tgz",
|
||||||
|
"integrity": "sha512-nYNzS2WRf4QJmjzFFgAxLOBjyBxAGRbCy9PVBPaglcYyYajh40VBn+v5Ngr96ZMc7oM0+aCJdtQnNejvdBnXMQ==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/typescript": {
|
"node_modules/typescript": {
|
||||||
"version": "7.0.2",
|
"version": "7.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz",
|
||||||
@@ -2282,9 +2285,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/undici-types": {
|
"node_modules/undici-types": {
|
||||||
"version": "6.21.0",
|
"version": "7.18.2",
|
||||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
|
||||||
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
|
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
|
||||||
"devOptional": true,
|
"devOptional": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
|||||||
+7
-2
@@ -2,9 +2,14 @@
|
|||||||
"name": "vpn-proxy-gateway",
|
"name": "vpn-proxy-gateway",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
|
"engines": {
|
||||||
|
"node": "24.21.x"
|
||||||
|
},
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"description": "Gateway-first VPN proxy control panel for sing-box TProxy deployments.",
|
"description": "Gateway-first VPN proxy control panel for sing-box TProxy deployments.",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"check:runtime": "node scripts/check-sqlite-runtime.mjs",
|
||||||
|
"pretest": "npm run check:runtime",
|
||||||
"dev": "vite --host 0.0.0.0",
|
"dev": "vite --host 0.0.0.0",
|
||||||
"generate:singbox-api": "XDG_CACHE_HOME=${TMPDIR:-/tmp}/harbor-buf-cache buf generate --template buf.gen.yaml",
|
"generate:singbox-api": "XDG_CACHE_HOME=${TMPDIR:-/tmp}/harbor-buf-cache buf generate --template buf.gen.yaml",
|
||||||
"build": "vite build",
|
"build": "vite build",
|
||||||
@@ -25,6 +30,7 @@
|
|||||||
"@vitejs/plugin-react": "^5.0.0",
|
"@vitejs/plugin-react": "^5.0.0",
|
||||||
"react": "^19.0.0",
|
"react": "^19.0.0",
|
||||||
"react-dom": "^19.0.0",
|
"react-dom": "^19.0.0",
|
||||||
|
"tldts": "7.4.12",
|
||||||
"vite": "^7.0.0"
|
"vite": "^7.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -32,8 +38,7 @@
|
|||||||
"@bufbuild/buf": "1.47.2",
|
"@bufbuild/buf": "1.47.2",
|
||||||
"@bufbuild/protoc-gen-es": "2.6.0",
|
"@bufbuild/protoc-gen-es": "2.6.0",
|
||||||
"@csstools/selector-specificity": "6.0.0",
|
"@csstools/selector-specificity": "6.0.0",
|
||||||
"@types/node": "22.19.17",
|
"@types/node": "24.13.4",
|
||||||
"@types/node18": "npm:@types/node@18.19.130",
|
|
||||||
"@types/react": "^19.2.18",
|
"@types/react": "^19.2.18",
|
||||||
"@types/react-dom": "^19.2.4",
|
"@types/react-dom": "^19.2.4",
|
||||||
"postcss": "8.5.14",
|
"postcss": "8.5.14",
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ GIT_REF="$(git rev-parse --short HEAD 2>/dev/null || echo manual)"
|
|||||||
IMAGE_TAG="${IMAGE_TAG:-${GIT_REF}-$(date +%Y%m%d%H%M%S)}"
|
IMAGE_TAG="${IMAGE_TAG:-${GIT_REF}-$(date +%Y%m%d%H%M%S)}"
|
||||||
GATEWAY_IMAGE="${GATEWAY_IMAGE:-${IMAGE_NAME}:${IMAGE_TAG}}"
|
GATEWAY_IMAGE="${GATEWAY_IMAGE:-${IMAGE_NAME}:${IMAGE_TAG}}"
|
||||||
BASE_IMAGE="${BASE_IMAGE:-vpn-proxy-runtime-base:bookworm-slim}"
|
BASE_IMAGE="${BASE_IMAGE:-vpn-proxy-runtime-base:bookworm-slim}"
|
||||||
NODE_BUILD_IMAGE="${NODE_BUILD_IMAGE:-node:20.19-alpine}"
|
NODE_BUILD_IMAGE="${NODE_BUILD_IMAGE:-node:24.21.0-bookworm}"
|
||||||
RUNTIME_BASE_SOURCE_IMAGE="${RUNTIME_BASE_SOURCE_IMAGE:-mirror.gcr.io/library/debian:bookworm-slim}"
|
RUNTIME_BASE_SOURCE_IMAGE="${RUNTIME_BASE_SOURCE_IMAGE:-mirror.gcr.io/library/debian:bookworm-slim}"
|
||||||
SINGBOX_VERSION="${SINGBOX_VERSION:-1.14.0-rc.5}"
|
SINGBOX_VERSION="${SINGBOX_VERSION:-1.14.0-rc.5}"
|
||||||
DOCKER_BUILD_PULL="${DOCKER_BUILD_PULL:-false}"
|
DOCKER_BUILD_PULL="${DOCKER_BUILD_PULL:-false}"
|
||||||
@@ -63,7 +63,7 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Building image on ${BUILD_HOST}"
|
echo "Building image on ${BUILD_HOST}"
|
||||||
BUILD_COMMAND="set -e; echo 'Docker context:' \$(docker context show 2>/dev/null || true); docker info 2>/dev/null | sed -n '/HTTP Proxy:/p;/HTTPS Proxy:/p;/Name:/p'; cd '${BUILD_PATH}'; if ! docker image inspect '${BASE_IMAGE}' >/dev/null 2>&1 || ! docker run --rm '${BASE_IMAGE}' sh -lc \"command -v npm >/dev/null && sing-box version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'\"; then if [ '${AUTO_BUILD_RUNTIME_BASE}' = 'true' ]; then echo 'Runtime base image ${BASE_IMAGE} is missing or does not contain sing-box ${SINGBOX_VERSION}; building it now.'; BASE_IMAGE='${RUNTIME_BASE_SOURCE_IMAGE}' RUNTIME_BASE_IMAGE='${BASE_IMAGE}' SINGBOX_VERSION='${SINGBOX_VERSION}' ./scripts/build-runtime-base.sh; else echo 'Runtime base image ${BASE_IMAGE} is missing or does not contain sing-box ${SINGBOX_VERSION} on ${BUILD_HOST}.'; echo 'Seed it once with: ./scripts/build-runtime-base.sh'; exit 1; fi; fi; docker run --rm '${BASE_IMAGE}' sh -lc \"command -v npm >/dev/null && sing-box version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'\"; npm ci && npm run build:production && docker build --pull='${DOCKER_BUILD_PULL}' --build-arg NODE_BUILD_IMAGE='${NODE_BUILD_IMAGE}' --build-arg BASE_IMAGE='${BASE_IMAGE}' --build-arg SINGBOX_VERSION='${SINGBOX_VERSION}' --build-arg INSTALL_RUNTIME_DEPS='${INSTALL_RUNTIME_DEPS}' --build-arg INSTALL_SINGBOX='${INSTALL_SINGBOX}' -t '${GATEWAY_IMAGE}' . && docker run --rm --entrypoint sing-box '${GATEWAY_IMAGE}' version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'"
|
BUILD_COMMAND="set -e; echo 'Docker context:' \$(docker context show 2>/dev/null || true); docker info 2>/dev/null | sed -n '/HTTP Proxy:/p;/HTTPS Proxy:/p;/Name:/p'; cd '${BUILD_PATH}'; if ! docker image inspect '${BASE_IMAGE}' >/dev/null 2>&1 || ! docker run --rm '${BASE_IMAGE}' sh -lc \"command -v npm >/dev/null && sing-box version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'\"; then if [ '${AUTO_BUILD_RUNTIME_BASE}' = 'true' ]; then echo 'Runtime base image ${BASE_IMAGE} is missing or does not contain sing-box ${SINGBOX_VERSION}; building it now.'; BASE_IMAGE='${RUNTIME_BASE_SOURCE_IMAGE}' RUNTIME_BASE_IMAGE='${BASE_IMAGE}' SINGBOX_VERSION='${SINGBOX_VERSION}' ./scripts/build-runtime-base.sh; else echo 'Runtime base image ${BASE_IMAGE} is missing or does not contain sing-box ${SINGBOX_VERSION} on ${BUILD_HOST}.'; echo 'Seed it once with: ./scripts/build-runtime-base.sh'; exit 1; fi; fi; docker run --rm '${BASE_IMAGE}' sh -lc \"command -v npm >/dev/null && sing-box version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'\"; node scripts/check-sqlite-runtime.mjs && npm ci && npm run build:production && docker build --pull='${DOCKER_BUILD_PULL}' --build-arg NODE_BUILD_IMAGE='${NODE_BUILD_IMAGE}' --build-arg BASE_IMAGE='${BASE_IMAGE}' --build-arg SINGBOX_VERSION='${SINGBOX_VERSION}' --build-arg INSTALL_RUNTIME_DEPS='${INSTALL_RUNTIME_DEPS}' --build-arg INSTALL_SINGBOX='${INSTALL_SINGBOX}' -t '${GATEWAY_IMAGE}' . && docker run --rm --entrypoint sing-box '${GATEWAY_IMAGE}' version 2>&1 | grep -Fx 'sing-box version ${SINGBOX_VERSION}'"
|
||||||
if [ "${BUILD_HOST}" = "local" ]; then
|
if [ "${BUILD_HOST}" = "local" ]; then
|
||||||
bash -lc "${BUILD_COMMAND}"
|
bash -lc "${BUILD_COMMAND}"
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
BASE_IMAGE="${BASE_IMAGE:-mirror.gcr.io/library/debian:bookworm-slim}"
|
BASE_IMAGE="${BASE_IMAGE:-mirror.gcr.io/library/debian:bookworm-slim}"
|
||||||
|
NODE_BUILD_IMAGE="${NODE_BUILD_IMAGE:-node:24.21.0-bookworm}"
|
||||||
RUNTIME_BASE_IMAGE="${RUNTIME_BASE_IMAGE:-vpn-proxy-runtime-base:bookworm-slim}"
|
RUNTIME_BASE_IMAGE="${RUNTIME_BASE_IMAGE:-vpn-proxy-runtime-base:bookworm-slim}"
|
||||||
SINGBOX_VERSION="${SINGBOX_VERSION:-1.14.0-rc.5}"
|
SINGBOX_VERSION="${SINGBOX_VERSION:-1.14.0-rc.5}"
|
||||||
APT_MIRROR="${APT_MIRROR:-http://mirror.yandex.ru/debian}"
|
APT_MIRROR="${APT_MIRROR:-http://mirror.yandex.ru/debian}"
|
||||||
@@ -18,6 +19,7 @@ if [ -n "${HTTP_PROXY}" ]; then echo "HTTP proxy: ${HTTP_PROXY}"; fi
|
|||||||
if [ -n "${HTTPS_PROXY}" ]; then echo "HTTPS proxy: ${HTTPS_PROXY}"; fi
|
if [ -n "${HTTPS_PROXY}" ]; then echo "HTTPS proxy: ${HTTPS_PROXY}"; fi
|
||||||
|
|
||||||
docker build \
|
docker build \
|
||||||
|
--build-arg NODE_BUILD_IMAGE="${NODE_BUILD_IMAGE}" \
|
||||||
--build-arg BASE_IMAGE="${BASE_IMAGE}" \
|
--build-arg BASE_IMAGE="${BASE_IMAGE}" \
|
||||||
--build-arg SINGBOX_VERSION="${SINGBOX_VERSION}" \
|
--build-arg SINGBOX_VERSION="${SINGBOX_VERSION}" \
|
||||||
--build-arg APT_MIRROR="${APT_MIRROR}" \
|
--build-arg APT_MIRROR="${APT_MIRROR}" \
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
|
||||||
|
assert.equal(process.versions.node, '24.21.0', 'Harbor requires the pinned Node 24.21.0 runtime');
|
||||||
|
const db = new DatabaseSync(':memory:');
|
||||||
|
try {
|
||||||
|
const version = db.prepare('SELECT sqlite_version() AS version').get().version;
|
||||||
|
const [major, minor, patch] = version.split('.').map(Number);
|
||||||
|
assert.ok(major > 3 || (major === 3 && (minor > 51 || (minor === 51 && patch >= 3))),
|
||||||
|
'Harbor requires SQLite >= 3.51.3 with the WAL-reset fix');
|
||||||
|
db.exec('CREATE TABLE probe (bytes INTEGER NOT NULL) STRICT');
|
||||||
|
db.prepare('INSERT INTO probe VALUES (?)').run(9007199254740993n);
|
||||||
|
const statement = db.prepare('SELECT bytes FROM probe');
|
||||||
|
statement.setReadBigInts(true);
|
||||||
|
assert.equal(statement.get().bytes, 9007199254740993n);
|
||||||
|
console.log(`Harbor runtime: Node ${process.versions.node}, SQLite ${version}, ${process.platform}/${process.arch}`);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
@@ -52,7 +52,7 @@ export function affectedComponents(files) {
|
|||||||
const add = (...components) => components.forEach((component) => affected.add(component));
|
const add = (...components) => components.forEach((component) => affected.add(component));
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
if (file === VERSION_FILE) continue;
|
if (file === VERSION_FILE) continue;
|
||||||
if (/^(?:\.dockerignore$|package(?:-lock)?\.json$|tsconfig\.base\.json$|src\/shared\/)/.test(file)) add(...COMPONENTS);
|
if (/^(?:\.dockerignore$|\.node-version$|scripts\/check-sqlite-runtime\.mjs$|package(?:-lock)?\.json$|tsconfig\.base\.json$|src\/shared\/)/.test(file)) add(...COMPONENTS);
|
||||||
else if (/^(src\/web\/|public\/|monitoring\/grafana\/|index\.html$|tsconfig\.web\.json$|vite\.config\.[cm]?[jt]s$)/.test(file)) {
|
else if (/^(src\/web\/|public\/|monitoring\/grafana\/|index\.html$|tsconfig\.web\.json$|vite\.config\.[cm]?[jt]s$)/.test(file)) {
|
||||||
add('macClient', 'gatewayClient');
|
add('macClient', 'gatewayClient');
|
||||||
} else if (/^(src\/server\/|tsconfig\.server\.json$)/.test(file)) add('macClient', 'gatewayBackend');
|
} else if (/^(src\/server\/|tsconfig\.server\.json$)/.test(file)) add('macClient', 'gatewayBackend');
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ const noRuntimeImpact = [
|
|||||||
/^tools\/test-singbox-(?:client-rc|gateway-native-traffic|native-traffic)\.sh$/,
|
/^tools\/test-singbox-(?:client-rc|gateway-native-traffic|native-traffic)\.sh$/,
|
||||||
];
|
];
|
||||||
const foundation = [
|
const foundation = [
|
||||||
|
/^\.node-version$/,
|
||||||
|
/^scripts\/check-sqlite-runtime\.mjs$/,
|
||||||
/^\.dockerignore$/,
|
/^\.dockerignore$/,
|
||||||
/^\.gitea\/workflows\//,
|
/^\.gitea\/workflows\//,
|
||||||
/^Dockerfile(?:\.runtime-base)?$/,
|
/^Dockerfile(?:\.runtime-base)?$/,
|
||||||
@@ -33,6 +35,8 @@ const foundation = [
|
|||||||
/^tsconfig(?:\.[^.]+)?\.json$/,
|
/^tsconfig(?:\.[^.]+)?\.json$/,
|
||||||
];
|
];
|
||||||
const controlAndDataplane = [
|
const controlAndDataplane = [
|
||||||
|
/^src\/server\/services\/(?:sqlite|trafficHistoryStore|trafficHistoryService|trafficHistoryWorker)\.ts$/,
|
||||||
|
/^src\/shared\/trafficHistory\.ts$/,
|
||||||
/^buf\.gen\.yaml$/,
|
/^buf\.gen\.yaml$/,
|
||||||
/^proto\//,
|
/^proto\//,
|
||||||
new RegExp(`^src/server/main${CODE_EXTENSION}`),
|
new RegExp(`^src/server/main${CODE_EXTENSION}`),
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ import {
|
|||||||
createLiveTrafficService,
|
createLiveTrafficService,
|
||||||
} from './services/liveTrafficService.js';
|
} from './services/liveTrafficService.js';
|
||||||
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
||||||
|
import { createTrafficHistoryService } from './services/trafficHistoryService.js';
|
||||||
|
import { parseTrafficHistoryQuery } from '../shared/trafficHistory.js';
|
||||||
|
|
||||||
const socketPath = settings.dataplaneSocket;
|
const socketPath = settings.dataplaneSocket;
|
||||||
const trafficMode = settings.singboxTrafficSource as 'snapshot' | 'shadow' | 'native';
|
const trafficMode = settings.singboxTrafficSource as 'snapshot' | 'shadow' | 'native';
|
||||||
@@ -83,6 +85,10 @@ let liveTraffic = createLiveTrafficService({
|
|||||||
isRuntimeRunning: () => false,
|
isRuntimeRunning: () => false,
|
||||||
resolveOrigin,
|
resolveOrigin,
|
||||||
});
|
});
|
||||||
|
const trafficHistory = createTrafficHistoryService({
|
||||||
|
filePath: path.join(settings.dataDir, 'traffic.sqlite'),
|
||||||
|
source: () => liveTrafficSnapshot().source.state,
|
||||||
|
});
|
||||||
let ready = false;
|
let ready = false;
|
||||||
let trafficTimer: NodeJS.Timeout | null = null;
|
let trafficTimer: NodeJS.Timeout | null = null;
|
||||||
let domainTrafficTimer: NodeJS.Timeout | null = null;
|
let domainTrafficTimer: NodeJS.Timeout | null = null;
|
||||||
@@ -281,6 +287,10 @@ const server = http.createServer(async (req: IncomingMessage, res: ServerRespons
|
|||||||
if (req.method === 'GET' && req.url === '/traffic/live') {
|
if (req.method === 'GET' && req.url === '/traffic/live') {
|
||||||
return sendJson(res, 200, liveTrafficSnapshot());
|
return sendJson(res, 200, liveTrafficSnapshot());
|
||||||
}
|
}
|
||||||
|
const url = new URL(req.url || '/', 'http://localhost');
|
||||||
|
if (req.method === 'GET' && url.pathname === '/traffic/history') {
|
||||||
|
return sendJson(res, 200, await trafficHistory.query(parseTrafficHistoryQuery(url.searchParams)));
|
||||||
|
}
|
||||||
if (req.method === 'GET' && req.url === '/device-policy') {
|
if (req.method === 'GET' && req.url === '/device-policy') {
|
||||||
return sendJson(res, 200, devicePolicy.snapshot());
|
return sendJson(res, 200, devicePolicy.snapshot());
|
||||||
}
|
}
|
||||||
@@ -378,6 +388,7 @@ server.listen(socketPath, async () => {
|
|||||||
resolveOrigin,
|
resolveOrigin,
|
||||||
authorization: () => runtime.nativeApiSecret,
|
authorization: () => runtime.nativeApiSecret,
|
||||||
onProjection: (batch) => {
|
onProjection: (batch) => {
|
||||||
|
trafficHistory.enqueue(batch);
|
||||||
nativeDomainTraffic.ingestNative(batch);
|
nativeDomainTraffic.ingestNative(batch);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -419,6 +430,7 @@ async function shutdown() {
|
|||||||
if (trafficTimer) clearInterval(trafficTimer);
|
if (trafficTimer) clearInterval(trafficTimer);
|
||||||
if (domainTrafficTimer) clearInterval(domainTrafficTimer);
|
if (domainTrafficTimer) clearInterval(domainTrafficTimer);
|
||||||
await liveTraffic.stop();
|
await liveTraffic.stop();
|
||||||
|
await trafficHistory.close();
|
||||||
await runtime.shutdown();
|
await runtime.shutdown();
|
||||||
server.close(() => {
|
server.close(() => {
|
||||||
fs.rmSync(socketPath, { force: true });
|
fs.rmSync(socketPath, { force: true });
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import http from 'node:http';
|
import http from 'node:http';
|
||||||
import { HarborError } from '../shared/errors.js';
|
import { HarborError } from '../shared/errors.js';
|
||||||
|
import { historyQueryParams, type TrafficHistoryQuery } from '../shared/trafficHistory.js';
|
||||||
|
|
||||||
type SendDataplaneRequest = (
|
type SendDataplaneRequest = (
|
||||||
socketPath: string,
|
socketPath: string,
|
||||||
@@ -74,6 +75,7 @@ export function createDataplaneClient(socketPath: string, send: SendDataplaneReq
|
|||||||
observeTraffic: () => send(socketPath, '/device-traffic', 'GET'),
|
observeTraffic: () => send(socketPath, '/device-traffic', 'GET'),
|
||||||
observeDomainTraffic: () => send(socketPath, '/domain-traffic', 'GET'),
|
observeDomainTraffic: () => send(socketPath, '/domain-traffic', 'GET'),
|
||||||
observeLiveTraffic: () => send(socketPath, '/traffic/live', 'GET'),
|
observeLiveTraffic: () => send(socketPath, '/traffic/live', 'GET'),
|
||||||
|
observeTrafficHistory: (query: TrafficHistoryQuery) => send(socketPath, `/traffic/history?${historyQueryParams(query)}`, 'GET', null, 12_000),
|
||||||
observeDevicePolicy: () => send(socketPath, '/device-policy', 'GET'),
|
observeDevicePolicy: () => send(socketPath, '/device-policy', 'GET'),
|
||||||
applyDevicePolicies: (devices: unknown) => send(socketPath, '/device-policy', 'PUT', { devices }),
|
applyDevicePolicies: (devices: unknown) => send(socketPath, '/device-policy', 'PUT', { devices }),
|
||||||
runConnectivityDiagnostics: async (services: unknown = [], target: unknown = null) => {
|
runConnectivityDiagnostics: async (services: unknown = [], target: unknown = null) => {
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||||
|
import { HarborError } from '../../../shared/errors.js';
|
||||||
|
import { assertTrafficHistorySnapshot, emptyTrafficHistory, parseTrafficHistoryQuery, type TrafficHistoryQuery } from '../../../shared/trafficHistory.js';
|
||||||
|
import { sendJson } from '../response.js';
|
||||||
|
|
||||||
|
export function createTrafficHistoryRoute({ readHistory, deviceInventory }: {
|
||||||
|
readHistory: ((query: TrafficHistoryQuery) => Promise<unknown>) | null;
|
||||||
|
deviceInventory?: { snapshot(): unknown } | null;
|
||||||
|
}) {
|
||||||
|
return {
|
||||||
|
async handle(req: IncomingMessage, res: ServerResponse) {
|
||||||
|
const url = new URL(req.url || '/', 'http://localhost');
|
||||||
|
if (url.pathname !== '/api/traffic/history') return false;
|
||||||
|
if (req.method !== 'GET') throw new HarborError('ENDPOINT_NOT_FOUND');
|
||||||
|
let query: TrafficHistoryQuery;
|
||||||
|
try { query = parseTrafficHistoryQuery(url.searchParams); }
|
||||||
|
catch (cause) { throw new HarborError('REQUEST_INVALID', { cause }); }
|
||||||
|
let snapshot;
|
||||||
|
try {
|
||||||
|
snapshot = readHistory ? assertTrafficHistorySnapshot(await readHistory(query)) : emptyTrafficHistory(query);
|
||||||
|
} catch {
|
||||||
|
snapshot = emptyTrafficHistory(query, 'stale');
|
||||||
|
snapshot.storage = { status: 'error', errorCode: 'TRAFFIC_HISTORY_UNAVAILABLE' };
|
||||||
|
snapshot.coverage.partial = true;
|
||||||
|
}
|
||||||
|
const inventory = deviceInventory?.snapshot() as { devices?: Array<{ id: string; alias?: string; hostname?: string; ip?: string }> } | undefined;
|
||||||
|
const labels = new Map((inventory?.devices || []).map((device) => [device.id, device.alias || device.hostname || device.ip]));
|
||||||
|
snapshot.origins = snapshot.origins.map((origin) => ({ ...origin, label: labels.get(origin.id) || origin.label }));
|
||||||
|
sendJson(res, 200, snapshot);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
+26
-136
@@ -39,18 +39,11 @@ import {
|
|||||||
} from '../shared/contracts/state.js';
|
} from '../shared/contracts/state.js';
|
||||||
import { serverIdentityKey } from '../shared/serverIdentity.js';
|
import { serverIdentityKey } from '../shared/serverIdentity.js';
|
||||||
import { HarborError, normalizeHarborError } from '../shared/errors.js';
|
import { HarborError, normalizeHarborError } from '../shared/errors.js';
|
||||||
import {
|
import { openHarborStorage } from './services/harborStorage.js';
|
||||||
atomicWriteFile,
|
|
||||||
createJsonStore,
|
|
||||||
createStateStore,
|
|
||||||
} from './services/stateStore.js';
|
|
||||||
import { createDevicePolicyService } from './services/devicePolicyService.js';
|
import { createDevicePolicyService } from './services/devicePolicyService.js';
|
||||||
import {
|
import {
|
||||||
createDeviceInventoryService,
|
createDeviceInventoryService,
|
||||||
createVendorLookup,
|
createVendorLookup,
|
||||||
DEVICE_INVENTORY_SCHEMA_VERSION,
|
|
||||||
migrateDeviceInventoryState,
|
|
||||||
type InventoryState,
|
|
||||||
} from './services/deviceInventoryService.js';
|
} from './services/deviceInventoryService.js';
|
||||||
import { buildVersionInfo } from './version.js';
|
import { buildVersionInfo } from './version.js';
|
||||||
import { createConnectivityDiagnosticsService } from './services/connectivityDiagnosticsService.js';
|
import { createConnectivityDiagnosticsService } from './services/connectivityDiagnosticsService.js';
|
||||||
@@ -86,6 +79,9 @@ import { createGatewayPresenceRoute } from './http/routes/gatewayPresenceRoute.j
|
|||||||
import { createSharedProxyRoute } from './http/routes/sharedProxyRoute.js';
|
import { createSharedProxyRoute } from './http/routes/sharedProxyRoute.js';
|
||||||
import { createVersionRoute } from './http/routes/versionRoute.js';
|
import { createVersionRoute } from './http/routes/versionRoute.js';
|
||||||
import { createLiveTrafficRoute } from './http/routes/liveTrafficRoute.js';
|
import { createLiveTrafficRoute } from './http/routes/liveTrafficRoute.js';
|
||||||
|
import { createTrafficHistoryRoute } from './http/routes/trafficHistoryRoute.js';
|
||||||
|
import { createTrafficHistoryService } from './services/trafficHistoryService.js';
|
||||||
|
import type { LiveTrafficSourceState } from '../shared/liveTraffic.js';
|
||||||
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
||||||
import { createFailoverService } from './features/failover/failoverService.js';
|
import { createFailoverService } from './features/failover/failoverService.js';
|
||||||
import { createFailoverRoute } from './http/routes/failoverRoute.js';
|
import { createFailoverRoute } from './http/routes/failoverRoute.js';
|
||||||
@@ -111,50 +107,12 @@ function errorMessage(error: unknown) {
|
|||||||
|
|
||||||
fs.mkdirSync(settings.dataDir, { recursive: true });
|
fs.mkdirSync(settings.dataDir, { recursive: true });
|
||||||
|
|
||||||
const stateFileExisted = fs.existsSync(settings.statePath);
|
const storage = openHarborStorage(settings.dataDir);
|
||||||
const legacyStateBytes = stateFileExisted
|
const stateStore = storage.state;
|
||||||
? fs.readFileSync(settings.statePath, 'utf8')
|
const deviceStore = storage.devices;
|
||||||
: null;
|
const initialStoredState = stateStore.read();
|
||||||
let legacyStateRecord: Record<string, unknown> = {};
|
if (storage.imported) console.log('[storage] SQLite migration committed; original JSON files retained as backups');
|
||||||
try {
|
const activityJournal = createActivityJournalService({ db: storage.db });
|
||||||
legacyStateRecord = record(legacyStateBytes === null ? null : JSON.parse(legacyStateBytes));
|
|
||||||
} catch {}
|
|
||||||
const legacyStateVersion = Number.isSafeInteger(legacyStateRecord.schemaVersion)
|
|
||||||
? Number(legacyStateRecord.schemaVersion)
|
|
||||||
: 0;
|
|
||||||
const legacySubscriptionCacheBytes = fs.existsSync(settings.subscriptionCachePath)
|
|
||||||
? fs.readFileSync(settings.subscriptionCachePath, 'utf8')
|
|
||||||
: null;
|
|
||||||
const subscriptionCacheStore = createJsonStore({
|
|
||||||
filePath: settings.subscriptionCachePath,
|
|
||||||
defaultValue: null,
|
|
||||||
});
|
|
||||||
const rawLegacySubscriptionCache = subscriptionCacheStore.read();
|
|
||||||
const legacyCacheRecord = record(rawLegacySubscriptionCache);
|
|
||||||
const legacyStateSubscriptionUrl = String(legacyStateRecord.subscriptionUrl || '').trim();
|
|
||||||
const legacyCacheSubscriptionUrl = String(legacyCacheRecord.url || '').trim();
|
|
||||||
const legacyCacheOwnerMismatch = legacyStateVersion < 5
|
|
||||||
&& Boolean(legacyCacheRecord.config)
|
|
||||||
&& (legacyStateSubscriptionUrl
|
|
||||||
? legacyCacheSubscriptionUrl !== legacyStateSubscriptionUrl
|
|
||||||
: !legacyCacheSubscriptionUrl);
|
|
||||||
let legacySubscriptionCache = rawLegacySubscriptionCache;
|
|
||||||
let legacySubscriptionCacheRejected = Boolean(subscriptionCacheStore.recovery);
|
|
||||||
if (legacyCacheOwnerMismatch) {
|
|
||||||
legacySubscriptionCache = null;
|
|
||||||
} else if (legacyCacheRecord.config) {
|
|
||||||
try {
|
|
||||||
legacySubscriptionCache = {
|
|
||||||
...legacyCacheRecord,
|
|
||||||
...normalizeSubscriptionConfig(legacyCacheRecord.config),
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
legacySubscriptionCache = null;
|
|
||||||
legacySubscriptionCacheRejected = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const stateStore = createStateStore(settings.statePath, { legacySubscriptionCache });
|
|
||||||
const activityJournal = createActivityJournalService({ filePath: settings.activityJournalPath });
|
|
||||||
const appendJournal = (event: ActivityJournalEventInput) => {
|
const appendJournal = (event: ActivityJournalEventInput) => {
|
||||||
try {
|
try {
|
||||||
activityJournal.append(event);
|
activityJournal.append(event);
|
||||||
@@ -162,83 +120,6 @@ const appendJournal = (event: ActivityJournalEventInput) => {
|
|||||||
console.warn(`[journal] событие не сохранено: ${errorMessage(error)}`);
|
console.warn(`[journal] событие не сохранено: ${errorMessage(error)}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
const deviceStore = createJsonStore<InventoryState>({
|
|
||||||
filePath: settings.deviceStatePath,
|
|
||||||
defaultValue: migrateDeviceInventoryState({}),
|
|
||||||
migrate: migrateDeviceInventoryState,
|
|
||||||
initializeMissing: true,
|
|
||||||
backupWhen: () => true,
|
|
||||||
});
|
|
||||||
deviceStore.read();
|
|
||||||
if (deviceStore.migration) {
|
|
||||||
console.log(`[storage] devices migrated to v${DEVICE_INVENTORY_SCHEMA_VERSION}; backup: ${deviceStore.migration.backupPath}`);
|
|
||||||
}
|
|
||||||
if (deviceStore.recovery) {
|
|
||||||
console.warn(`[storage] corrupt devices recovered; backup: ${deviceStore.recovery.backupPath}`);
|
|
||||||
}
|
|
||||||
let initialStoredState = stateStore.read();
|
|
||||||
if (stateStore.migration) {
|
|
||||||
console.log(`[storage] state migrated to v${stateStore.migration.toVersion}; backup: ${stateStore.migration.backupPath}`);
|
|
||||||
}
|
|
||||||
if (stateStore.recovery) {
|
|
||||||
console.warn(`[storage] corrupt state recovered; backup: ${stateStore.recovery.backupPath}`);
|
|
||||||
}
|
|
||||||
if (subscriptionCacheStore.recovery) {
|
|
||||||
console.warn(`[storage] corrupt subscription cache recovered; backup: ${subscriptionCacheStore.recovery.backupPath}`);
|
|
||||||
}
|
|
||||||
const rejectedLegacyMigration = legacySubscriptionCacheRejected
|
|
||||||
&& (
|
|
||||||
!stateFileExisted
|
|
||||||
|| Boolean(stateStore.recovery)
|
|
||||||
|| Boolean(stateStore.migration && stateStore.migration.fromVersion < 5)
|
|
||||||
);
|
|
||||||
const mismatchedLegacyMigration = legacyCacheOwnerMismatch
|
|
||||||
&& (
|
|
||||||
!stateFileExisted
|
|
||||||
|| Boolean(stateStore.recovery)
|
|
||||||
|| Boolean(stateStore.migration && stateStore.migration.fromVersion < 5)
|
|
||||||
);
|
|
||||||
if (rejectedLegacyMigration) {
|
|
||||||
initialStoredState = stateStore.update((state) => ({
|
|
||||||
...state,
|
|
||||||
profiles: [],
|
|
||||||
desiredProfileId: '',
|
|
||||||
appliedProfileId: '',
|
|
||||||
appliedServerId: '',
|
|
||||||
appliedServerSnapshot: null,
|
|
||||||
connectionDesired: 'stopped',
|
|
||||||
}));
|
|
||||||
removeSingboxConfig();
|
|
||||||
} else if (mismatchedLegacyMigration) {
|
|
||||||
initialStoredState = stateStore.update((state) => ({
|
|
||||||
...state,
|
|
||||||
appliedProfileId: '',
|
|
||||||
appliedServerId: '',
|
|
||||||
appliedServerSnapshot: null,
|
|
||||||
connectionDesired: 'stopped',
|
|
||||||
}));
|
|
||||||
removeSingboxConfig();
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
legacySubscriptionCacheBytes !== null
|
|
||||||
&& (
|
|
||||||
Boolean(subscriptionCacheStore.recovery)
|
|
||||||
|| (
|
|
||||||
Boolean(legacyCacheRecord.config)
|
|
||||||
&& (
|
|
||||||
legacySubscriptionCacheRejected
|
|
||||||
|| legacyCacheOwnerMismatch
|
|
||||||
|| normalizeStoredState(initialStoredState).profiles.some((profile) => profile.subscriptionConfig)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
const backupPath = subscriptionCacheStore.recovery?.backupPath
|
|
||||||
|| `${settings.subscriptionCachePath}.backup-v1-${new Date().toISOString().replace(/[:.]/g, '-')}`;
|
|
||||||
if (!subscriptionCacheStore.recovery) atomicWriteFile(backupPath, legacySubscriptionCacheBytes);
|
|
||||||
subscriptionCacheStore.remove();
|
|
||||||
console.log(`[storage] legacy subscription cache migrated; backup: ${backupPath}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function readProfileConfig(profileId = '') {
|
function readProfileConfig(profileId = '') {
|
||||||
const state = normalizeStoredState(stateStore.read());
|
const state = normalizeStoredState(stateStore.read());
|
||||||
@@ -262,11 +143,17 @@ function selectRuntime() {
|
|||||||
throw new Error('Harbor runtime is not configured');
|
throw new Error('Harbor runtime is not configured');
|
||||||
}
|
}
|
||||||
const singboxRuntime = selectRuntime();
|
const singboxRuntime = selectRuntime();
|
||||||
const clientLiveTraffic = settings.appMode === 'client'
|
let clientLiveTraffic: ReturnType<typeof import('./services/liveTrafficService.js').createLiveTrafficService> | null = null;
|
||||||
|
const clientHistory = settings.appMode === 'client' ? createTrafficHistoryService({
|
||||||
|
filePath: path.join(settings.dataDir, 'traffic.sqlite'),
|
||||||
|
source: (): LiveTrafficSourceState => clientLiveTraffic?.snapshot().source.state || 'disabled',
|
||||||
|
}) : null;
|
||||||
|
clientLiveTraffic = settings.appMode === 'client'
|
||||||
? (await import('./services/liveTrafficService.js')).createLiveTrafficService({
|
? (await import('./services/liveTrafficService.js')).createLiveTrafficService({
|
||||||
port: settings.singboxNativeApiPort,
|
port: settings.singboxNativeApiPort,
|
||||||
enabled: settings.singboxTrafficSource === 'native',
|
enabled: settings.singboxTrafficSource === 'native',
|
||||||
isRuntimeRunning: () => Boolean(localRuntime?.running),
|
isRuntimeRunning: () => Boolean(localRuntime?.running),
|
||||||
|
onProjection: (batch) => clientHistory?.enqueue(batch),
|
||||||
})
|
})
|
||||||
: null;
|
: null;
|
||||||
const liveTraffic = clientLiveTraffic || (remoteDataplane ? {
|
const liveTraffic = clientLiveTraffic || (remoteDataplane ? {
|
||||||
@@ -354,12 +241,7 @@ function requireLocalConnectivityDiagnostics() {
|
|||||||
}
|
}
|
||||||
let deviceDiscoveryTimer: NodeJS.Timeout | null = null;
|
let deviceDiscoveryTimer: NodeJS.Timeout | null = null;
|
||||||
let controlOperation: Promise<unknown> = Promise.resolve();
|
let controlOperation: Promise<unknown> = Promise.resolve();
|
||||||
let operationState: OperationState = stateStore.recovery ? {
|
let operationState: OperationState = { kind: null, status: 'idle', startedAt: null, error: null };
|
||||||
kind: 'storage-recovery',
|
|
||||||
status: 'failed',
|
|
||||||
startedAt: stateStore.recovery.recoveredAt,
|
|
||||||
error: `Повреждённый state сохранён: ${path.basename(stateStore.recovery.backupPath)}`,
|
|
||||||
} : { kind: null, status: 'idle', startedAt: null, error: null };
|
|
||||||
let revision = normalizeStoredState(initialStoredState).revision;
|
let revision = normalizeStoredState(initialStoredState).revision;
|
||||||
const gatewayAutoService = createGatewayAutoService({
|
const gatewayAutoService = createGatewayAutoService({
|
||||||
appMode: settings.appMode,
|
appMode: settings.appMode,
|
||||||
@@ -557,6 +439,11 @@ const liveTrafficRoute = createLiveTrafficRoute({
|
|||||||
readBody,
|
readBody,
|
||||||
sendState: (res) => stateRoute.send(res),
|
sendState: (res) => stateRoute.send(res),
|
||||||
});
|
});
|
||||||
|
const trafficHistoryRoute = createTrafficHistoryRoute({
|
||||||
|
readHistory: clientHistory ? (query) => clientHistory.query(query)
|
||||||
|
: remoteRuntime ? (query) => remoteRuntime.observeTrafficHistory(query) : null,
|
||||||
|
deviceInventory,
|
||||||
|
});
|
||||||
const subscriptionValidationRoute = createSubscriptionValidationRoute({
|
const subscriptionValidationRoute = createSubscriptionValidationRoute({
|
||||||
validateSubscription: createValidateSubscription(fetchSubscription),
|
validateSubscription: createValidateSubscription(fetchSubscription),
|
||||||
readBody,
|
readBody,
|
||||||
@@ -1011,6 +898,7 @@ async function handleApi(req: IncomingMessage, res: ServerResponse) {
|
|||||||
if (await versionRoute.handle(req, res)) return;
|
if (await versionRoute.handle(req, res)) return;
|
||||||
|
|
||||||
if (await liveTrafficRoute.handle(req, res)) return;
|
if (await liveTrafficRoute.handle(req, res)) return;
|
||||||
|
if (await trafficHistoryRoute.handle(req, res)) return;
|
||||||
|
|
||||||
if (await sharedProxyRoute.handle(req, res)) return;
|
if (await sharedProxyRoute.handle(req, res)) return;
|
||||||
|
|
||||||
@@ -1062,7 +950,9 @@ async function shutdown() {
|
|||||||
if (deviceDiscoveryTimer) clearInterval(deviceDiscoveryTimer);
|
if (deviceDiscoveryTimer) clearInterval(deviceDiscoveryTimer);
|
||||||
await gatewayFailover?.shutdown().catch((error) => console.warn(`[control] failover shutdown: ${errorMessage(error)}`));
|
await gatewayFailover?.shutdown().catch((error) => console.warn(`[control] failover shutdown: ${errorMessage(error)}`));
|
||||||
await clientLiveTraffic?.stop().catch((error) => console.warn(`[control] traffic shutdown: ${errorMessage(error)}`));
|
await clientLiveTraffic?.stop().catch((error) => console.warn(`[control] traffic shutdown: ${errorMessage(error)}`));
|
||||||
|
await clientHistory?.close();
|
||||||
await serializeControl(() => singboxRuntime.shutdown());
|
await serializeControl(() => singboxRuntime.shutdown());
|
||||||
|
storage.close();
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,151 +1,75 @@
|
|||||||
import crypto from 'node:crypto';
|
import crypto from 'node:crypto';
|
||||||
import fs from 'node:fs';
|
import type { DatabaseSync } from 'node:sqlite';
|
||||||
import {
|
import {
|
||||||
ACTIVITY_JOURNAL_MAX_EVENTS,
|
ACTIVITY_JOURNAL_MAX_EVENTS,
|
||||||
ACTIVITY_JOURNAL_RETENTION_DAYS,
|
ACTIVITY_JOURNAL_RETENTION_DAYS,
|
||||||
normalizeActivityEventInput,
|
normalizeActivityEventInput,
|
||||||
normalizeStoredActivityEvent,
|
|
||||||
type ActivityJournalEvent,
|
type ActivityJournalEvent,
|
||||||
type ActivityJournalEventInput,
|
type ActivityJournalEventInput,
|
||||||
type ActivityJournalPage,
|
type ActivityJournalPage,
|
||||||
} from '../../shared/activityJournal.js';
|
} from '../../shared/activityJournal.js';
|
||||||
import { createJsonStore } from './stateStore.js';
|
import { transaction } from './sqlite.js';
|
||||||
|
|
||||||
interface JournalState {
|
export function createActivityJournalService({ db, now = () => new Date() }: {
|
||||||
schemaVersion: 1;
|
db: DatabaseSync;
|
||||||
events: ActivityJournalEvent[];
|
|
||||||
}
|
|
||||||
|
|
||||||
const migrateJournal = (value: unknown): JournalState => {
|
|
||||||
const candidate = value && typeof value === 'object' && !Array.isArray(value)
|
|
||||||
? value as Record<string, unknown>
|
|
||||||
: {};
|
|
||||||
return {
|
|
||||||
schemaVersion: 1,
|
|
||||||
events: (Array.isArray(candidate.events) ? candidate.events : [])
|
|
||||||
.map(normalizeStoredActivityEvent)
|
|
||||||
.filter((event): event is ActivityJournalEvent => Boolean(event)),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export function createActivityJournalService({
|
|
||||||
filePath,
|
|
||||||
now = () => new Date(),
|
|
||||||
}: {
|
|
||||||
filePath: string;
|
|
||||||
now?: () => Date;
|
now?: () => Date;
|
||||||
}) {
|
}) {
|
||||||
const store = createJsonStore<JournalState>({
|
|
||||||
filePath,
|
|
||||||
defaultValue: { schemaVersion: 1, events: [] },
|
|
||||||
migrate: migrateJournal,
|
|
||||||
});
|
|
||||||
let recoveryRecorded = false;
|
|
||||||
let writeFailed = false;
|
let writeFailed = false;
|
||||||
|
const insert = db.prepare('INSERT INTO journal(id, occurred_at, dedupe_key, value) VALUES (?, ?, ?, ?) ON CONFLICT(dedupe_key) DO NOTHING');
|
||||||
function retained(events: ActivityJournalEvent[]) {
|
function prune() {
|
||||||
const cutoff = now().getTime() - ACTIVITY_JOURNAL_RETENTION_DAYS * 86_400_000;
|
const cutoff = new Date(now().getTime() - ACTIVITY_JOURNAL_RETENTION_DAYS * 86_400_000).toISOString();
|
||||||
return events
|
db.prepare('DELETE FROM journal WHERE occurred_at < ?').run(cutoff);
|
||||||
.filter(({ occurredAt }) => Date.parse(occurredAt) >= cutoff)
|
db.prepare(`DELETE FROM journal WHERE sequence IN (
|
||||||
.slice(-ACTIVITY_JOURNAL_MAX_EVENTS);
|
SELECT sequence FROM journal ORDER BY sequence DESC LIMIT -1 OFFSET ?
|
||||||
|
)`).run(ACTIVITY_JOURNAL_MAX_EVENTS);
|
||||||
}
|
}
|
||||||
|
|
||||||
function append(value: ActivityJournalEventInput) {
|
function append(value: ActivityJournalEventInput) {
|
||||||
const input = normalizeActivityEventInput(value);
|
const input = normalizeActivityEventInput(value);
|
||||||
const storedInput = input.dedupeKey ? {
|
const event: ActivityJournalEvent = {
|
||||||
...input,
|
...input,
|
||||||
dedupeKey: `${input.type}:sha256:${crypto.createHash('sha256').update(input.dedupeKey).digest('hex')}`,
|
|
||||||
} : input;
|
|
||||||
let appended: ActivityJournalEvent | null = null;
|
|
||||||
try {
|
|
||||||
store.update((state) => {
|
|
||||||
const events = retained(state.events);
|
|
||||||
if (storedInput.dedupeKey && events.some(({ dedupeKey }) => dedupeKey === storedInput.dedupeKey)) {
|
|
||||||
return { schemaVersion: 1, events };
|
|
||||||
}
|
|
||||||
appended = {
|
|
||||||
id: crypto.randomUUID(),
|
id: crypto.randomUUID(),
|
||||||
occurredAt: now().toISOString(),
|
occurredAt: now().toISOString(),
|
||||||
...storedInput,
|
dedupeKey: input.dedupeKey
|
||||||
|
? `${input.type}:sha256:${crypto.createHash('sha256').update(input.dedupeKey).digest('hex')}`
|
||||||
|
: null,
|
||||||
};
|
};
|
||||||
return { schemaVersion: 1, events: retained([...events, appended]) };
|
try {
|
||||||
|
const appended = transaction(db, () => {
|
||||||
|
prune();
|
||||||
|
const { changes } = insert.run(event.id, event.occurredAt, event.dedupeKey, JSON.stringify(event));
|
||||||
|
prune();
|
||||||
|
return Number(changes) ? event : null;
|
||||||
});
|
});
|
||||||
writeFailed = false;
|
writeFailed = false;
|
||||||
|
return appended;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
writeFailed = true;
|
writeFailed = true;
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
return appended;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function ensureRecoveryEvent() {
|
|
||||||
if (!store.recovery || recoveryRecorded) return;
|
|
||||||
append({
|
|
||||||
type: 'journal.recovered',
|
|
||||||
severity: 'warning',
|
|
||||||
source: 'storage',
|
|
||||||
dedupeKey: `journal.recovered:${store.recovery.recoveredAt}`,
|
|
||||||
data: {},
|
|
||||||
});
|
|
||||||
recoveryRecorded = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function page(limitValue: unknown = 50, cursorValue: unknown = null): ActivityJournalPage {
|
function page(limitValue: unknown = 50, cursorValue: unknown = null): ActivityJournalPage {
|
||||||
|
const base = { retentionDays: ACTIVITY_JOURNAL_RETENTION_DAYS, generatedAt: now().toISOString() } as const;
|
||||||
try {
|
try {
|
||||||
let state = store.read();
|
transaction(db, prune);
|
||||||
ensureRecoveryEvent();
|
|
||||||
if (store.recovery) state = store.read();
|
|
||||||
const retainedEvents = retained(state.events);
|
|
||||||
if (retainedEvents.length !== state.events.length) {
|
|
||||||
state = store.update(() => ({ schemaVersion: 1, events: retainedEvents }));
|
|
||||||
}
|
|
||||||
const events = [...state.events].reverse();
|
|
||||||
const limit = Math.min(100, Math.max(1, Number.isSafeInteger(limitValue) ? Number(limitValue) : 50));
|
const limit = Math.min(100, Math.max(1, Number.isSafeInteger(limitValue) ? Number(limitValue) : 50));
|
||||||
const cursor = typeof cursorValue === 'string' ? cursorValue : '';
|
const cursor = typeof cursorValue === 'string' ? cursorValue : '';
|
||||||
const cursorIndex = cursor ? events.findIndex(({ id }) => id === cursor) : -1;
|
const before = cursor ? db.prepare('SELECT sequence FROM journal WHERE id = ?').get(cursor) : null;
|
||||||
if (cursor && cursorIndex < 0) return {
|
const rows = cursor && !before ? [] : db.prepare(`
|
||||||
events: [],
|
SELECT id, value FROM journal WHERE (? IS NULL OR sequence < ?) ORDER BY sequence DESC LIMIT ?
|
||||||
nextCursor: null,
|
`).all(before?.sequence ?? null, before?.sequence ?? null, limit + 1);
|
||||||
retentionDays: 30,
|
const selected = rows.slice(0, limit);
|
||||||
generatedAt: now().toISOString(),
|
|
||||||
storage: writeFailed
|
|
||||||
? { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' }
|
|
||||||
: { status: 'ready', errorCode: null },
|
|
||||||
};
|
|
||||||
const safeStart = cursorIndex + 1;
|
|
||||||
const selected = events.slice(safeStart, safeStart + limit);
|
|
||||||
return {
|
return {
|
||||||
events: selected.map((event) => ({ ...event, dedupeKey: null })),
|
...base,
|
||||||
nextCursor: safeStart + selected.length < events.length ? selected.at(-1)?.id || null : null,
|
events: selected.map((row) => ({ ...JSON.parse(String(row.value)) as ActivityJournalEvent, dedupeKey: null })),
|
||||||
retentionDays: 30,
|
nextCursor: rows.length > limit ? String(selected.at(-1)?.id) : null,
|
||||||
generatedAt: now().toISOString(),
|
|
||||||
storage: writeFailed
|
storage: writeFailed
|
||||||
? { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' }
|
? { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' }
|
||||||
: { status: 'ready', errorCode: null },
|
: { status: 'ready', errorCode: null },
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return {
|
return { ...base, events: [], nextCursor: null, storage: { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' } };
|
||||||
events: [],
|
|
||||||
nextCursor: null,
|
|
||||||
retentionDays: 30,
|
|
||||||
generatedAt: now().toISOString(),
|
|
||||||
storage: { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' },
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fs.existsSync(filePath)) {
|
|
||||||
try {
|
|
||||||
const state = store.read();
|
|
||||||
const retainedEvents = retained(state.events);
|
|
||||||
if (retainedEvents.length !== state.events.length) {
|
|
||||||
store.update(() => ({ schemaVersion: 1, events: retainedEvents }));
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
writeFailed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { append, page };
|
return { append, page };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import type { DatabaseSync } from 'node:sqlite';
|
||||||
|
import { normalizeStoredActivityEvent, type ActivityJournalEvent } from '../../shared/activityJournal.js';
|
||||||
|
import { normalizeSubscriptionConfig } from '../subscription.js';
|
||||||
|
import { migrateDeviceInventoryState } from './deviceInventoryService.js';
|
||||||
|
import { migrateStoredState } from './stateStore.js';
|
||||||
|
import { openSqlite, transaction } from './sqlite.js';
|
||||||
|
|
||||||
|
function object(value: unknown): Record<string, unknown> {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||||
|
throw new Error('Invalid stored Harbor document');
|
||||||
|
}
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function legacyDocument(directory: string, name: string) {
|
||||||
|
const file = path.join(directory, name);
|
||||||
|
if (!fs.existsSync(file)) return null;
|
||||||
|
// Never run the legacy JSON recovery writer during import: originals are the backup.
|
||||||
|
try {
|
||||||
|
const value: unknown = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||||
|
return value === null && name === 'subscription-cache.json' ? null : object(value);
|
||||||
|
}
|
||||||
|
catch { throw new Error(`Cannot migrate ${name}: invalid JSON document; original retained`); }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createSqliteDocumentStore<T>(db: DatabaseSync, key: string, migrate: (value: unknown) => T) {
|
||||||
|
const select = db.prepare('SELECT value FROM documents WHERE key = ?');
|
||||||
|
const save = db.prepare('INSERT INTO documents(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
|
||||||
|
function read(): T {
|
||||||
|
const row = select.get(key);
|
||||||
|
if (!row || typeof row.value !== 'string') throw new Error(`Missing Harbor document: ${key}`);
|
||||||
|
return migrate(JSON.parse(row.value));
|
||||||
|
}
|
||||||
|
function write(value: T) {
|
||||||
|
const next = migrate(structuredClone(value));
|
||||||
|
save.run(key, JSON.stringify(next));
|
||||||
|
return structuredClone(next);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
read,
|
||||||
|
write,
|
||||||
|
update: (change: (value: T) => T) => transaction(db, () => {
|
||||||
|
const next = change(read());
|
||||||
|
if (next && typeof next === 'object' && 'then' in next) throw new TypeError('State store mutator must be synchronous');
|
||||||
|
return write(next);
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function openHarborStorage(directory: string) {
|
||||||
|
const db = openSqlite(path.join(directory, 'harbor.sqlite'));
|
||||||
|
let imported = false;
|
||||||
|
try {
|
||||||
|
const version = Number(db.prepare('PRAGMA user_version').get()?.user_version);
|
||||||
|
if (version !== 0 && version !== 1) throw new Error(`Unsupported Harbor database version: ${version}`);
|
||||||
|
transaction(db, () => {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS documents (key TEXT PRIMARY KEY, value TEXT NOT NULL CHECK(json_valid(value))) STRICT;
|
||||||
|
CREATE TABLE IF NOT EXISTS journal (
|
||||||
|
sequence INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, occurred_at TEXT NOT NULL,
|
||||||
|
dedupe_key TEXT UNIQUE, value TEXT NOT NULL CHECK(json_valid(value))
|
||||||
|
) STRICT;
|
||||||
|
CREATE INDEX IF NOT EXISTS journal_time ON journal(occurred_at);
|
||||||
|
`);
|
||||||
|
const initialized = db.prepare("SELECT value FROM documents WHERE key = 'storage-version'").get();
|
||||||
|
if (!initialized) {
|
||||||
|
// All inputs are read and normalized before any imported record is committed.
|
||||||
|
const rawState = legacyDocument(directory, 'state.json') || {};
|
||||||
|
const rawDevices = legacyDocument(directory, 'devices.json') || {};
|
||||||
|
const rawJournal = legacyDocument(directory, 'activity-journal.json');
|
||||||
|
const stateVersion = Number(rawState.schemaVersion || 0);
|
||||||
|
const legacyCache = stateVersion < 5 && !Array.isArray(rawState.profiles)
|
||||||
|
? legacyDocument(directory, 'subscription-cache.json')
|
||||||
|
: null;
|
||||||
|
let cache: unknown = null;
|
||||||
|
if (legacyCache?.config) {
|
||||||
|
const stateUrl = String(rawState.subscriptionUrl || '').trim();
|
||||||
|
const cacheUrl = String(legacyCache.url || '').trim();
|
||||||
|
if (!cacheUrl || (stateUrl && cacheUrl !== stateUrl)) {
|
||||||
|
throw new Error('Cannot migrate subscription cache: owner mismatch; originals retained');
|
||||||
|
}
|
||||||
|
cache = { ...legacyCache, ...normalizeSubscriptionConfig(legacyCache.config) };
|
||||||
|
}
|
||||||
|
const state = migrateStoredState(rawState, cache);
|
||||||
|
const devices = migrateDeviceInventoryState(rawDevices);
|
||||||
|
let events: ActivityJournalEvent[] = [];
|
||||||
|
if (rawJournal) {
|
||||||
|
if (rawJournal.schemaVersion !== 1 || !Array.isArray(rawJournal.events)) {
|
||||||
|
throw new Error('Unsupported activity journal document; original retained');
|
||||||
|
}
|
||||||
|
events = rawJournal.events.map((raw) => {
|
||||||
|
const event = normalizeStoredActivityEvent(raw);
|
||||||
|
if (!event) throw new Error('Invalid activity journal event; original retained');
|
||||||
|
return event;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const insert = db.prepare('INSERT INTO documents(key, value) VALUES (?, ?)');
|
||||||
|
insert.run('state', JSON.stringify(state));
|
||||||
|
insert.run('devices', JSON.stringify(devices));
|
||||||
|
const insertEvent = db.prepare('INSERT INTO journal(id, occurred_at, dedupe_key, value) VALUES (?, ?, ?, ?)');
|
||||||
|
for (const event of events) insertEvent.run(event.id, event.occurredAt, event.dedupeKey, JSON.stringify(event));
|
||||||
|
insert.run('storage-version', '1');
|
||||||
|
imported = true;
|
||||||
|
} else if (initialized.value !== '1') {
|
||||||
|
throw new Error('Unsupported Harbor storage version');
|
||||||
|
}
|
||||||
|
db.exec('PRAGMA user_version = 1');
|
||||||
|
});
|
||||||
|
const state = createSqliteDocumentStore(db, 'state', migrateStoredState);
|
||||||
|
const devices = createSqliteDocumentStore(db, 'devices', migrateDeviceInventoryState);
|
||||||
|
// Validate stored versions before startup. SQLite corruption never falls back to JSON.
|
||||||
|
state.read();
|
||||||
|
devices.read();
|
||||||
|
return { db, state, devices, imported, close: () => db.close() };
|
||||||
|
} catch (error) {
|
||||||
|
db.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
|
||||||
|
export function openSqlite(filePath: string) {
|
||||||
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||||
|
const db = new DatabaseSync(filePath);
|
||||||
|
try {
|
||||||
|
fs.chmodSync(filePath, 0o600);
|
||||||
|
db.exec('PRAGMA busy_timeout = 5000; PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON;');
|
||||||
|
return db;
|
||||||
|
} catch (error) {
|
||||||
|
db.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function transaction<T>(db: DatabaseSync, change: () => T): T {
|
||||||
|
db.exec('BEGIN IMMEDIATE');
|
||||||
|
try {
|
||||||
|
const result = change();
|
||||||
|
if (result && typeof result === 'object' && 'then' in result) {
|
||||||
|
throw new TypeError('SQLite transaction must be synchronous');
|
||||||
|
}
|
||||||
|
db.exec('COMMIT');
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
db.exec('ROLLBACK');
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import { Worker } from 'node:worker_threads';
|
||||||
|
import type { LiveTrafficSourceState } from '../../shared/liveTraffic.js';
|
||||||
|
import {
|
||||||
|
assertTrafficHistorySnapshot,
|
||||||
|
emptyTrafficHistory,
|
||||||
|
type TrafficHistoryQuery,
|
||||||
|
} from '../../shared/trafficHistory.js';
|
||||||
|
import type { NativeTrafficProjectionBatch } from './liveTrafficService.js';
|
||||||
|
import type { HistoryWorkerRequest } from './trafficHistoryWorker.js';
|
||||||
|
|
||||||
|
type Request = HistoryWorkerRequest extends infer R ? R extends { id: number } ? Omit<R, 'id'> : never : never;
|
||||||
|
const MAX_QUEUED_CONNECTIONS = 16_384;
|
||||||
|
|
||||||
|
export function createTrafficHistoryService({ filePath, source }: {
|
||||||
|
filePath: string;
|
||||||
|
source: () => LiveTrafficSourceState;
|
||||||
|
}) {
|
||||||
|
let worker: Worker | null = null;
|
||||||
|
let sequence = 0;
|
||||||
|
let pending: NativeTrafficProjectionBatch[] = [];
|
||||||
|
let queued = 0;
|
||||||
|
let missedSince: number | null = null;
|
||||||
|
let storageError = false;
|
||||||
|
let stopping = false;
|
||||||
|
let flushing: Promise<void> | null = null;
|
||||||
|
const requests = new Map<number, { resolve: (value: unknown) => void; reject: (error: Error) => void; timer: NodeJS.Timeout }>();
|
||||||
|
|
||||||
|
function failed() {
|
||||||
|
storageError = true;
|
||||||
|
missedSince ??= Date.now();
|
||||||
|
for (const request of requests.values()) {
|
||||||
|
clearTimeout(request.timer);
|
||||||
|
request.reject(new Error('TRAFFIC_HISTORY_UNAVAILABLE'));
|
||||||
|
}
|
||||||
|
requests.clear();
|
||||||
|
}
|
||||||
|
function ensureWorker() {
|
||||||
|
if (worker) return worker;
|
||||||
|
const current = new Worker(new URL('./trafficHistoryWorker.js', import.meta.url), { workerData: { filePath } });
|
||||||
|
worker = current;
|
||||||
|
current.on('message', (message: { id: number; result?: unknown; error?: string }) => {
|
||||||
|
const request = requests.get(message.id);
|
||||||
|
if (!request) return;
|
||||||
|
requests.delete(message.id);
|
||||||
|
clearTimeout(request.timer);
|
||||||
|
if (message.error) request.reject(new Error('TRAFFIC_HISTORY_UNAVAILABLE'));
|
||||||
|
else request.resolve(message.result);
|
||||||
|
});
|
||||||
|
current.on('error', () => {
|
||||||
|
if (worker !== current) return;
|
||||||
|
worker = null;
|
||||||
|
failed();
|
||||||
|
});
|
||||||
|
current.on('exit', () => {
|
||||||
|
if (worker !== current) return;
|
||||||
|
worker = null;
|
||||||
|
if (!stopping) failed();
|
||||||
|
});
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
function request(message: Request): Promise<unknown> {
|
||||||
|
if (requests.size >= 32) return Promise.reject(new Error('TRAFFIC_HISTORY_BUSY'));
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const id = ++sequence;
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
requests.delete(id);
|
||||||
|
reject(new Error('TRAFFIC_HISTORY_UNAVAILABLE'));
|
||||||
|
// Bound worker mailbox growth as well as the main-thread queue after slow SQL.
|
||||||
|
const current = worker;
|
||||||
|
worker = null;
|
||||||
|
void current?.terminate();
|
||||||
|
failed();
|
||||||
|
}, 5_000);
|
||||||
|
requests.set(id, { resolve, reject, timer });
|
||||||
|
try { ensureWorker().postMessage({ ...message, id }); }
|
||||||
|
catch {
|
||||||
|
requests.delete(id);
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(new Error('TRAFFIC_HISTORY_UNAVAILABLE'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function enqueue(batch: NativeTrafficProjectionBatch) {
|
||||||
|
if (stopping) return;
|
||||||
|
if (queued + batch.connections.length > MAX_QUEUED_CONNECTIONS || pending.length >= 120) {
|
||||||
|
missedSince ??= Date.parse(batch.observedAt);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pending.push(batch);
|
||||||
|
queued += batch.connections.length;
|
||||||
|
}
|
||||||
|
function flush(): Promise<void> {
|
||||||
|
if (flushing) return flushing;
|
||||||
|
const batches = pending;
|
||||||
|
const missed = missedSince;
|
||||||
|
pending = [];
|
||||||
|
queued = 0;
|
||||||
|
missedSince = null;
|
||||||
|
flushing = request({ kind: 'ingest', batches, source: source(), missedSince: missed })
|
||||||
|
.then(() => { storageError = false; })
|
||||||
|
.catch(() => {
|
||||||
|
storageError = true;
|
||||||
|
missedSince = Math.min(missedSince ?? Infinity, missed ?? Infinity,
|
||||||
|
batches.length ? Date.parse(batches[0].observedAt) : Date.now());
|
||||||
|
// Recoverable cumulative counters will reconcile on the next batch. Lost closed flows
|
||||||
|
// stay an explicit gap; history must never backpressure the native/metrics collector.
|
||||||
|
})
|
||||||
|
.finally(() => { flushing = null; });
|
||||||
|
return flushing;
|
||||||
|
}
|
||||||
|
const timer = setInterval(() => { if (!stopping) void flush(); }, 1_000);
|
||||||
|
timer.unref();
|
||||||
|
|
||||||
|
return {
|
||||||
|
enqueue,
|
||||||
|
flush,
|
||||||
|
async query(query: TrafficHistoryQuery) {
|
||||||
|
try {
|
||||||
|
await flush();
|
||||||
|
const result = assertTrafficHistorySnapshot(await request({ kind: 'query', query }));
|
||||||
|
result.source = source();
|
||||||
|
if (missedSince !== null || storageError) result.coverage.partial = true;
|
||||||
|
if (storageError) result.storage = { status: 'error', errorCode: 'TRAFFIC_HISTORY_UNAVAILABLE' };
|
||||||
|
return result;
|
||||||
|
} catch {
|
||||||
|
const result = emptyTrafficHistory(query, source());
|
||||||
|
result.storage = { status: 'error', errorCode: 'TRAFFIC_HISTORY_UNAVAILABLE' };
|
||||||
|
result.coverage.partial = true;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async close() {
|
||||||
|
if (stopping) return;
|
||||||
|
stopping = true;
|
||||||
|
clearInterval(timer);
|
||||||
|
await flushing;
|
||||||
|
await flush();
|
||||||
|
if (worker) {
|
||||||
|
try { await request({ kind: 'close' }); } catch { /* shutdown remains bounded */ }
|
||||||
|
const current = worker;
|
||||||
|
worker = null;
|
||||||
|
await current?.terminate();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
import net from 'node:net';
|
||||||
|
import { domainToASCII } from 'node:url';
|
||||||
|
import { getDomain } from 'tldts';
|
||||||
|
import type { LiveTrafficSourceState } from '../../shared/liveTraffic.js';
|
||||||
|
import { emptyTrafficHistory, type TrafficHistoryQuery, type TrafficHistorySnapshot } from '../../shared/trafficHistory.js';
|
||||||
|
import type { NativeTrafficProjectionBatch } from './liveTrafficService.js';
|
||||||
|
import { classifyDomain } from './domainTrafficService.js';
|
||||||
|
import { openSqlite, transaction } from './sqlite.js';
|
||||||
|
|
||||||
|
const MINUTE = 60_000;
|
||||||
|
const HOUR = 60 * MINUTE;
|
||||||
|
const DAY = 24 * HOUR;
|
||||||
|
const MAX_INTEGER = (1n << 63n) - 1n;
|
||||||
|
|
||||||
|
function hostname(value: string | null) {
|
||||||
|
const name = domainToASCII((value || '').trim().replace(/\.$/, '')).toLowerCase();
|
||||||
|
return name.length <= 253 && !net.isIP(name)
|
||||||
|
&& name.split('.').every((part) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(part)) ? name : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function openTrafficHistoryStore(filePath: string, now = Date.now) {
|
||||||
|
const db = openSqlite(filePath);
|
||||||
|
try {
|
||||||
|
const version = Number(db.prepare('PRAGMA user_version').get()?.user_version);
|
||||||
|
if (version !== 0 && version !== 1) throw new Error(`Unsupported traffic database version: ${version}`);
|
||||||
|
transaction(db, () => db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL) STRICT;
|
||||||
|
CREATE TABLE IF NOT EXISTS dimensions (
|
||||||
|
id INTEGER PRIMARY KEY, identity TEXT NOT NULL UNIQUE,
|
||||||
|
origin_id TEXT NOT NULL, origin_label TEXT NOT NULL, source_ip TEXT NOT NULL,
|
||||||
|
inbound TEXT NOT NULL, service TEXT NOT NULL, domain TEXT NOT NULL,
|
||||||
|
hostname TEXT NOT NULL, ip TEXT NOT NULL, route TEXT NOT NULL, outbound TEXT NOT NULL
|
||||||
|
) STRICT;
|
||||||
|
CREATE TABLE IF NOT EXISTS buckets (
|
||||||
|
at INTEGER NOT NULL, resolution INTEGER NOT NULL, dimension_id INTEGER NOT NULL REFERENCES dimensions(id),
|
||||||
|
upload INTEGER NOT NULL CHECK(upload >= 0), download INTEGER NOT NULL CHECK(download >= 0),
|
||||||
|
PRIMARY KEY(at, resolution, dimension_id)
|
||||||
|
) STRICT, WITHOUT ROWID;
|
||||||
|
CREATE TABLE IF NOT EXISTS checkpoints (
|
||||||
|
identity TEXT PRIMARY KEY, upload INTEGER NOT NULL, download INTEGER NOT NULL,
|
||||||
|
last_seen INTEGER NOT NULL, closed INTEGER NOT NULL
|
||||||
|
) STRICT;
|
||||||
|
CREATE INDEX IF NOT EXISTS checkpoint_age ON checkpoints(last_seen);
|
||||||
|
CREATE INDEX IF NOT EXISTS checkpoint_active ON checkpoints(json_extract(identity, '$[1]')) WHERE closed = 0;
|
||||||
|
CREATE TABLE IF NOT EXISTS gaps (at INTEGER PRIMARY KEY, until_at INTEGER NOT NULL) STRICT;
|
||||||
|
PRAGMA user_version = 1;
|
||||||
|
`));
|
||||||
|
} catch (error) { db.close(); throw error; }
|
||||||
|
// SQLite's built-in lower() handles ASCII only; service labels also use Cyrillic.
|
||||||
|
db.function('lower_unicode', { deterministic: true }, (value) => String(value).toLowerCase());
|
||||||
|
const meta = (key: string) => db.prepare('SELECT value FROM meta WHERE key = ?').get(key)?.value as string | undefined;
|
||||||
|
const setMeta = db.prepare('INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
|
||||||
|
const checkpoint = db.prepare('SELECT upload, download, last_seen FROM checkpoints WHERE identity = ?');
|
||||||
|
checkpoint.setReadBigInts(true);
|
||||||
|
const saveCheckpoint = db.prepare(`INSERT INTO checkpoints VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(identity) DO UPDATE SET upload = excluded.upload, download = excluded.download,
|
||||||
|
last_seen = excluded.last_seen, closed = excluded.closed`);
|
||||||
|
const saveDimension = db.prepare(`INSERT INTO dimensions(identity, origin_id, origin_label, source_ip, inbound,
|
||||||
|
service, domain, hostname, ip, route, outbound) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(identity) DO UPDATE SET origin_label = excluded.origin_label RETURNING id`);
|
||||||
|
const saveBucket = db.prepare(`INSERT INTO buckets VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(at, resolution, dimension_id) DO UPDATE SET
|
||||||
|
upload = upload + excluded.upload, download = download + excluded.download`);
|
||||||
|
const saveGap = db.prepare('INSERT INTO gaps VALUES (?, ?) ON CONFLICT(at) DO UPDATE SET until_at = MAX(until_at, excluded.until_at)');
|
||||||
|
let lastMaintenance = 0;
|
||||||
|
|
||||||
|
function gap(from: number, to: number) {
|
||||||
|
saveGap.run(Math.floor(from / MINUTE) * MINUTE, Math.max(from, to));
|
||||||
|
}
|
||||||
|
// A reopened collector cannot prove that short-lived flows were observed while it was down.
|
||||||
|
const previousObservation = Number(meta('observed') || 0);
|
||||||
|
if (previousObservation) transaction(db, () => gap(previousObservation, now()));
|
||||||
|
function maintain(timestamp = now()) {
|
||||||
|
const hourCutoff = Math.floor((timestamp - 7 * DAY) / HOUR) * HOUR;
|
||||||
|
const cutoff = Math.floor((timestamp - 90 * DAY) / HOUR) * HOUR;
|
||||||
|
transaction(db, () => {
|
||||||
|
db.prepare('DELETE FROM buckets WHERE at < ?').run(cutoff);
|
||||||
|
db.prepare(`INSERT INTO buckets(at, resolution, dimension_id, upload, download)
|
||||||
|
SELECT (at / ?) * ?, ?, dimension_id, SUM(upload), SUM(download)
|
||||||
|
FROM buckets WHERE resolution = ? AND at < ? GROUP BY (at / ?), dimension_id
|
||||||
|
ON CONFLICT(at, resolution, dimension_id) DO UPDATE SET
|
||||||
|
upload = upload + excluded.upload, download = download + excluded.download
|
||||||
|
`).run(HOUR, HOUR, HOUR, MINUTE, hourCutoff, HOUR);
|
||||||
|
db.prepare('DELETE FROM buckets WHERE resolution = ? AND at < ?').run(MINUTE, hourCutoff);
|
||||||
|
// Idle active flows can outlive retention; their cumulative baseline is still required.
|
||||||
|
// ponytail: closed tombstones can grow for 90 days; tighter pruning needs an upstream replay watermark.
|
||||||
|
db.prepare('DELETE FROM checkpoints WHERE closed = 1 AND last_seen < ?').run(cutoff);
|
||||||
|
db.prepare('DELETE FROM gaps WHERE until_at < ?').run(cutoff);
|
||||||
|
db.exec('DELETE FROM dimensions WHERE id NOT IN (SELECT dimension_id FROM buckets)');
|
||||||
|
});
|
||||||
|
lastMaintenance = timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ingest(batches: NativeTrafficProjectionBatch[], source: LiveTrafficSourceState, missedSince: number | null = null) {
|
||||||
|
const timestamp = now();
|
||||||
|
transaction(db, () => {
|
||||||
|
const collectionStart = Number(meta('started') || batches[0] && Date.parse(batches[0].observedAt) || timestamp);
|
||||||
|
if (!meta('started') && batches.length) setMeta.run('started', String(collectionStart));
|
||||||
|
let lastAt = Number(meta('observed') || 0);
|
||||||
|
if (missedSince !== null) gap(missedSince, timestamp);
|
||||||
|
for (const batch of batches) {
|
||||||
|
const at = Date.parse(batch.observedAt);
|
||||||
|
if (!Number.isFinite(at) || !batch.epoch) continue;
|
||||||
|
if (meta('epoch') !== batch.epoch) {
|
||||||
|
// Projection batches are FIFO. A new sing-box epoch cannot replay old lifecycles.
|
||||||
|
db.exec('DELETE FROM checkpoints');
|
||||||
|
setMeta.run('epoch', batch.epoch);
|
||||||
|
}
|
||||||
|
if (batch.reset) db.prepare('UPDATE checkpoints SET closed = 1, last_seen = MAX(last_seen, ?) WHERE closed = 0').run(at);
|
||||||
|
if (lastAt && (batch.reset || at - lastAt > 5_000)) gap(lastAt, at);
|
||||||
|
for (const connection of batch.connections) {
|
||||||
|
const started = Date.parse(connection.startedAt);
|
||||||
|
if (!Number.isFinite(started)) continue;
|
||||||
|
if (connection.closedAt && Date.parse(connection.closedAt) < timestamp - 90 * DAY) continue;
|
||||||
|
const identity = JSON.stringify([batch.epoch, connection.id, connection.startedAt]);
|
||||||
|
const old = checkpoint.get(identity);
|
||||||
|
const upload = BigInt(connection.traffic.uploadBytes);
|
||||||
|
const download = BigInt(connection.traffic.downloadBytes);
|
||||||
|
if (upload < 0n || download < 0n || upload > MAX_INTEGER || download > MAX_INTEGER) {
|
||||||
|
gap(lastAt || at, at);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (old && at < Number(old.last_seen)) continue;
|
||||||
|
// Initial snapshots baseline pre-existing flows; they are not historical observations.
|
||||||
|
const baseline = !old && started < collectionStart;
|
||||||
|
let up = baseline ? 0n : upload - BigInt(String(old?.upload ?? 0));
|
||||||
|
let down = baseline ? 0n : download - BigInt(String(old?.download ?? 0));
|
||||||
|
if (up < 0n || down < 0n) {
|
||||||
|
gap(old ? Number(old.last_seen) : at, at);
|
||||||
|
up = up < 0n ? 0n : up;
|
||||||
|
down = down < 0n ? 0n : down;
|
||||||
|
}
|
||||||
|
saveCheckpoint.run(identity, old && upload < BigInt(String(old.upload)) ? old.upload : upload,
|
||||||
|
old && download < BigInt(String(old.download)) ? old.download : download,
|
||||||
|
at, connection.closedAt === null ? 0 : 1);
|
||||||
|
if (up === 0n && down === 0n) continue;
|
||||||
|
const host = hostname(connection.destination.domain);
|
||||||
|
const domain = host ? getDomain(host, { allowPrivateDomains: true }) || host : '';
|
||||||
|
const classification = classifyDomain(host);
|
||||||
|
const service = ['yandex.ru', 'yandex.com', 'yandex.net', 'yastatic.net'].includes(domain) ? 'Яндекс'
|
||||||
|
: classification && classification.service !== classification.domain ? classification.service : domain;
|
||||||
|
const origin = connection.origin.kind === 'this-mac' ? 'this-mac'
|
||||||
|
: connection.origin.id || `unknown:${connection.source.ip}`;
|
||||||
|
const ip = net.isIP(connection.destination.ip || '') ? connection.destination.ip! : '';
|
||||||
|
const dims = [origin, connection.source.ip, connection.inbound.tag, service, domain,
|
||||||
|
host, ip, connection.route.kind, connection.route.outbound || ''];
|
||||||
|
const dimension = saveDimension.get(JSON.stringify(dims), origin, connection.origin.label,
|
||||||
|
...dims.slice(1));
|
||||||
|
const resolution = at < Math.floor((timestamp - 7 * DAY) / HOUR) * HOUR ? HOUR : MINUTE;
|
||||||
|
saveBucket.run(Math.floor(at / resolution) * resolution, resolution, dimension!.id, up, down);
|
||||||
|
}
|
||||||
|
if (batch.closedIds.length) {
|
||||||
|
const closed = db.prepare(`UPDATE checkpoints SET closed = 1, last_seen = MAX(last_seen, ?)
|
||||||
|
WHERE closed = 0 AND json_extract(identity, '$[1]') IN (SELECT value FROM json_each(?))`)
|
||||||
|
.run(at, JSON.stringify(batch.closedIds));
|
||||||
|
if (closed.changes) gap(lastAt || at, at); // Terminal identity without final byte totals.
|
||||||
|
}
|
||||||
|
lastAt = Math.max(lastAt, at);
|
||||||
|
}
|
||||||
|
if (lastAt) setMeta.run('observed', String(lastAt));
|
||||||
|
if (source === 'degraded' || source === 'stale' || source === 'incompatible'
|
||||||
|
|| (source === 'connecting' && lastAt)) gap(lastAt || timestamp, timestamp);
|
||||||
|
setMeta.run('source', source);
|
||||||
|
});
|
||||||
|
if (timestamp - lastMaintenance >= HOUR) maintain(timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
|
function query(input: TrafficHistoryQuery): TrafficHistorySnapshot {
|
||||||
|
const timestamp = now();
|
||||||
|
if (timestamp - lastMaintenance >= HOUR) maintain(timestamp);
|
||||||
|
const result = emptyTrafficHistory(input, (meta('source') || 'connecting') as LiveTrafficSourceState, timestamp);
|
||||||
|
const requestedTo = Date.parse(result.period.to);
|
||||||
|
const retentionStart = Math.floor((timestamp - 90 * DAY) / HOUR) * HOUR;
|
||||||
|
const from = Math.max(Date.parse(result.period.from), retentionStart);
|
||||||
|
const minuteFrom = Math.floor((timestamp - 7 * DAY) / HOUR) * HOUR;
|
||||||
|
const terminalResolution = requestedTo < minuteFrom ? HOUR : MINUTE;
|
||||||
|
const to = Math.floor(requestedTo / terminalResolution) * terminalResolution;
|
||||||
|
result.period.to = new Date(to).toISOString();
|
||||||
|
result.query.until = to;
|
||||||
|
// Bounds describe complete stored buckets; old data is explicitly hourly, not minute-precise.
|
||||||
|
const effectiveFrom = Math.min(to, Math.floor(from / (from < minuteFrom ? HOUR : MINUTE)) * (from < minuteFrom ? HOUR : MINUTE));
|
||||||
|
result.period.from = new Date(effectiveFrom).toISOString();
|
||||||
|
result.period.minuteFrom = new Date(minuteFrom).toISOString();
|
||||||
|
result.period.availableFrom = meta('started') ? new Date(Math.max(Number(meta('started')), retentionStart)).toISOString() : null;
|
||||||
|
const lastAt = Number(meta('observed') || 0);
|
||||||
|
result.coverage.lastObservedAt = lastAt ? new Date(lastAt).toISOString() : null;
|
||||||
|
result.coverage.gapCount = Number(db.prepare('SELECT COUNT(*) AS count FROM gaps WHERE until_at >= ? AND at < ?').get(effectiveFrom, to)?.count);
|
||||||
|
result.coverage.partial = result.coverage.gapCount > 0 || ['stale', 'degraded', 'incompatible'].includes(result.source)
|
||||||
|
|| (lastAt > 0 && result.source === 'live' && timestamp - lastAt > 5_000);
|
||||||
|
const where = ['b.at >= ?', 'b.at < ?'];
|
||||||
|
const args: Array<string | number> = [effectiveFrom, to];
|
||||||
|
for (const [column, value] of [
|
||||||
|
['origin_id', input.originId], ['service', input.service], ['domain', input.domain], ['hostname', input.hostname],
|
||||||
|
['route', input.route === 'all' ? '' : input.route],
|
||||||
|
]) if (value) { where.push(`d.${column} = ?`); args.push(value); }
|
||||||
|
// Empty domain is a real IP-only group. Parent filters need an explicit level, not truthiness alone.
|
||||||
|
if (input.level !== 'service') { where.push('d.service = ?'); args.push(input.service); }
|
||||||
|
if (input.level === 'hostname' || input.level === 'ip') { where.push('d.domain = ?'); args.push(input.domain); }
|
||||||
|
if (input.level === 'ip') { where.push('d.hostname = ?'); args.push(input.hostname); }
|
||||||
|
if (input.search) {
|
||||||
|
where.push("instr(lower_unicode(d.hostname || ' ' || d.ip || ' ' || d.service), ?) > 0"); args.push(input.search.toLowerCase());
|
||||||
|
}
|
||||||
|
const joined = `FROM buckets b JOIN dimensions d ON d.id = b.dimension_id WHERE ${where.join(' AND ')}`;
|
||||||
|
const totals = db.prepare(`SELECT COALESCE(SUM(b.upload), 0) AS upload, COALESCE(SUM(b.download), 0) AS download ${joined}`);
|
||||||
|
totals.setReadBigInts(true);
|
||||||
|
const sum = totals.get(...args)!;
|
||||||
|
result.totals = { uploadBytes: String(sum.upload), downloadBytes: String(sum.download) };
|
||||||
|
const rows = db.prepare(`SELECT d.${input.level} AS key, SUM(b.upload) AS upload, SUM(b.download) AS download,
|
||||||
|
CASE WHEN COUNT(DISTINCT d.route) = 1 THEN MIN(d.route) ELSE 'mixed' END AS route
|
||||||
|
${joined} GROUP BY d.${input.level} ORDER BY SUM(b.upload) + SUM(b.download) DESC, key LIMIT 101 OFFSET ?`);
|
||||||
|
rows.setReadBigInts(true);
|
||||||
|
const page = rows.all(...args, input.offset);
|
||||||
|
result.rows = page.slice(0, 100).map((row) => ({
|
||||||
|
key: String(row.key), label: String(row.key) || (input.level === 'ip' ? 'IP неизвестен' : 'Без домена'),
|
||||||
|
uploadBytes: String(row.upload), downloadBytes: String(row.download), route: String(row.route) as 'mixed',
|
||||||
|
}));
|
||||||
|
result.nextOffset = page.length > 100 ? input.offset + 100 : null;
|
||||||
|
const origins = db.prepare(`SELECT d.origin_id AS id, MAX(d.origin_label) AS label FROM dimensions d
|
||||||
|
JOIN buckets b ON b.dimension_id = d.id WHERE b.at >= ? AND b.at < ?
|
||||||
|
GROUP BY d.origin_id ORDER BY id LIMIT 257`).all(effectiveFrom, to);
|
||||||
|
result.origins = origins.slice(0, 256).map((row) => ({ id: String(row.id), label: String(row.label) }));
|
||||||
|
result.originsTruncated = origins.length > 256;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return { ingest, query, maintain, close: () => db.close() };
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { parentPort, workerData } from 'node:worker_threads';
|
||||||
|
import type { LiveTrafficSourceState } from '../../shared/liveTraffic.js';
|
||||||
|
import type { TrafficHistoryQuery } from '../../shared/trafficHistory.js';
|
||||||
|
import type { NativeTrafficProjectionBatch } from './liveTrafficService.js';
|
||||||
|
import { openTrafficHistoryStore } from './trafficHistoryStore.js';
|
||||||
|
|
||||||
|
export type HistoryWorkerRequest =
|
||||||
|
| { id: number; kind: 'ingest'; batches: NativeTrafficProjectionBatch[]; source: LiveTrafficSourceState; missedSince: number | null }
|
||||||
|
| { id: number; kind: 'query'; query: TrafficHistoryQuery }
|
||||||
|
| { id: number; kind: 'close' };
|
||||||
|
|
||||||
|
const store = openTrafficHistoryStore(workerData.filePath);
|
||||||
|
parentPort!.on('message', (message: HistoryWorkerRequest) => {
|
||||||
|
try {
|
||||||
|
let result: unknown = null;
|
||||||
|
if (message.kind === 'ingest') store.ingest(message.batches, message.source, message.missedSince);
|
||||||
|
else if (message.kind === 'query') result = store.query(message.query);
|
||||||
|
else store.close();
|
||||||
|
parentPort!.postMessage({ id: message.id, result });
|
||||||
|
if (message.kind === 'close') parentPort!.close();
|
||||||
|
} catch {
|
||||||
|
// Do not expose filesystem paths, SQL or native metadata through the public error.
|
||||||
|
parentPort!.postMessage({ id: message.id, error: 'TRAFFIC_HISTORY_UNAVAILABLE' });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import type { LiveTrafficSourceState } from './liveTraffic.js';
|
||||||
|
|
||||||
|
export const TRAFFIC_HISTORY_RANGES = { '24h': 1, '7d': 7, '30d': 30, '90d': 90 } as const;
|
||||||
|
export type TrafficHistoryRange = keyof typeof TRAFFIC_HISTORY_RANGES;
|
||||||
|
export type TrafficHistoryLevel = 'service' | 'domain' | 'hostname' | 'ip';
|
||||||
|
export interface TrafficHistoryQuery {
|
||||||
|
range: TrafficHistoryRange;
|
||||||
|
level: TrafficHistoryLevel;
|
||||||
|
originId: string;
|
||||||
|
search: string;
|
||||||
|
route: 'all' | 'vpn' | 'direct' | 'other';
|
||||||
|
service: string;
|
||||||
|
domain: string;
|
||||||
|
hostname: string;
|
||||||
|
offset: number;
|
||||||
|
until: number | null;
|
||||||
|
}
|
||||||
|
export interface TrafficHistoryRow {
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
uploadBytes: string;
|
||||||
|
downloadBytes: string;
|
||||||
|
route: 'vpn' | 'direct' | 'other' | 'mixed';
|
||||||
|
}
|
||||||
|
export interface TrafficHistorySnapshot {
|
||||||
|
apiVersion: 1;
|
||||||
|
generatedAt: string;
|
||||||
|
query: TrafficHistoryQuery;
|
||||||
|
period: { from: string; to: string; availableFrom: string | null; minuteFrom: string; retentionDays: 90 };
|
||||||
|
storage: { status: 'ready' | 'error'; errorCode: string | null };
|
||||||
|
source: LiveTrafficSourceState;
|
||||||
|
coverage: { partial: boolean; gapCount: number; lastObservedAt: string | null };
|
||||||
|
totals: { uploadBytes: string; downloadBytes: string };
|
||||||
|
rows: TrafficHistoryRow[];
|
||||||
|
nextOffset: number | null;
|
||||||
|
origins: Array<{ id: string; label: string }>;
|
||||||
|
originsTruncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseTrafficHistoryQuery(params: URLSearchParams): TrafficHistoryQuery {
|
||||||
|
const range = params.get('range') || '24h';
|
||||||
|
const level = params.get('level') || 'service';
|
||||||
|
const route = params.get('route') || 'all';
|
||||||
|
if (!Object.hasOwn(TRAFFIC_HISTORY_RANGES, range)
|
||||||
|
|| !['service', 'domain', 'hostname', 'ip'].includes(level)
|
||||||
|
|| !['all', 'vpn', 'direct', 'other'].includes(route)) throw new TypeError('Invalid history query');
|
||||||
|
const text = (key: string, max = 253) => {
|
||||||
|
const value = params.get(key) || '';
|
||||||
|
if (value.length > max || /[\x00-\x1f]/.test(value)) throw new TypeError('Invalid history filter');
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
const offset = Number(params.get('offset') || 0);
|
||||||
|
const until = params.has('until') ? Number(params.get('until')) : null;
|
||||||
|
if (!Number.isSafeInteger(offset) || offset < 0 || offset > 1_000_000
|
||||||
|
|| (until !== null && (!Number.isSafeInteger(until) || until <= 0 || until > 8_640_000_000_000_000))) throw new TypeError('Invalid history page');
|
||||||
|
return {
|
||||||
|
range: range as TrafficHistoryRange, level: level as TrafficHistoryLevel,
|
||||||
|
route: route as TrafficHistoryQuery['route'], originId: text('originId', 128), search: text('search', 200).trim(),
|
||||||
|
service: text('service'), domain: text('domain'), hostname: text('hostname'), offset, until,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function historyQueryParams(query: TrafficHistoryQuery) {
|
||||||
|
return new URLSearchParams(Object.entries(query).filter(([, value]) => value !== null)
|
||||||
|
.map(([key, value]): [string, string] => [key, String(value)]));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function emptyTrafficHistory(query: TrafficHistoryQuery, source: LiveTrafficSourceState = 'disabled', now = Date.now()): TrafficHistorySnapshot {
|
||||||
|
const to = Math.min(query.until ?? now, now);
|
||||||
|
return {
|
||||||
|
apiVersion: 1, generatedAt: new Date(now).toISOString(), query: { ...query, until: to },
|
||||||
|
period: { from: new Date(to - TRAFFIC_HISTORY_RANGES[query.range] * 86_400_000).toISOString(),
|
||||||
|
to: new Date(to).toISOString(), availableFrom: null,
|
||||||
|
minuteFrom: new Date(now - 7 * 86_400_000).toISOString(), retentionDays: 90 },
|
||||||
|
storage: { status: 'ready', errorCode: null }, source,
|
||||||
|
coverage: { partial: false, gapCount: 0, lastObservedAt: null },
|
||||||
|
totals: { uploadBytes: '0', downloadBytes: '0' }, rows: [], nextOffset: null, origins: [], originsTruncated: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertTrafficHistorySnapshot(value: unknown): TrafficHistorySnapshot {
|
||||||
|
if (!value || typeof value !== 'object') throw new TypeError('Invalid history snapshot');
|
||||||
|
const v = value as TrafficHistorySnapshot;
|
||||||
|
const iso = (s: unknown) => typeof s === 'string' && Number.isFinite(Date.parse(s));
|
||||||
|
const bytes = (s: unknown) => typeof s === 'string' && /^\d+$/.test(s);
|
||||||
|
if (v.apiVersion !== 1 || !iso(v.generatedAt) || !v.query || !v.period || !v.storage || !v.coverage || !v.totals
|
||||||
|
|| v.period.retentionDays !== 90 || !iso(v.period.from) || !iso(v.period.to) || !iso(v.period.minuteFrom)
|
||||||
|
|| !(v.period.availableFrom === null || iso(v.period.availableFrom))
|
||||||
|
|| !['ready', 'error'].includes(v.storage.status)
|
||||||
|
|| !(v.storage.errorCode === null || typeof v.storage.errorCode === 'string')
|
||||||
|
|| !['live', 'connecting', 'disabled', 'stopped', 'stale', 'degraded', 'incompatible'].includes(v.source)
|
||||||
|
|| typeof v.coverage.partial !== 'boolean' || !Number.isSafeInteger(v.coverage.gapCount) || v.coverage.gapCount < 0
|
||||||
|
|| !(v.coverage.lastObservedAt === null || iso(v.coverage.lastObservedAt))
|
||||||
|
|| !bytes(v.totals.uploadBytes) || !bytes(v.totals.downloadBytes)
|
||||||
|
|| !Array.isArray(v.rows) || v.rows.length > 100
|
||||||
|
|| !Array.isArray(v.origins) || v.origins.length > 256 || typeof v.originsTruncated !== 'boolean'
|
||||||
|
|| !(v.nextOffset === null || (Number.isSafeInteger(v.nextOffset) && v.nextOffset >= 0))) {
|
||||||
|
throw new TypeError('Invalid history snapshot');
|
||||||
|
}
|
||||||
|
parseTrafficHistoryQuery(historyQueryParams(v.query));
|
||||||
|
for (const row of v.rows) {
|
||||||
|
if (!row || typeof row.key !== 'string' || typeof row.label !== 'string'
|
||||||
|
|| row.key.length > 253 || row.label.length > 253
|
||||||
|
|| !bytes(row.uploadBytes) || !bytes(row.downloadBytes)
|
||||||
|
|| !['vpn', 'direct', 'other', 'mixed'].includes(row.route)) throw new TypeError('Invalid history row');
|
||||||
|
}
|
||||||
|
for (const origin of v.origins) {
|
||||||
|
if (!origin || typeof origin.id !== 'string' || typeof origin.label !== 'string') throw new TypeError('Invalid history origin');
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
export const HARBOR_VERSIONS = Object.freeze({
|
export const HARBOR_VERSIONS = Object.freeze({
|
||||||
macClient: '0.36.1',
|
macClient: '0.37.0',
|
||||||
gatewayClient: '0.38.1',
|
gatewayClient: '0.39.0',
|
||||||
gatewayBackend: '0.38.1',
|
gatewayBackend: '0.39.0',
|
||||||
});
|
});
|
||||||
|
|
||||||
export interface ParsedVersion {
|
export interface ParsedVersion {
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ const componentActions = {
|
|||||||
runDnsDiagnostics: api.diagnostics.dns,
|
runDnsDiagnostics: api.diagnostics.dns,
|
||||||
loadActivityJournal: api.activityJournal.page,
|
loadActivityJournal: api.activityJournal.page,
|
||||||
loadLiveTraffic: api.traffic.live,
|
loadLiveTraffic: api.traffic.live,
|
||||||
|
loadTrafficHistory: api.traffic.history,
|
||||||
};
|
};
|
||||||
|
|
||||||
interface UiError {
|
interface UiError {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { ERROR_DEFINITIONS, errorDefinition } from '../../shared/errors.js';
|
import { ERROR_DEFINITIONS, errorDefinition } from '../../shared/errors.js';
|
||||||
import { assertStateSnapshot, type StateSnapshot } from '../../shared/contracts/state.js';
|
import { assertStateSnapshot, type StateSnapshot } from '../../shared/contracts/state.js';
|
||||||
import { ROUTE_RULES_CONTRACT_VERSION } from '../../shared/routingRules.js';
|
import { ROUTE_RULES_CONTRACT_VERSION } from '../../shared/routingRules.js';
|
||||||
|
import { historyQueryParams, type TrafficHistoryQuery } from '../../shared/trafficHistory.js';
|
||||||
|
|
||||||
type RequestOptions = Omit<RequestInit, 'headers'> & {
|
type RequestOptions = Omit<RequestInit, 'headers'> & {
|
||||||
headers?: Record<string, string>;
|
headers?: Record<string, string>;
|
||||||
@@ -239,6 +240,7 @@ export const api = {
|
|||||||
page: (cursor: string | null = null) => request(`/api/activity-journal?limit=50${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`),
|
page: (cursor: string | null = null) => request(`/api/activity-journal?limit=50${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`),
|
||||||
},
|
},
|
||||||
traffic: {
|
traffic: {
|
||||||
|
history: (query: TrafficHistoryQuery, signal?: AbortSignal) => request(`/api/traffic/history?${historyQueryParams(query)}`, { signal }),
|
||||||
live: () => request('/api/traffic/live'),
|
live: () => request('/api/traffic/live'),
|
||||||
updateSettings: (settings: unknown, expectedRevision: number) => request(
|
updateSettings: (settings: unknown, expectedRevision: number) => request(
|
||||||
'/api/traffic/settings',
|
'/api/traffic/settings',
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ interface VersionBadgeProps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface ComponentActions {
|
interface ComponentActions {
|
||||||
|
loadTrafficHistory: import('../features/traffic/index.js').LoadTrafficHistory;
|
||||||
listDevices: () => Promise<unknown>;
|
listDevices: () => Promise<unknown>;
|
||||||
refreshDevices: () => Promise<unknown>;
|
refreshDevices: () => Promise<unknown>;
|
||||||
resetDeviceTraffic: (expectedRevision: number) => Promise<unknown>;
|
resetDeviceTraffic: (expectedRevision: number) => Promise<unknown>;
|
||||||
@@ -600,6 +601,7 @@ export function ClientOverviewPage({
|
|||||||
enabled: true,
|
enabled: true,
|
||||||
isGateway,
|
isGateway,
|
||||||
loadLiveTraffic: actions.loadLiveTraffic,
|
loadLiveTraffic: actions.loadLiveTraffic,
|
||||||
|
loadHistory: actions.loadTrafficHistory,
|
||||||
settings: state.traffic,
|
settings: state.traffic,
|
||||||
updateSettings: onUpdateTrafficSettings,
|
updateSettings: onUpdateTrafficSettings,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import {
|
|||||||
import { Drawer } from '../../ui/Drawer.js';
|
import { Drawer } from '../../ui/Drawer.js';
|
||||||
import { RailAction } from '../../ui/RailAction.js';
|
import { RailAction } from '../../ui/RailAction.js';
|
||||||
import { formatByteString } from '../../utils/format.js';
|
import { formatByteString } from '../../utils/format.js';
|
||||||
|
import { TrafficHistoryPanel, type LoadTrafficHistory } from './TrafficHistoryPanel.js';
|
||||||
import {
|
import {
|
||||||
groupTrafficConnections,
|
groupTrafficConnections,
|
||||||
reconcileTrafficGroups,
|
reconcileTrafficGroups,
|
||||||
@@ -34,6 +35,7 @@ interface TrafficFeatureOptions {
|
|||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
isGateway: boolean;
|
isGateway: boolean;
|
||||||
loadLiveTraffic: () => Promise<unknown>;
|
loadLiveTraffic: () => Promise<unknown>;
|
||||||
|
loadHistory: LoadTrafficHistory;
|
||||||
settings: TrafficSettings;
|
settings: TrafficSettings;
|
||||||
updateSettings: (settings: TrafficSettings) => Promise<unknown>;
|
updateSettings: (settings: TrafficSettings) => Promise<unknown>;
|
||||||
}
|
}
|
||||||
@@ -65,11 +67,13 @@ export function useTrafficFeature({
|
|||||||
enabled,
|
enabled,
|
||||||
isGateway,
|
isGateway,
|
||||||
loadLiveTraffic,
|
loadLiveTraffic,
|
||||||
|
loadHistory,
|
||||||
settings,
|
settings,
|
||||||
updateSettings,
|
updateSettings,
|
||||||
}: TrafficFeatureOptions) {
|
}: TrafficFeatureOptions) {
|
||||||
const [isOpen, setIsOpen] = useState(false);
|
const [isOpen, setIsOpen] = useState(false);
|
||||||
const [paused, setPaused] = useState(false);
|
const [paused, setPaused] = useState(false);
|
||||||
|
const [view, setView] = useState<'live' | 'history'>('live');
|
||||||
const [snapshot, setSnapshot] = useState<LiveTrafficSnapshot | null>(null);
|
const [snapshot, setSnapshot] = useState<LiveTrafficSnapshot | null>(null);
|
||||||
const [requestState, setRequestState] = useState<RequestState>('idle');
|
const [requestState, setRequestState] = useState<RequestState>('idle');
|
||||||
const panelRef = useRef<HTMLElement>(null);
|
const panelRef = useRef<HTMLElement>(null);
|
||||||
@@ -83,7 +87,7 @@ export function useTrafficFeature({
|
|||||||
}, [enabled]);
|
}, [enabled]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!enabled || !isOpen || paused) return undefined;
|
if (!enabled || !isOpen || paused || view !== 'live') return undefined;
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
setRequestState((current) => current === 'idle' ? 'loading' : current);
|
setRequestState((current) => current === 'idle' ? 'loading' : current);
|
||||||
@@ -107,7 +111,7 @@ export function useTrafficFeature({
|
|||||||
cancelled = true;
|
cancelled = true;
|
||||||
if (timer) clearTimeout(timer);
|
if (timer) clearTimeout(timer);
|
||||||
};
|
};
|
||||||
}, [enabled, isOpen, paused, loadLiveTraffic]);
|
}, [enabled, isOpen, paused, view, loadLiveTraffic]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!isOpen) return undefined;
|
if (!isOpen) return undefined;
|
||||||
@@ -146,6 +150,9 @@ export function useTrafficFeature({
|
|||||||
isGateway,
|
isGateway,
|
||||||
isOpen,
|
isOpen,
|
||||||
paused,
|
paused,
|
||||||
|
view,
|
||||||
|
setView,
|
||||||
|
loadHistory,
|
||||||
snapshot,
|
snapshot,
|
||||||
requestState,
|
requestState,
|
||||||
settings,
|
settings,
|
||||||
@@ -429,8 +436,8 @@ export function TrafficPanel({ feature }: { feature: TrafficFeature }) {
|
|||||||
>
|
>
|
||||||
<header className="client-traffic-header">
|
<header className="client-traffic-header">
|
||||||
<div className="client-traffic-meta">
|
<div className="client-traffic-meta">
|
||||||
<span>{feature.isGateway ? 'GATEWAY' : 'MAC'} · {snapshot?.summary.active || 0} АКТИВНЫХ</span>
|
<span>{feature.isGateway ? 'GATEWAY' : 'MAC'} · {feature.view === 'live' ? `${snapshot?.summary.active || 0} АКТИВНЫХ` : 'ИСТОРИЯ'}</span>
|
||||||
<time dateTime={snapshot?.observedAt || undefined}>{updatedAt(snapshot?.observedAt)}</time>
|
<time dateTime={feature.view === 'live' ? snapshot?.observedAt || undefined : undefined}>{feature.view === 'live' ? updatedAt(snapshot?.observedAt) : '\u00a0'}</time>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
aria-pressed={feature.paused}
|
aria-pressed={feature.paused}
|
||||||
@@ -441,6 +448,15 @@ export function TrafficPanel({ feature }: { feature: TrafficFeature }) {
|
|||||||
<p>Соединения сгруппированы по назначению, протоколу и маршруту.</p>
|
<p>Соединения сгруппированы по назначению, протоколу и маршруту.</p>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
<div className="client-traffic-tools">
|
||||||
|
<div className="client-traffic-filters" role="group" aria-label="Режим трафика">
|
||||||
|
<button type="button" aria-pressed={feature.view === 'live'} onClick={() => feature.setView('live')}>Сейчас</button>
|
||||||
|
<button type="button" aria-pressed={feature.view === 'history'} onClick={() => feature.setView('history')}>История</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{feature.view === 'history' ? <TrafficHistoryPanel active={feature.isOpen && !feature.paused} isGateway={feature.isGateway} load={feature.loadHistory} /> : <>
|
||||||
|
|
||||||
{snapshot && <div className="client-traffic-summary" aria-label="Качество распознавания трафика">
|
{snapshot && <div className="client-traffic-summary" aria-label="Качество распознавания трафика">
|
||||||
<span><b>Активных распознано</b> {snapshot.summary.recognized}</span>
|
<span><b>Активных распознано</b> {snapshot.summary.recognized}</span>
|
||||||
<span><b>Активных требует внимания</b> {snapshot.summary.unresolved}</span>
|
<span><b>Активных требует внимания</b> {snapshot.summary.unresolved}</span>
|
||||||
@@ -623,6 +639,7 @@ export function TrafficPanel({ feature }: { feature: TrafficFeature }) {
|
|||||||
{snapshot?.summary.truncated && <p className="client-traffic-truncated" role="status">
|
{snapshot?.summary.truncated && <p className="client-traffic-truncated" role="status">
|
||||||
Снимок ограничен 256 соединениями; активные показаны первыми.
|
Снимок ограничен 256 соединениями; активные показаны первыми.
|
||||||
</p>}
|
</p>}
|
||||||
|
</>}
|
||||||
<p className="client-traffic-honesty">
|
<p className="client-traffic-honesty">
|
||||||
{feature.isGateway
|
{feature.isGateway
|
||||||
? 'Только соединения, прошедшие через sing-box Gateway. Трафик, обходящий sing-box напрямую, здесь не виден.'
|
? 'Только соединения, прошедшие через sing-box Gateway. Трафик, обходящий sing-box напрямую, здесь не виден.'
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
import { useEffect, useId, useMemo, useState } from 'react';
|
||||||
|
import {
|
||||||
|
assertTrafficHistorySnapshot,
|
||||||
|
parseTrafficHistoryQuery,
|
||||||
|
type TrafficHistoryLevel,
|
||||||
|
type TrafficHistoryQuery,
|
||||||
|
type TrafficHistorySnapshot,
|
||||||
|
} from '../../../shared/trafficHistory.js';
|
||||||
|
import { formatByteString } from '../../utils/format.js';
|
||||||
|
|
||||||
|
export type LoadTrafficHistory = (query: TrafficHistoryQuery, signal?: AbortSignal) => Promise<unknown>;
|
||||||
|
const nextLevel: Record<TrafficHistoryLevel, TrafficHistoryLevel | null> = {
|
||||||
|
service: 'domain', domain: 'hostname', hostname: 'ip', ip: null,
|
||||||
|
};
|
||||||
|
const levelLabels = { service: 'Сервис', domain: 'Домен', hostname: 'Полное имя', ip: 'IP' };
|
||||||
|
const routes = { vpn: 'VPN', direct: 'Direct', other: 'Другое', mixed: 'Разные' };
|
||||||
|
|
||||||
|
function useHistory(active: boolean, query: TrafficHistoryQuery, load: LoadTrafficHistory) {
|
||||||
|
const key = JSON.stringify(query);
|
||||||
|
const [result, setResult] = useState<{ key: string; snapshot: TrafficHistorySnapshot | null; status: 'loading' | 'ready' | 'error' }>({
|
||||||
|
key, snapshot: null, status: 'loading',
|
||||||
|
});
|
||||||
|
useEffect(() => {
|
||||||
|
if (!active) return;
|
||||||
|
const controller = new AbortController();
|
||||||
|
let timer: ReturnType<typeof setTimeout>;
|
||||||
|
const poll = async () => {
|
||||||
|
try {
|
||||||
|
const snapshot = assertTrafficHistorySnapshot(await load(JSON.parse(key) as TrafficHistoryQuery, controller.signal));
|
||||||
|
if (snapshot.storage.status === 'error') throw new Error('History unavailable');
|
||||||
|
if (!controller.signal.aborted) setResult({ key, snapshot, status: 'ready' });
|
||||||
|
} catch {
|
||||||
|
if (!controller.signal.aborted) setResult((previous) => ({
|
||||||
|
key, snapshot: previous.key === key ? previous.snapshot : null, status: 'error',
|
||||||
|
}));
|
||||||
|
} finally {
|
||||||
|
if (!controller.signal.aborted) timer = setTimeout(poll, 15_000);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
timer = setTimeout(poll, 200);
|
||||||
|
return () => { controller.abort(); clearTimeout(timer); };
|
||||||
|
}, [active, key, load]);
|
||||||
|
return result.key === key ? result : { key, snapshot: null, status: 'loading' as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
function HistoryRows({ snapshot, active, load }: { snapshot: TrafficHistorySnapshot; active: boolean; load: LoadTrafficHistory }) {
|
||||||
|
const [expanded, setExpanded] = useState<string | null>(null);
|
||||||
|
const detailsId = useId();
|
||||||
|
const { query } = snapshot;
|
||||||
|
const childLevel = nextLevel[query.level];
|
||||||
|
return <div className="client-traffic-list" role="list" aria-label={`История: ${levelLabels[query.level]}`}>
|
||||||
|
{snapshot.rows.map((row, index) => <div className="client-traffic-history-row" role="listitem" key={row.key}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="client-traffic-connection-summary"
|
||||||
|
aria-expanded={childLevel ? expanded === row.key : undefined}
|
||||||
|
aria-controls={childLevel ? `${detailsId}-${index}` : undefined}
|
||||||
|
disabled={!childLevel}
|
||||||
|
onClick={() => setExpanded((current) => current === row.key ? null : row.key)}
|
||||||
|
>
|
||||||
|
<span className="client-traffic-identity"><strong title={row.label}>{row.label}</strong><small>{levelLabels[query.level]}</small></span>
|
||||||
|
<span className="client-traffic-route" data-route={row.route}>{routes[row.route]}</span>
|
||||||
|
<span className="client-traffic-values"><strong>
|
||||||
|
<span>↓ {formatByteString(row.downloadBytes)}</span><span>↑ {formatByteString(row.uploadBytes)}</span>
|
||||||
|
</strong></span>
|
||||||
|
{childLevel && <svg className="client-traffic-chevron" viewBox="0 0 16 16" aria-hidden="true"><path d="m5 6 3 3 3-3" /></svg>}
|
||||||
|
</button>
|
||||||
|
{childLevel && expanded === row.key && <div id={`${detailsId}-${index}`} className="client-traffic-history-children">
|
||||||
|
<HistoryBranch key={row.key} active={active} load={load} query={{
|
||||||
|
...query, level: childLevel, [query.level]: row.key, offset: 0,
|
||||||
|
}} />
|
||||||
|
</div>}
|
||||||
|
</div>)}
|
||||||
|
</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function HistoryBranch({ active, query, load }: { active: boolean; query: TrafficHistoryQuery; load: LoadTrafficHistory }) {
|
||||||
|
const [offset, setOffset] = useState(0);
|
||||||
|
const requested = useMemo(() => ({ ...query, offset }), [query, offset]);
|
||||||
|
const result = useHistory(active, requested, load);
|
||||||
|
return <>
|
||||||
|
<p className="client-traffic-history-status" role="status">{result.status === 'loading' ? 'Загружаем…'
|
||||||
|
: result.status === 'error' ? 'Данные временно недоступны.' : '\u00a0'}</p>
|
||||||
|
{result.snapshot && <>
|
||||||
|
<HistoryRows snapshot={result.snapshot} active={active} load={load} />
|
||||||
|
<HistoryPages snapshot={result.snapshot} offset={offset} change={setOffset} />
|
||||||
|
</>}
|
||||||
|
</>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function HistoryPages({ snapshot, offset, change }: { snapshot: TrafficHistorySnapshot; offset: number; change: (offset: number) => void }) {
|
||||||
|
if (!offset && snapshot.nextOffset === null) return null;
|
||||||
|
return <div className="client-traffic-filters" role="group" aria-label="Страницы истории">
|
||||||
|
<button type="button" disabled={!offset} onClick={() => change(Math.max(0, offset - 100))}>Назад</button>
|
||||||
|
<button type="button" disabled={snapshot.nextOffset === null} onClick={() => change(snapshot.nextOffset!)}>Далее</button>
|
||||||
|
</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TrafficHistoryPanel({ active, isGateway, load }: { active: boolean; isGateway: boolean; load: LoadTrafficHistory }) {
|
||||||
|
const [query, setQuery] = useState(() => parseTrafficHistoryQuery(new URLSearchParams()));
|
||||||
|
const [deviceSearch, setDeviceSearch] = useState('');
|
||||||
|
const [devicesExpanded, setDevicesExpanded] = useState(false);
|
||||||
|
const result = useHistory(active, query, load);
|
||||||
|
const snapshot = result.snapshot;
|
||||||
|
const origins = (snapshot?.origins || []).filter((origin) => origin.label.toLocaleLowerCase('ru').includes(deviceSearch.toLocaleLowerCase('ru')));
|
||||||
|
function change(patch: Partial<TrafficHistoryQuery>) { setQuery((current) => ({ ...current, ...patch, offset: 0, until: null })); }
|
||||||
|
return <>
|
||||||
|
<div className="client-traffic-tools">
|
||||||
|
<div className="client-traffic-option-row"><span>Период</span>
|
||||||
|
<div className="client-traffic-filters" role="group" aria-label="Период истории">
|
||||||
|
{([['24h', '24 часа'], ['7d', '7 дней'], ['30d', '30 дней'], ['90d', '90 дней']] as const).map(([value, label]) => <button
|
||||||
|
type="button" key={value} aria-pressed={query.range === value} onClick={() => change({ range: value })}
|
||||||
|
>{label}</button>)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="client-traffic-option-row"><span>Маршрут</span>
|
||||||
|
<div className="client-traffic-filters" role="group" aria-label="Маршрут в истории">
|
||||||
|
{([['all', 'Все'], ['vpn', 'VPN'], ['direct', 'Direct'], ['other', 'Другое']] as const).map(([value, label]) => <button
|
||||||
|
type="button" key={value} aria-pressed={query.route === value} onClick={() => change({ route: value })}
|
||||||
|
>{label}</button>)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{isGateway && <section className="client-traffic-devices" aria-label="Устройства в истории">
|
||||||
|
<div className="client-traffic-devices-heading"><h3>Устройства</h3>
|
||||||
|
<button type="button" aria-pressed={!query.originId} onClick={() => change({ originId: '' })}>Все устройства</button>
|
||||||
|
</div>
|
||||||
|
<label className="client-traffic-search is-device-search">
|
||||||
|
<span aria-hidden="true" /><span className="client-live-region">Найти устройство</span>
|
||||||
|
<input type="search" value={deviceSearch} aria-label="Найти устройство в истории" placeholder="Найти устройство" onChange={(e) => setDeviceSearch(e.target.value)} />
|
||||||
|
</label>
|
||||||
|
<div className="client-traffic-device-list" id="client-history-devices">
|
||||||
|
{(devicesExpanded ? origins : origins.slice(0, 3)).map((origin) => <button type="button" key={origin.id}
|
||||||
|
aria-pressed={query.originId === origin.id} onClick={() => change({ originId: query.originId === origin.id ? '' : origin.id })}
|
||||||
|
><strong>{origin.label}</strong></button>)}
|
||||||
|
</div>
|
||||||
|
{origins.length > 3 && <button type="button" className="client-traffic-devices-more" aria-expanded={devicesExpanded}
|
||||||
|
aria-controls="client-history-devices" onClick={() => setDevicesExpanded((value) => !value)}
|
||||||
|
>{devicesExpanded ? 'Свернуть устройства' : `Ещё ${origins.length - 3} устройств`}</button>}
|
||||||
|
{snapshot?.originsTruncated && <p className="client-traffic-notice">Показаны первые 256 устройств.</p>}
|
||||||
|
</section>}
|
||||||
|
<label className="client-traffic-search client-traffic-list-search">
|
||||||
|
<svg viewBox="0 0 24 24" aria-hidden="true"><circle cx="10.5" cy="10.5" r="6" /><path d="m15 15 5 5" /></svg>
|
||||||
|
<span className="client-live-region">Поиск истории</span>
|
||||||
|
<input type="search" value={query.search} aria-label="Найти сайт, IP или сервис в истории" placeholder="Найти сайт, IP или сервис"
|
||||||
|
onChange={(event) => change({ search: event.target.value })} />
|
||||||
|
</label>
|
||||||
|
<p className="client-traffic-history-status" role="status">{result.status === 'loading' ? 'Загружаем историю…'
|
||||||
|
: result.status === 'error' ? `История временно недоступна.${snapshot ? ' Показаны последние полученные данные.' : ''}`
|
||||||
|
: snapshot?.source === 'disabled' ? 'Сбор истории выключен; сохранённые данные доступны.'
|
||||||
|
: snapshot?.source === 'stopped' ? 'VPN остановлен; показана сохранённая история.'
|
||||||
|
: snapshot?.source === 'connecting' ? 'Сборщик подключается; показана сохранённая история.'
|
||||||
|
: snapshot?.coverage.partial ? 'В выбранном периоде есть пропуски или неточное распределение по времени.' : '\u00a0'}</p>
|
||||||
|
{snapshot && <>
|
||||||
|
<div className="client-traffic-summary" aria-label="Расход за выбранный период">
|
||||||
|
<span><b>Скачано</b>{formatByteString(snapshot.totals.downloadBytes)}</span>
|
||||||
|
<span><b>Отправлено</b>{formatByteString(snapshot.totals.uploadBytes)}</span>
|
||||||
|
</div>
|
||||||
|
{snapshot.rows.length ? <HistoryRows key={JSON.stringify(query)} snapshot={snapshot} active={active} load={load} />
|
||||||
|
: <p className="client-traffic-state">{query.search || query.originId || query.route !== 'all'
|
||||||
|
? 'По выбранным фильтрам ничего не найдено.' : 'История пока пуста. Данные появятся после начала сбора трафика.'}</p>}
|
||||||
|
<HistoryPages snapshot={snapshot} offset={query.offset} change={(offset) => setQuery((current) => ({ ...current, offset, until: snapshot.query.until }))} />
|
||||||
|
<p className="client-traffic-honesty">
|
||||||
|
Локальная история за 90 дней: завершённые минуты за последние 7 дней, ранее по часам.
|
||||||
|
{` Данные до ${new Date(snapshot.period.to).toLocaleString('ru-RU')}.`}
|
||||||
|
{snapshot.period.availableFrom && ` Сбор с ${new Date(snapshot.period.availableFrom).toLocaleString('ru-RU')}.`}
|
||||||
|
{snapshot.coverage.lastObservedAt && ` Обновлено ${new Date(snapshot.coverage.lastObservedAt).toLocaleString('ru-RU')}.`}
|
||||||
|
{' '}Внешний Prometheus не требуется. IP без домена не означает распознанный сайт.
|
||||||
|
</p>
|
||||||
|
</>}
|
||||||
|
</>;
|
||||||
|
}
|
||||||
@@ -4,3 +4,4 @@ export {
|
|||||||
useTrafficFeature,
|
useTrafficFeature,
|
||||||
type TrafficFeature,
|
type TrafficFeature,
|
||||||
} from './TrafficFeature.js';
|
} from './TrafficFeature.js';
|
||||||
|
export type { LoadTrafficHistory } from './TrafficHistoryPanel.js';
|
||||||
|
|||||||
@@ -3,6 +3,25 @@
|
|||||||
height: 24px;
|
height: 24px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.client-traffic-history-status {
|
||||||
|
min-height: 48px;
|
||||||
|
margin: 0 8px 12px;
|
||||||
|
color: var(--client-muted);
|
||||||
|
font: var(--type-control);
|
||||||
|
letter-spacing: var(--type-control-tracking);
|
||||||
|
text-transform: var(--type-control-transform);
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
|
||||||
|
.client-traffic-history-children {
|
||||||
|
padding-left: 8px;
|
||||||
|
animation: client-traffic-details-in 180ms cubic-bezier(0.16, 1, 0.3, 1) both;
|
||||||
|
}
|
||||||
|
|
||||||
|
.client-traffic-history-row > button:disabled {
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
.client-traffic-sheet {
|
.client-traffic-sheet {
|
||||||
padding: 54px 72px 72px 34px;
|
padding: 54px 72px 72px 34px;
|
||||||
}
|
}
|
||||||
@@ -590,6 +609,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media (prefers-reduced-motion: reduce) {
|
@media (prefers-reduced-motion: reduce) {
|
||||||
|
.client-traffic-history-children,
|
||||||
.client-traffic-connection,
|
.client-traffic-connection,
|
||||||
.client-traffic-details,
|
.client-traffic-details,
|
||||||
.client-traffic-device-list {
|
.client-traffic-device-list {
|
||||||
|
|||||||
@@ -103,6 +103,13 @@ test('native traffic contracts and collector restart both Gateway processes', ()
|
|||||||
'src/server/generated/daemon/started_service_pb.ts',
|
'src/server/generated/daemon/started_service_pb.ts',
|
||||||
'src/server/services/liveTrafficService.ts',
|
'src/server/services/liveTrafficService.ts',
|
||||||
'src/shared/liveTraffic.ts',
|
'src/shared/liveTraffic.ts',
|
||||||
|
'.node-version',
|
||||||
|
'scripts/check-sqlite-runtime.mjs',
|
||||||
|
'src/server/services/sqlite.ts',
|
||||||
|
'src/server/services/trafficHistoryStore.ts',
|
||||||
|
'src/server/services/trafficHistoryService.ts',
|
||||||
|
'src/server/services/trafficHistoryWorker.ts',
|
||||||
|
'src/shared/trafficHistory.ts',
|
||||||
]) {
|
]) {
|
||||||
assert.deepEqual(classifyRuntimeImpact([file]), {
|
assert.deepEqual(classifyRuntimeImpact([file]), {
|
||||||
affectedComponents: ['control', 'dataplane'],
|
affectedComponents: ['control', 'dataplane'],
|
||||||
|
|||||||
@@ -5,8 +5,20 @@ import path from 'node:path';
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
|
||||||
import { createActivityJournalService } from '../../dist/server/services/activityJournalService.js';
|
import { createActivityJournalService } from '../../dist/server/services/activityJournalService.js';
|
||||||
|
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
import { assertActivityJournalPage } from '../../dist/shared/activityJournal.js';
|
import { assertActivityJournalPage } from '../../dist/shared/activityJournal.js';
|
||||||
|
|
||||||
|
function serviceFor(t, filePath, now) {
|
||||||
|
const storage = openHarborStorage(path.dirname(filePath));
|
||||||
|
t.after(() => storage.close());
|
||||||
|
return createActivityJournalService({ db: storage.db, now });
|
||||||
|
}
|
||||||
|
function persisted(filePath) {
|
||||||
|
const db = new DatabaseSync(path.join(path.dirname(filePath), 'harbor.sqlite'), { readOnly: true });
|
||||||
|
try { return { schemaVersion: 1, events: db.prepare('SELECT value FROM journal ORDER BY sequence').all().map((row) => JSON.parse(row.value)) }; }
|
||||||
|
finally { db.close(); }
|
||||||
|
}
|
||||||
function fixture(t) {
|
function fixture(t) {
|
||||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-journal-'));
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-journal-'));
|
||||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||||
@@ -24,7 +36,7 @@ const event = (dedupeKey, profileLabel = 'Home') => ({
|
|||||||
test('journal appends typed events, deduplicates and keeps stable newest-first cursors', (t) => {
|
test('journal appends typed events, deduplicates and keeps stable newest-first cursors', (t) => {
|
||||||
let clock = new Date('2026-08-19T10:00:00.000Z');
|
let clock = new Date('2026-08-19T10:00:00.000Z');
|
||||||
const filePath = fixture(t);
|
const filePath = fixture(t);
|
||||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
const service = serviceFor(t, filePath, () => clock);
|
||||||
service.append(event('refresh:1', 'One'));
|
service.append(event('refresh:1', 'One'));
|
||||||
clock = new Date('2026-08-19T10:01:00.000Z');
|
clock = new Date('2026-08-19T10:01:00.000Z');
|
||||||
service.append(event('refresh:2', 'Two'));
|
service.append(event('refresh:2', 'Two'));
|
||||||
@@ -38,20 +50,20 @@ test('journal appends typed events, deduplicates and keeps stable newest-first c
|
|||||||
const older = service.page(10, first.nextCursor);
|
const older = service.page(10, first.nextCursor);
|
||||||
assert.deepEqual(older.events.map(({ data }) => data.profileLabel), ['One']);
|
assert.deepEqual(older.events.map(({ data }) => data.profileLabel), ['One']);
|
||||||
assert.equal(first.events[0].dedupeKey, null);
|
assert.equal(first.events[0].dedupeKey, null);
|
||||||
const inode = fs.statSync(filePath).ino;
|
const inode = fs.statSync(path.join(path.dirname(filePath), 'harbor.sqlite')).ino;
|
||||||
assert.equal(service.page(10).events.length, 3);
|
assert.equal(service.page(10).events.length, 3);
|
||||||
assert.equal(fs.statSync(filePath).ino, inode);
|
assert.equal(fs.statSync(path.join(path.dirname(filePath), 'harbor.sqlite')).ino, inode);
|
||||||
assert.deepEqual(service.page(10, 'expired-cursor').events, []);
|
assert.deepEqual(service.page(10, 'expired-cursor').events, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('journal prunes events older than 30 days and rejects unsafe payloads', (t) => {
|
test('journal prunes events older than 30 days and rejects unsafe payloads', (t) => {
|
||||||
let clock = new Date('2026-07-01T00:00:00.000Z');
|
let clock = new Date('2026-07-01T00:00:00.000Z');
|
||||||
const filePath = fixture(t);
|
const filePath = fixture(t);
|
||||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
const service = serviceFor(t, filePath, () => clock);
|
||||||
service.append(event('old'));
|
service.append(event('old'));
|
||||||
clock = new Date('2026-08-19T00:00:00.000Z');
|
clock = new Date('2026-08-19T00:00:00.000Z');
|
||||||
assert.deepEqual(service.page().events, []);
|
assert.deepEqual(service.page().events, []);
|
||||||
assert.deepEqual(JSON.parse(fs.readFileSync(filePath, 'utf8')).events, []);
|
assert.deepEqual(persisted(filePath).events, []);
|
||||||
service.append(event('new'));
|
service.append(event('new'));
|
||||||
assert.throws(() => service.append({ ...event('unsafe'), data: { rawUrl: 'https://secret' } }), /Unsafe/);
|
assert.throws(() => service.append({ ...event('unsafe'), data: { rawUrl: 'https://secret' } }), /Unsafe/);
|
||||||
service.append({ ...event('ip'), data: { ...event('ip').data, host: '192.168.1.1' } });
|
service.append({ ...event('ip'), data: { ...event('ip').data, host: '192.168.1.1' } });
|
||||||
@@ -68,9 +80,9 @@ test('journal prunes events older than 30 days and rejects unsafe payloads', (t)
|
|||||||
'subscription.refreshed:user:pass',
|
'subscription.refreshed:user:pass',
|
||||||
]) service.append({ ...event('safe'), dedupeKey });
|
]) service.append({ ...event('safe'), dedupeKey });
|
||||||
service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:user:pass' });
|
service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:user:pass' });
|
||||||
const persisted = fs.readFileSync(filePath, 'utf8');
|
const persistedText = JSON.stringify(persisted(filePath));
|
||||||
assert.doesNotMatch(persisted, /user:pass|192\.168\.1\.1|192\.168\.1\.7|2001:db8|token=secret/);
|
assert.doesNotMatch(persistedText, /user:pass|192\.168\.1\.1|192\.168\.1\.7|2001:db8|token=secret/);
|
||||||
assert.match(persisted, /subscription\.refreshed:sha256:[a-f0-9]{64}/);
|
assert.match(persistedText, /subscription\.refreshed:sha256:[a-f0-9]{64}/);
|
||||||
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'https://user:pass@example.test/private?token=x' }), /dedupe/i);
|
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'https://user:pass@example.test/private?token=x' }), /dedupe/i);
|
||||||
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:private/path' }), /dedupe/i);
|
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:private/path' }), /dedupe/i);
|
||||||
});
|
});
|
||||||
@@ -84,36 +96,24 @@ test('journal persists the 10,000 event cap when opening an oversized store', (t
|
|||||||
...event(`event:${index}`),
|
...event(`event:${index}`),
|
||||||
}));
|
}));
|
||||||
fs.writeFileSync(filePath, JSON.stringify({ schemaVersion: 1, events }));
|
fs.writeFileSync(filePath, JSON.stringify({ schemaVersion: 1, events }));
|
||||||
const service = createActivityJournalService({
|
const service = serviceFor(t, filePath, () => new Date('2026-08-19T01:00:00.000Z'));
|
||||||
filePath,
|
|
||||||
now: () => new Date('2026-08-19T01:00:00.000Z'),
|
|
||||||
});
|
|
||||||
assert.equal(JSON.parse(fs.readFileSync(filePath, 'utf8')).events.length, 10_000);
|
|
||||||
assert.equal(service.page(1).events.length, 1);
|
assert.equal(service.page(1).events.length, 1);
|
||||||
|
assert.equal(persisted(filePath).events.length, 10_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('corrupt journal is isolated and recovery becomes a safe event', (t) => {
|
test('corrupt journal blocks import and leaves the original intact', (t) => {
|
||||||
const filePath = fixture(t);
|
const filePath = fixture(t);
|
||||||
fs.writeFileSync(filePath, '{broken');
|
fs.writeFileSync(filePath, '{broken');
|
||||||
const service = createActivityJournalService({
|
assert.throws(() => serviceFor(t, filePath), /Cannot migrate activity-journal.json/);
|
||||||
filePath,
|
assert.equal(fs.readFileSync(filePath, 'utf8'), '{broken');
|
||||||
now: () => new Date('2026-08-19T12:00:00.000Z'),
|
|
||||||
});
|
|
||||||
const page = service.page();
|
|
||||||
assert.equal(page.storage.status, 'ready');
|
|
||||||
assert.equal(page.events[0].type, 'journal.recovered');
|
|
||||||
assert.ok(fs.readdirSync(path.dirname(filePath)).some((name) => name.includes('.corrupt-')));
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test('journal exposes a latched write failure until a later append succeeds', (t) => {
|
test('journal exposes a latched write failure until a later append succeeds', (t) => {
|
||||||
const filePath = fixture(t);
|
const filePath = fixture(t);
|
||||||
const service = createActivityJournalService({ filePath });
|
const service = serviceFor(t, filePath);
|
||||||
service.append(event('before-error'));
|
service.append(event('before-error'));
|
||||||
const renameSync = fs.renameSync;
|
const db = new DatabaseSync(path.join(path.dirname(filePath), 'harbor.sqlite'));
|
||||||
fs.renameSync = (source, target) => {
|
db.exec("CREATE TRIGGER reject_journal BEFORE INSERT ON journal BEGIN SELECT RAISE(ABORT, 'simulated journal failure'); END");
|
||||||
if (target === filePath) throw new Error('simulated journal write failure');
|
|
||||||
return renameSync(source, target);
|
|
||||||
};
|
|
||||||
try {
|
try {
|
||||||
assert.throws(() => service.append(event('lost')));
|
assert.throws(() => service.append(event('lost')));
|
||||||
const failed = service.page();
|
const failed = service.page();
|
||||||
@@ -122,7 +122,8 @@ test('journal exposes a latched write failure until a later append succeeds', (t
|
|||||||
assert.equal(failed.events.length, 1);
|
assert.equal(failed.events.length, 1);
|
||||||
assert.equal(failed.events[0].dedupeKey, null);
|
assert.equal(failed.events[0].dedupeKey, null);
|
||||||
} finally {
|
} finally {
|
||||||
fs.renameSync = renameSync;
|
db.exec('DROP TRIGGER reject_journal');
|
||||||
|
db.close();
|
||||||
}
|
}
|
||||||
service.append(event('recovered'));
|
service.append(event('recovered'));
|
||||||
assert.equal(service.page().storage.status, 'ready');
|
assert.equal(service.page().storage.status, 'ready');
|
||||||
@@ -145,7 +146,7 @@ test('schema version 1 keeps legacy recovery and accepts per-channel health even
|
|||||||
}],
|
}],
|
||||||
}));
|
}));
|
||||||
let clock = new Date('2026-08-19T10:00:00.000Z');
|
let clock = new Date('2026-08-19T10:00:00.000Z');
|
||||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
const service = serviceFor(t, filePath, () => clock);
|
||||||
const inputs = [
|
const inputs = [
|
||||||
['failover.primary_unavailable', 'primary', 'warning', 'probe-failed'],
|
['failover.primary_unavailable', 'primary', 'warning', 'probe-failed'],
|
||||||
['failover.primary_recovered', 'primary', 'info', 'probe-recovered'],
|
['failover.primary_recovered', 'primary', 'info', 'probe-recovered'],
|
||||||
@@ -173,10 +174,10 @@ test('schema version 1 keeps legacy recovery and accepts per-channel health even
|
|||||||
]);
|
]);
|
||||||
assert.deepEqual(page.events.at(-1).data, { role: 'primary', reason: 'primary-recovered' });
|
assert.deepEqual(page.events.at(-1).data, { role: 'primary', reason: 'primary-recovered' });
|
||||||
assert.equal(page.retentionDays, 30);
|
assert.equal(page.retentionDays, 30);
|
||||||
assert.equal(JSON.parse(fs.readFileSync(filePath, 'utf8')).schemaVersion, 1);
|
assert.equal(persisted(filePath).schemaVersion, 1);
|
||||||
});
|
});
|
||||||
test('journal page parser rejects malformed wire data and strips unknown event fields', (t) => {
|
test('journal page parser rejects malformed wire data and strips unknown event fields', (t) => {
|
||||||
const service = createActivityJournalService({ filePath: fixture(t) });
|
const service = serviceFor(t, fixture(t));
|
||||||
service.append(event('wire'));
|
service.append(event('wire'));
|
||||||
const page = service.page();
|
const page = service.page();
|
||||||
const parsed = assertActivityJournalPage({
|
const parsed = assertActivityJournalPage({
|
||||||
|
|||||||
@@ -108,6 +108,13 @@ test('compiled dispatcher starts and stops control and dataplane contracts', asy
|
|||||||
assert.equal(page.status, 200);
|
assert.equal(page.status, 200);
|
||||||
assert.match(page.type, /^text\/html/);
|
assert.match(page.type, /^text\/html/);
|
||||||
assert.match(page.body, /<div id="root"><\/div>/);
|
assert.match(page.body, /<div id="root"><\/div>/);
|
||||||
|
const historyResponse = await fetch(`http://127.0.0.1:${port}/api/traffic/history?range=90d`);
|
||||||
|
const history = await historyResponse.json();
|
||||||
|
assert.equal(historyResponse.status, 200);
|
||||||
|
assert.equal(history.storage.status, 'ready');
|
||||||
|
assert.equal(history.period.retentionDays, 90);
|
||||||
|
assert.deepEqual(history.rows, []);
|
||||||
|
assert.equal(fs.existsSync(path.join(controlData, 'harbor.sqlite')), true);
|
||||||
await stop(control.child);
|
await stop(control.child);
|
||||||
|
|
||||||
const dataplane = start({
|
const dataplane = start({
|
||||||
@@ -123,6 +130,11 @@ test('compiled dispatcher starts and stops control and dataplane contracts', asy
|
|||||||
return response.status === 200 ? response.body : null;
|
return response.status === 200 ? response.body : null;
|
||||||
}, dataplane.child, dataplane.stderr);
|
}, dataplane.child, dataplane.stderr);
|
||||||
assert.equal(status.ready, true);
|
assert.equal(status.ready, true);
|
||||||
|
const gatewayHistory = await socketRequest(socketPath, '/traffic/history?range=7d');
|
||||||
|
assert.equal(gatewayHistory.status, 200);
|
||||||
|
assert.equal(gatewayHistory.body.storage.status, 'ready');
|
||||||
|
assert.equal(gatewayHistory.body.query.range, '7d');
|
||||||
|
assert.equal(fs.existsSync(path.join(dataplaneData, 'traffic.sqlite')), true);
|
||||||
await stop(dataplane.child);
|
await stop(dataplane.child);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -148,7 +160,7 @@ test('production paths use only the compiled dispatcher', () => {
|
|||||||
assert.doesNotMatch(entrypoint, /\/app\/src/);
|
assert.doesNotMatch(entrypoint, /\/app\/src/);
|
||||||
}
|
}
|
||||||
assert.match(workflow, /npm run build:production/);
|
assert.match(workflow, /npm run build:production/);
|
||||||
assert.match(workflow, /NODE_BUILD_IMAGE: mirror\.gcr\.io\/library\/node:20\.19-bookworm/);
|
assert.match(workflow, /NODE_BUILD_IMAGE: mirror\.gcr\.io\/library\/node:24\.21\.0-bookworm/);
|
||||||
assert.match(workflow, /command -v npm[^']+command -v git[^']+test -x \/bin\/bash/);
|
assert.match(workflow, /command -v npm[^']+command -v git[^']+test -x \/bin\/bash/);
|
||||||
assert.doesNotMatch(workflow, /docker image inspect "\$\{\{ env\.NODE_BUILD_IMAGE \}\}"/);
|
assert.doesNotMatch(workflow, /docker image inspect "\$\{\{ env\.NODE_BUILD_IMAGE \}\}"/);
|
||||||
assert.match(legacyBuild, /npm run build:production && docker build/);
|
assert.match(legacyBuild, /npm run build:production && docker build/);
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||||
|
import { migrateStoredState } from '../../dist/server/services/stateStore.js';
|
||||||
|
import { migrateDeviceInventoryState } from '../../dist/server/services/deviceInventoryService.js';
|
||||||
|
import { createActivityJournalService } from '../../dist/server/services/activityJournalService.js';
|
||||||
|
|
||||||
|
function fixture(t) {
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-sqlite-'));
|
||||||
|
const stores = [];
|
||||||
|
t.after(() => {
|
||||||
|
for (const store of stores) if (store.db.isOpen) store.close();
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
return { directory, open: () => { const store = openHarborStorage(directory); stores.push(store); return store; } };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('atomic import preserves revisions, rules, device checkpoints and journal; JSON becomes backup only', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const state = migrateStoredState({ revision: 41, routeRules: [{ type: 'domain_suffix', value: 'example.org', enabled: true }] });
|
||||||
|
const mac = 'aa:bb:cc:dd:ee:ff';
|
||||||
|
const devices = migrateDeviceInventoryState({ revision: 17, traffic: {
|
||||||
|
baselinesByMac: { [mac]: { epoch: 'kernel-1', uploadBytes: '9007199254740993', downloadBytes: '123' } },
|
||||||
|
totalsByMac: { [mac]: { uploadBytes: '9007199254740993', downloadBytes: '987', observedAt: '2026-09-10T10:00:00.000Z' } },
|
||||||
|
} });
|
||||||
|
const event = { id: '00000000-0000-4000-8000-000000000001', occurredAt: new Date().toISOString(),
|
||||||
|
type: 'connection.stopped', severity: 'info', source: 'connection', dedupeKey: null, data: {} };
|
||||||
|
for (const [name, value] of [['state.json', state], ['devices.json', devices], ['activity-journal.json', { schemaVersion: 1, events: [event] }]]) {
|
||||||
|
fs.writeFileSync(path.join(f.directory, name), JSON.stringify(value));
|
||||||
|
}
|
||||||
|
const originals = ['state.json', 'devices.json', 'activity-journal.json'].map((name) => fs.readFileSync(path.join(f.directory, name), 'utf8'));
|
||||||
|
let store = f.open();
|
||||||
|
assert.equal(store.imported, true);
|
||||||
|
assert.deepEqual(store.state.read(), state);
|
||||||
|
assert.deepEqual(store.devices.read(), devices);
|
||||||
|
assert.deepEqual(createActivityJournalService({ db: store.db }).page().events, [event]);
|
||||||
|
store.state.update((value) => ({ ...value, revision: 42 }));
|
||||||
|
store.devices.update((value) => ({ ...value, revision: 18 }));
|
||||||
|
store.close();
|
||||||
|
// Even broken obsolete files cannot override or break the canonical SQL state.
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'state.json'), '{obsolete');
|
||||||
|
store = f.open();
|
||||||
|
assert.equal(store.imported, false);
|
||||||
|
assert.equal(store.state.read().revision, 42);
|
||||||
|
assert.equal(store.devices.read().revision, 18);
|
||||||
|
assert.equal(store.devices.read().traffic.baselinesByMac[mac].uploadBytes, '9007199254740993');
|
||||||
|
assert.equal(fs.readFileSync(path.join(f.directory, 'devices.json'), 'utf8'), originals[1]);
|
||||||
|
assert.equal(fs.readFileSync(path.join(f.directory, 'activity-journal.json'), 'utf8'), originals[2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('failed import leaves no partial documents and can be retried after fixing the original', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'state.json'), JSON.stringify({ revision: 9 }));
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'devices.json'), '{broken');
|
||||||
|
assert.throws(f.open, /Cannot migrate devices.json/);
|
||||||
|
assert.equal(fs.readFileSync(path.join(f.directory, 'devices.json'), 'utf8'), '{broken');
|
||||||
|
const inspect = new DatabaseSync(path.join(f.directory, 'harbor.sqlite'));
|
||||||
|
assert.equal(inspect.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'documents'").get().n, 0);
|
||||||
|
inspect.close();
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'devices.json'), '{}');
|
||||||
|
assert.equal(f.open().state.read().revision, 9);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an optional legacy null subscription cache migrates without changing the backup', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'state.json'), JSON.stringify({ schemaVersion: 4, revision: 7 }));
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'subscription-cache.json'), 'null');
|
||||||
|
assert.equal(f.open().state.read().revision, 7);
|
||||||
|
assert.equal(fs.readFileSync(path.join(f.directory, 'subscription-cache.json'), 'utf8'), 'null');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a database constraint failure rolls back the entire import including journal and marker', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const event = { id: '00000000-0000-4000-8000-000000000001', occurredAt: new Date().toISOString(),
|
||||||
|
type: 'connection.stopped', severity: 'info', source: 'connection', dedupeKey: null, data: {} };
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'activity-journal.json'), JSON.stringify({ schemaVersion: 1, events: [event, event] }));
|
||||||
|
assert.throws(f.open, /UNIQUE/);
|
||||||
|
const db = new DatabaseSync(path.join(f.directory, 'harbor.sqlite'));
|
||||||
|
assert.equal(db.prepare('PRAGMA user_version').get().user_version, 0);
|
||||||
|
assert.equal(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table'").get().n, 0);
|
||||||
|
db.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('durable DB corruption, future versions and invalid synchronous mutations never fall back to old JSON', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const store = f.open();
|
||||||
|
assert.throws(() => store.state.update(async (state) => state), /synchronous/);
|
||||||
|
assert.equal(store.state.read().revision, 0);
|
||||||
|
store.db.exec('PRAGMA user_version = 99');
|
||||||
|
store.close();
|
||||||
|
fs.writeFileSync(path.join(f.directory, 'state.json'), '{}');
|
||||||
|
assert.throws(f.open, /Unsupported Harbor database/);
|
||||||
|
});
|
||||||
@@ -5,6 +5,7 @@ import http from 'node:http';
|
|||||||
import os from 'node:os';
|
import os from 'node:os';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
|
||||||
import { buildGatewayPresence } from '../../dist/server/gatewayPresence.js';
|
import { buildGatewayPresence } from '../../dist/server/gatewayPresence.js';
|
||||||
import { normalizeSubscriptionConfig } from '../../dist/server/subscription.js';
|
import { normalizeSubscriptionConfig } from '../../dist/server/subscription.js';
|
||||||
@@ -151,6 +152,7 @@ async function startClientFixture(t, {
|
|||||||
hostNetwork,
|
hostNetwork,
|
||||||
gatewayPresencePort,
|
gatewayPresencePort,
|
||||||
trafficSource,
|
trafficSource,
|
||||||
|
expectMigrationError,
|
||||||
}) {
|
}) {
|
||||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-startup-recovery-'));
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-startup-recovery-'));
|
||||||
const { binDirectory, markerPath } = fakeSingbox(directory);
|
const { binDirectory, markerPath } = fakeSingbox(directory);
|
||||||
@@ -188,6 +190,15 @@ async function startClientFixture(t, {
|
|||||||
await stopChild(child);
|
await stopChild(child);
|
||||||
fs.rmSync(directory, { recursive: true, force: true });
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
if (expectMigrationError) {
|
||||||
|
await assert.rejects(waitForState(port, child, () => stderr), expectMigrationError);
|
||||||
|
assert.notEqual(child.exitCode, 0);
|
||||||
|
assert.equal(fs.existsSync(markerPath), false);
|
||||||
|
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'state.json'), 'utf8')), state);
|
||||||
|
assert.equal(fs.readFileSync(path.join(directory, 'subscription-cache.json'), 'utf8'), cacheContents);
|
||||||
|
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'sing-box-config.json'), 'utf8')), config);
|
||||||
|
return { directory, markerPath };
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
directory,
|
directory,
|
||||||
markerPath,
|
markerPath,
|
||||||
@@ -257,7 +268,9 @@ test('gateway-direct boot keeps the local proxy and diagnostics but omits every
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
const config = JSON.parse(fs.readFileSync(path.join(fixture.directory, 'sing-box-config.json'), 'utf8'));
|
const config = JSON.parse(fs.readFileSync(path.join(fixture.directory, 'sing-box-config.json'), 'utf8'));
|
||||||
const stored = JSON.parse(fs.readFileSync(path.join(fixture.directory, 'state.json'), 'utf8'));
|
const db = new DatabaseSync(path.join(fixture.directory, 'harbor.sqlite'), { readOnly: true });
|
||||||
|
const stored = JSON.parse(db.prepare("SELECT value FROM documents WHERE key = 'state'").get().value);
|
||||||
|
db.close();
|
||||||
|
|
||||||
assert.equal(fixture.state.route.mode, 'gateway-direct');
|
assert.equal(fixture.state.route.mode, 'gateway-direct');
|
||||||
assert.deepEqual(fixture.state.route.activeLocalRules, []);
|
assert.deepEqual(fixture.state.route.activeLocalRules, []);
|
||||||
@@ -279,23 +292,15 @@ test('corrupt legacy cache with no canonical subscription fails closed instead o
|
|||||||
port: 443,
|
port: 443,
|
||||||
protocol: 'vless',
|
protocol: 'vless',
|
||||||
};
|
};
|
||||||
const fixture = await startClientFixture(t, {
|
await startClientFixture(t, {
|
||||||
state: { schemaVersion: 4, revision: 2, connectionDesired: 'running' },
|
state: { schemaVersion: 4, revision: 2, connectionDesired: 'running' },
|
||||||
cacheContents: '{broken',
|
cacheContents: '{broken',
|
||||||
config: generatedConfig(staleServer),
|
config: generatedConfig(staleServer),
|
||||||
|
expectMigrationError: /Cannot migrate subscription-cache.json/,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
assert.equal(fixture.state.subscription.status, 'missing');
|
test('legacy cache owned by another URL aborts migration without mixing providers or changing originals', async (t) => {
|
||||||
assert.deepEqual(fixture.state.profiles, []);
|
|
||||||
assert.equal(fixture.state.connection.desired, 'stopped');
|
|
||||||
assert.equal(fixture.state.connection.process, 'stopped');
|
|
||||||
assert.equal(fs.existsSync(fixture.markerPath), false);
|
|
||||||
assert.ok(fs.readdirSync(fixture.directory).some((name) => (
|
|
||||||
name.startsWith('subscription-cache.json.corrupt-')
|
|
||||||
)));
|
|
||||||
});
|
|
||||||
|
|
||||||
test('legacy cache owned by another URL is backed up without mixing providers and boots stopped', async (t) => {
|
|
||||||
const stateServer = {
|
const stateServer = {
|
||||||
id: 'server-a',
|
id: 'server-a',
|
||||||
label: 'State server',
|
label: 'State server',
|
||||||
@@ -314,7 +319,8 @@ test('legacy cache owned by another URL is backed up without mixing providers an
|
|||||||
server: 'cache.example',
|
server: 'cache.example',
|
||||||
server_port: 8443,
|
server_port: 8443,
|
||||||
};
|
};
|
||||||
const fixture = await startClientFixture(t, {
|
await startClientFixture(t, {
|
||||||
|
expectMigrationError: /owner mismatch/,
|
||||||
state: {
|
state: {
|
||||||
schemaVersion: 4,
|
schemaVersion: 4,
|
||||||
revision: 2,
|
revision: 2,
|
||||||
@@ -331,20 +337,6 @@ test('legacy cache owned by another URL is backed up without mixing providers an
|
|||||||
}),
|
}),
|
||||||
config: generatedConfig(stateServer),
|
config: generatedConfig(stateServer),
|
||||||
});
|
});
|
||||||
|
|
||||||
assert.equal(fixture.state.profiles.length, 1);
|
|
||||||
assert.equal(fixture.state.profiles[0].subscription.host, 'state.example/…');
|
|
||||||
assert.deepEqual(fixture.state.profiles[0].servers.map(({ id }) => id), [stateServer.id]);
|
|
||||||
assert.equal(JSON.stringify(fixture.state).includes('cache.example'), false);
|
|
||||||
assert.equal(fixture.state.connection.desired, 'stopped');
|
|
||||||
assert.equal(fixture.state.connection.process, 'stopped');
|
|
||||||
assert.equal(fixture.state.selection.appliedServerId, '');
|
|
||||||
assert.equal(fs.existsSync(path.join(fixture.directory, 'subscription-cache.json')), false);
|
|
||||||
assert.ok(fs.readdirSync(fixture.directory).some((name) => (
|
|
||||||
name.startsWith('subscription-cache.json.backup-v1-')
|
|
||||||
)));
|
|
||||||
assert.equal(fs.existsSync(path.join(fixture.directory, 'sing-box-config.json')), false);
|
|
||||||
assert.equal(fs.existsSync(fixture.markerPath), false);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test('boot rejects an existing config whose route mode disagrees with the current route', async (t) => {
|
test('boot rejects an existing config whose route mode disagrees with the current route', async (t) => {
|
||||||
|
|||||||
@@ -5,6 +5,13 @@ import http from 'node:http';
|
|||||||
import os from 'node:os';
|
import os from 'node:os';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
|
||||||
|
function readState(directory) {
|
||||||
|
const db = new DatabaseSync(path.join(directory, 'harbor.sqlite'), { readOnly: true });
|
||||||
|
try { return JSON.parse(db.prepare("SELECT value FROM documents WHERE key = 'state'").get().value); }
|
||||||
|
finally { db.close(); }
|
||||||
|
}
|
||||||
|
|
||||||
import {
|
import {
|
||||||
assertStateSnapshot,
|
assertStateSnapshot,
|
||||||
@@ -99,7 +106,7 @@ test('state v1 projects legacy storage through the canonical profile snapshot',
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('startup discards a rejected cached subscription and returns to first-run', async (t) => {
|
test('startup rejects an invalid legacy subscription without changing migration originals', async (t) => {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-rejected-cache-'));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-rejected-cache-'));
|
||||||
const port = await freePort();
|
const port = await freePort();
|
||||||
const subscriptionUrl = 'https://provider.example/disabled';
|
const subscriptionUrl = 'https://provider.example/disabled';
|
||||||
@@ -141,16 +148,11 @@ test('startup discards a rejected cached subscription and returns to first-run',
|
|||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
const state = await waitForState(port, child, () => stderr);
|
await assert.rejects(waitForState(port, child, () => stderr), /Harbor exited early/);
|
||||||
assert.equal(state.subscription.status, 'missing');
|
assert.notEqual(child.exitCode, 0);
|
||||||
assert.equal(state.hasSubscription, false);
|
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||||
assert.deepEqual(state.servers, []);
|
assert.equal(fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8'), '{}');
|
||||||
assert.ok(state.route.localRules.some((rule) => (
|
assert.equal(JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8')).subscriptionUrl, subscriptionUrl);
|
||||||
rule.type === 'domain_suffix' && rule.value === 'example.org' && rule.enabled
|
|
||||||
)));
|
|
||||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
|
||||||
assert.equal(fs.existsSync(path.join(dir, 'sing-box-config.json')), false);
|
|
||||||
assert.equal(child.exitCode, null);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test('data invariant: canonical profile API mutates one snapshot and preserves migrated profile data', async (t) => {
|
test('data invariant: canonical profile API mutates one snapshot and preserves migrated profile data', async (t) => {
|
||||||
@@ -305,15 +307,15 @@ setInterval(() => {}, 60_000);
|
|||||||
assert.equal(initial.route.localRulesRevision, 0);
|
assert.equal(initial.route.localRulesRevision, 0);
|
||||||
assert.equal(initial.route.localRulesPendingRestart, false);
|
assert.equal(initial.route.localRulesPendingRestart, false);
|
||||||
assert.equal(JSON.stringify(initial).includes(subscriptionUrl), false);
|
assert.equal(JSON.stringify(initial).includes(subscriptionUrl), false);
|
||||||
const migratedState = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
const migratedState = readState(dir);
|
||||||
assert.equal(migratedState.schemaVersion, STATE_SCHEMA_VERSION);
|
assert.equal(migratedState.schemaVersion, STATE_SCHEMA_VERSION);
|
||||||
assert.equal(migratedState.profiles.length, 1);
|
assert.equal(migratedState.profiles.length, 1);
|
||||||
assert.equal(migratedState.profiles[0].subscriptionUrl, subscriptionUrl);
|
assert.equal(migratedState.profiles[0].subscriptionUrl, subscriptionUrl);
|
||||||
assert.equal(migratedState.profiles[0].desiredServerId, testServerId);
|
assert.equal(migratedState.profiles[0].desiredServerId, testServerId);
|
||||||
assert.equal(migratedState.profiles[0].subscriptionConfig.outbounds[0].tag, testServerId);
|
assert.equal(migratedState.profiles[0].subscriptionConfig.outbounds[0].tag, testServerId);
|
||||||
assert.equal(Object.hasOwn(migratedState, 'subscriptionUrl'), false);
|
assert.equal(Object.hasOwn(migratedState, 'subscriptionUrl'), false);
|
||||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||||
assert.ok(fs.readdirSync(dir).some((name) => name.startsWith('subscription-cache.json.backup-v1-')));
|
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'subscription-cache.json'), 'utf8')), { url: subscriptionUrl, config });
|
||||||
const stateKeys = Object.keys(initial).sort();
|
const stateKeys = Object.keys(initial).sort();
|
||||||
assert.deepEqual(stateKeys, [
|
assert.deepEqual(stateKeys, [
|
||||||
'apiVersion',
|
'apiVersion',
|
||||||
@@ -373,7 +375,7 @@ setInterval(() => {}, 60_000);
|
|||||||
|
|
||||||
const primaryProfileId = initial.profiles[0].id;
|
const primaryProfileId = initial.profiles[0].id;
|
||||||
const preservedPrimary = structuredClone(
|
const preservedPrimary = structuredClone(
|
||||||
JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8')).profiles[0],
|
readState(dir).profiles[0],
|
||||||
);
|
);
|
||||||
const preservedConfig = fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8');
|
const preservedConfig = fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8');
|
||||||
for (const [pathname, expectedCode] of [
|
for (const [pathname, expectedCode] of [
|
||||||
@@ -394,9 +396,9 @@ setInterval(() => {}, 60_000);
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
assert.equal(failedAdd.payload.error.code, expectedCode);
|
assert.equal(failedAdd.payload.error.code, expectedCode);
|
||||||
const storedAfterFailure = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
const storedAfterFailure = readState(dir);
|
||||||
assert.deepEqual(storedAfterFailure.profiles, [preservedPrimary]);
|
assert.deepEqual(storedAfterFailure.profiles, [preservedPrimary]);
|
||||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8'),
|
fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8'),
|
||||||
preservedConfig,
|
preservedConfig,
|
||||||
@@ -475,7 +477,7 @@ setInterval(() => {}, 60_000);
|
|||||||
assert.equal(added.state.selection.desiredProfileId, primaryProfileId);
|
assert.equal(added.state.selection.desiredProfileId, primaryProfileId);
|
||||||
assert.equal(added.state.profiles.find(({ id }) => id === workProfileId).desiredServerId, '');
|
assert.equal(added.state.profiles.find(({ id }) => id === workProfileId).desiredServerId, '');
|
||||||
assert.equal(
|
assert.equal(
|
||||||
JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'))
|
readState(dir)
|
||||||
.profiles.find(({ id }) => id === workProfileId).subscriptionConfig.outbounds[0].tag,
|
.profiles.find(({ id }) => id === workProfileId).subscriptionConfig.outbounds[0].tag,
|
||||||
testServerId,
|
testServerId,
|
||||||
);
|
);
|
||||||
@@ -517,7 +519,7 @@ setInterval(() => {}, 60_000);
|
|||||||
);
|
);
|
||||||
assert.equal(failedRefresh.response.status, 400);
|
assert.equal(failedRefresh.response.status, 400);
|
||||||
assert.equal(failedRefresh.payload.error.code, 'SUBSCRIPTION_INVALID');
|
assert.equal(failedRefresh.payload.error.code, 'SUBSCRIPTION_INVALID');
|
||||||
const storedAfterFailedRefresh = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
const storedAfterFailedRefresh = readState(dir);
|
||||||
const staleWorkProfile = storedAfterFailedRefresh.profiles.find(({ id }) => id === workProfileId);
|
const staleWorkProfile = storedAfterFailedRefresh.profiles.find(({ id }) => id === workProfileId);
|
||||||
assert.equal(staleWorkProfile.desiredServerId, testServerId);
|
assert.equal(staleWorkProfile.desiredServerId, testServerId);
|
||||||
assert.equal(staleWorkProfile.lastRefreshErrorCode, 'SUBSCRIPTION_INVALID');
|
assert.equal(staleWorkProfile.lastRefreshErrorCode, 'SUBSCRIPTION_INVALID');
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { createTrafficHistoryRoute } from '../../dist/server/http/routes/trafficHistoryRoute.js';
|
||||||
|
import { emptyTrafficHistory, parseTrafficHistoryQuery } from '../../dist/shared/trafficHistory.js';
|
||||||
|
|
||||||
|
function response() {
|
||||||
|
return { writeHead(status) { this.status = status; }, end(body) { this.payload = JSON.parse(body); } };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('history API reads only its local source, enriches labels and returns explicit unavailable coverage', async () => {
|
||||||
|
const route = createTrafficHistoryRoute({
|
||||||
|
readHistory: async (query) => ({ ...emptyTrafficHistory(query, 'live'), origins: [{ id: 'dev-a', label: 'IP' }] }),
|
||||||
|
deviceInventory: { snapshot: () => ({ devices: [{ id: 'dev-a', alias: 'Ноутбук' }] }) },
|
||||||
|
});
|
||||||
|
const res = response();
|
||||||
|
assert.equal(await route.handle({ method: 'GET', url: '/api/traffic/history?range=90d&search=yandex' }, res), true);
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
assert.equal(res.payload.query.search, 'yandex');
|
||||||
|
assert.equal(res.payload.period.retentionDays, 90);
|
||||||
|
assert.equal(res.payload.origins[0].label, 'Ноутбук');
|
||||||
|
const failed = createTrafficHistoryRoute({ readHistory: async () => { throw Error('disk'); } });
|
||||||
|
await failed.handle({ method: 'GET', url: '/api/traffic/history' }, res);
|
||||||
|
assert.equal(res.payload.storage.status, 'error');
|
||||||
|
assert.equal(res.payload.coverage.partial, true);
|
||||||
|
await createTrafficHistoryRoute({ readHistory: null }).handle({ method: 'GET', url: '/api/traffic/history' }, res);
|
||||||
|
assert.equal(res.payload.source, 'disabled');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('history API rejects invalid queries and mutation methods before reading storage', async () => {
|
||||||
|
let reads = 0;
|
||||||
|
const route = createTrafficHistoryRoute({ readHistory: async (query) => { reads++; return emptyTrafficHistory(query); } });
|
||||||
|
for (const params of ['range=forever', 'level=ip%3BDROP', 'route=no', 'offset=-1', 'until=Infinity', 'search=%00', `search=${'x'.repeat(201)}`]) {
|
||||||
|
await assert.rejects(route.handle({ method: 'GET', url: `/api/traffic/history?${params}` }, response()),
|
||||||
|
(error) => error.code === 'REQUEST_INVALID');
|
||||||
|
}
|
||||||
|
await assert.rejects(route.handle({ method: 'DELETE', url: '/api/traffic/history' }, response()),
|
||||||
|
(error) => error.code === 'ENDPOINT_NOT_FOUND');
|
||||||
|
assert.equal(await route.handle({ method: 'GET', url: '/api/state' }, response()), false);
|
||||||
|
assert.equal(reads, 0);
|
||||||
|
assert.equal(parseTrafficHistoryQuery(new URLSearchParams('range=90d')).range, '90d');
|
||||||
|
});
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
|
import { performance } from 'node:perf_hooks';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { openTrafficHistoryStore } from '../../dist/server/services/trafficHistoryStore.js';
|
||||||
|
import { createTrafficHistoryService } from '../../dist/server/services/trafficHistoryService.js';
|
||||||
|
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||||
|
import { createDomainTrafficService } from '../../dist/server/services/domainTrafficService.js';
|
||||||
|
import { assertTrafficHistorySnapshot, parseTrafficHistoryQuery } from '../../dist/shared/trafficHistory.js';
|
||||||
|
|
||||||
|
const DAY = 86_400_000;
|
||||||
|
const base = Date.parse('2026-06-01T10:00:00.000Z');
|
||||||
|
const query = (fields = {}) => ({ ...parseTrafficHistoryQuery(new URLSearchParams('range=90d')), ...fields });
|
||||||
|
function connection(id, up, down, fields = {}) {
|
||||||
|
return {
|
||||||
|
id, startedAt: new Date(base + 1_000).toISOString(), closedAt: null,
|
||||||
|
inbound: { tag: 'tproxy-in', type: 'tproxy' }, network: 'tcp', protocol: 'tls',
|
||||||
|
source: { ip: '192.0.2.10', port: 50_000 },
|
||||||
|
destination: { domain: 'www.yandex.ru', ip: '203.0.113.10', port: 443, provenance: 'sing-box' },
|
||||||
|
origin: { kind: 'device', id: 'dev_0123456789abcdef', label: 'Laptop', provenance: 'source-ip' },
|
||||||
|
route: { kind: 'vpn', scope: 'local-sing-box', outbound: 'vpn-one', outboundType: 'vless', chain: [], rule: null },
|
||||||
|
traffic: { uploadBytes: String(up), downloadBytes: String(down), uploadBytesPerSecond: '0', downloadBytesPerSecond: '0' },
|
||||||
|
...fields,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const batch = (at, connections = [], reset = false, epoch = 'sing-box-1') => ({ epoch,
|
||||||
|
observedAt: new Date(at).toISOString(), connections, reset, closedIds: connections.filter((c) => c.closedAt).map((c) => c.id) });
|
||||||
|
function fixture(t) {
|
||||||
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-traffic-sql-'));
|
||||||
|
const closers = new Set();
|
||||||
|
t.after(async () => {
|
||||||
|
for (const close of closers) await close();
|
||||||
|
fs.rmSync(directory, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
return { directory, file: path.join(directory, 'traffic.sqlite'), register(store) {
|
||||||
|
const original = store.close;
|
||||||
|
store.close = () => { closers.delete(store.close); return original(); };
|
||||||
|
closers.add(store.close);
|
||||||
|
return store;
|
||||||
|
} };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('SQL checkpoints survive reopen, reset/replayed CLOSED and same-UUID new lifecycles without double count', (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
let clock = base;
|
||||||
|
let store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', 10, 20)])], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', 15, 27)])], 'live');
|
||||||
|
store.close();
|
||||||
|
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [connection('a', 15, 27)], true)], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
const closed = connection('a', 18, 30, { closedAt: new Date(clock).toISOString() });
|
||||||
|
store.ingest([batch(clock, [closed])], 'live');
|
||||||
|
store.close();
|
||||||
|
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [closed], true)], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.deepEqual(store.query(query()).totals, { uploadBytes: '18', downloadBytes: '30' });
|
||||||
|
store.ingest([batch(clock, [connection('a', 7, 9, { startedAt: new Date(clock).toISOString() })])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.deepEqual(store.query(query()).totals, { uploadBytes: '25', downloadBytes: '39' });
|
||||||
|
const result = assertTrafficHistorySnapshot(store.query(query()));
|
||||||
|
assert.equal(result.rows[0].label, 'Яндекс');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('full hostname, PSL registered domain, separate IP/device/routes and IP-only destinations remain queryable', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [
|
||||||
|
connection('a', 10, 20),
|
||||||
|
connection('b', 3, 4, { destination: { domain: 'mail.yandex.com', ip: '203.0.113.20' } }),
|
||||||
|
connection('c', 5, 6, { destination: { domain: 'api.example.co.uk', ip: '203.0.113.30' } }),
|
||||||
|
connection('d', 7, 8, { destination: { domain: null, ip: '2001:db8::1' }, origin: { kind: 'unknown', id: null, label: 'Unknown' } }),
|
||||||
|
])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
const roots = store.query(query());
|
||||||
|
assert.deepEqual(new Set(roots.rows.map((row) => row.key)), new Set(['Яндекс', 'example.co.uk', '']));
|
||||||
|
const domains = store.query(query({ level: 'domain', service: 'Яндекс' }));
|
||||||
|
assert.deepEqual(new Set(domains.rows.map((row) => row.key)), new Set(['yandex.ru', 'yandex.com']));
|
||||||
|
assert.equal(store.query(query({ level: 'hostname', service: 'Яндекс', domain: 'yandex.ru' })).rows[0].key, 'www.yandex.ru');
|
||||||
|
const ip = store.query(query({ level: 'ip', service: 'Яндекс', domain: 'yandex.ru', hostname: 'www.yandex.ru' }));
|
||||||
|
assert.equal(ip.rows[0].key, '203.0.113.10');
|
||||||
|
assert.equal(store.query(query({ level: 'ip', service: '', domain: '', hostname: '' })).rows[0].key, '2001:db8::1');
|
||||||
|
assert.equal(store.query(query({ originId: 'unknown:192.0.2.10' })).totals.downloadBytes, '8');
|
||||||
|
assert.equal(store.query(query({ search: "x' OR 1=1 --" })).rows.length, 0);
|
||||||
|
assert.equal(store.query(query({ search: 'яндекс' })).totals.downloadBytes, '24');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('initial snapshot is a baseline and downtime is explicitly partial', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [connection('old', 1_000, 2_000, { startedAt: new Date(base - DAY).toISOString() })], true)], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
assert.equal(store.query(query()).totals.uploadBytes, '0');
|
||||||
|
store.ingest([batch(clock, [connection('old', 1_010, 2_020, { startedAt: new Date(base - DAY).toISOString() })])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
store.ingest([batch(clock, [connection('old', 1_040, 2_050, { startedAt: new Date(base - DAY).toISOString() })], true)], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.deepEqual(store.query(query()).totals, { uploadBytes: '40', downloadBytes: '50' });
|
||||||
|
assert.equal(store.query(query()).coverage.partial, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('counter regression retains high watermarks and reconnect counts new post-collection lifecycles', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live');
|
||||||
|
for (const total of [100, 90, 100]) {
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', total, total)], true)], 'live');
|
||||||
|
}
|
||||||
|
clock += 60_000;
|
||||||
|
const result = store.query(query());
|
||||||
|
assert.deepEqual(result.totals, { uploadBytes: '100', downloadBytes: '100' });
|
||||||
|
assert.equal(result.coverage.partial, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('fast collector reopen marks missing coverage even if the next source state is live', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
let store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live');
|
||||||
|
store.close(); clock += 1_000;
|
||||||
|
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [connection('a', 10, 20)], true)], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.equal(store.query(query()).coverage.gapCount, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('frozen history cutoff excludes its incomplete terminal bucket before and after hourly rollup', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', 100, 200)])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', 110, 220)])], 'live');
|
||||||
|
const frozen = store.query(query());
|
||||||
|
assert.equal(frozen.query.until, base + 60_000);
|
||||||
|
assert.equal(frozen.totals.uploadBytes, '100');
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', 120, 240)])], 'live');
|
||||||
|
assert.deepEqual(store.query(frozen.query).totals, frozen.totals);
|
||||||
|
clock = base + 8 * DAY;
|
||||||
|
const hourly = store.query(query({ until: base + 3_600_000 + 30_000 }));
|
||||||
|
assert.equal(hourly.query.until, base + 3_600_000);
|
||||||
|
assert.equal(hourly.totals.uploadBytes, '120');
|
||||||
|
const expired = store.query(query({ until: base - 100 * DAY }));
|
||||||
|
assert.equal(expired.rows.length, 0);
|
||||||
|
assert.equal(expired.period.from, expired.period.to);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('minute to hour rollup preserves exact bytes and identities; 90-day cleanup leaves settings and metrics untouched', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
const settings = f.register(openHarborStorage(f.directory));
|
||||||
|
settings.state.update((state) => ({ ...state, revision: 42 }));
|
||||||
|
const metrics = createDomainTrafficService({ observe: () => ({ connections: [] }), devices: () => [] });
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
const data = batch(clock, [connection('a', '9007199254740993', '500')]);
|
||||||
|
store.ingest([data], 'live'); metrics.ingestNative(data);
|
||||||
|
const metricsBefore = JSON.stringify(metrics.snapshot());
|
||||||
|
clock += 60_000;
|
||||||
|
store.ingest([batch(clock, [connection('a', '9007199254741000', '550')])], 'live');
|
||||||
|
clock = base + 8 * DAY;
|
||||||
|
store.maintain(); store.maintain();
|
||||||
|
assert.deepEqual(store.query(query()).totals, { uploadBytes: '9007199254741000', downloadBytes: '550' });
|
||||||
|
const inspect = new DatabaseSync(f.file);
|
||||||
|
assert.deepEqual(inspect.prepare('SELECT DISTINCT resolution FROM buckets').all().map((r) => r.resolution), [3_600_000]);
|
||||||
|
inspect.close();
|
||||||
|
clock = base + 91 * DAY;
|
||||||
|
store.maintain();
|
||||||
|
assert.equal(store.query(query()).rows.length, 0);
|
||||||
|
assert.equal(settings.state.read().revision, 42);
|
||||||
|
assert.equal(JSON.stringify(metrics.snapshot()), metricsBefore);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a failed bucket transaction does not advance checkpoints, and retry counts exactly once', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
const blocker = new DatabaseSync(f.file);
|
||||||
|
blocker.exec("CREATE TRIGGER reject_bucket BEFORE INSERT ON buckets BEGIN SELECT RAISE(ABORT, 'simulated disk failure'); END");
|
||||||
|
const data = batch(clock, [connection('a', 10, 20)]);
|
||||||
|
assert.throws(() => store.ingest([data], 'live'), /simulated/);
|
||||||
|
assert.equal(blocker.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||||
|
blocker.exec('DROP TRIGGER reject_bucket'); blocker.close();
|
||||||
|
store.ingest([data], 'live'); store.ingest([data], 'live'); clock += 60_000;
|
||||||
|
assert.deepEqual(store.query(query()).totals, { uploadBytes: '10', downloadBytes: '20' });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('idle active checkpoints survive retention; a new runtime epoch removes old checkpoints but not buckets', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('idle', 100, 200)])], 'live');
|
||||||
|
clock += 91 * DAY;
|
||||||
|
store.maintain();
|
||||||
|
store.ingest([batch(clock, [connection('idle', 110, 220)])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||||
|
store.ingest([batch(clock, [], true, 'sing-box-2')], 'live');
|
||||||
|
const db = new DatabaseSync(f.file);
|
||||||
|
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||||
|
db.close();
|
||||||
|
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('reset disappearance and terminal identities retire checkpoints without forgetting still-active baselines', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('gone', 10, 20), connection('idle', 100, 200), connection('terminal', 7, 8)])], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([{ ...batch(clock), closedIds: ['terminal'] }], 'live');
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, [connection('idle', 100, 200)], true)], 'live');
|
||||||
|
clock += 91 * DAY;
|
||||||
|
store.maintain();
|
||||||
|
const db = new DatabaseSync(f.file);
|
||||||
|
assert.deepEqual(db.prepare("SELECT json_extract(identity, '$[1]') AS id, closed FROM checkpoints").all()
|
||||||
|
.map((row) => ({ ...row })), [{ id: 'idle', closed: 0 }]);
|
||||||
|
db.close();
|
||||||
|
store.ingest([batch(clock, [connection('idle', 110, 220)])], 'live');
|
||||||
|
clock += 60_000;
|
||||||
|
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('worker ingestion is independent of Prometheus and errors remain isolated; many closed events are not UI-capped', async (t) => {
|
||||||
|
const f = fixture(t);
|
||||||
|
const service = f.register(createTrafficHistoryService({ filePath: f.file, source: () => 'live' }));
|
||||||
|
const at = Date.now() - 120_000;
|
||||||
|
service.enqueue(batch(at, [], true));
|
||||||
|
service.enqueue(batch(at + 1, Array.from({ length: 2_049 }, (_, index) => connection(`closed-${index}`, 1, 2, {
|
||||||
|
startedAt: new Date(at + 1).toISOString(), closedAt: new Date(at + 1).toISOString(),
|
||||||
|
}))));
|
||||||
|
let ticked = false;
|
||||||
|
setImmediate(() => { ticked = true; });
|
||||||
|
const result = await service.query(query());
|
||||||
|
assert.equal(ticked, true);
|
||||||
|
assert.equal(result.storage.status, 'ready');
|
||||||
|
assert.deepEqual(result.totals, { uploadBytes: '2049', downloadBytes: '4098' });
|
||||||
|
const invalid = f.register(createTrafficHistoryService({ filePath: f.directory, source: () => 'live' }));
|
||||||
|
assert.doesNotThrow(() => invalid.enqueue(batch(at, [])));
|
||||||
|
assert.equal((await invalid.query(query())).storage.status, 'error');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('bounded 90-day history performance sample reports write/query size without claiming a hardware guarantee', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live');
|
||||||
|
const start = performance.now();
|
||||||
|
for (let day = 0; day < 90; day++) {
|
||||||
|
clock = base + day * DAY + 1_000;
|
||||||
|
store.ingest([batch(clock, Array.from({ length: 100 }, (_, index) => connection(`${day}-${index}`, 100, 1_000, {
|
||||||
|
startedAt: new Date(clock).toISOString(), destination: { domain: `host${index}.example.org`, ip: `203.0.113.${index + 1}` },
|
||||||
|
})))], 'live');
|
||||||
|
}
|
||||||
|
const writeMs = performance.now() - start;
|
||||||
|
clock += 60_000;
|
||||||
|
const queryStart = performance.now();
|
||||||
|
const result = store.query(query());
|
||||||
|
const queryMs = performance.now() - queryStart;
|
||||||
|
assert.equal(result.totals.downloadBytes, '9000000');
|
||||||
|
const sizes = Object.fromEntries(['', '-wal'].map((suffix) => [suffix || 'db', fs.statSync(f.file + suffix).size]));
|
||||||
|
t.diagnostic(JSON.stringify({ samples: 9_000, simulatedDays: 90, writeMs, queryMs, sizes }));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('high-churn sample measures closed-lifecycle storage and epoch reclamation', (t) => {
|
||||||
|
const f = fixture(t); let clock = base;
|
||||||
|
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||||
|
store.ingest([batch(clock, [], true)], 'live');
|
||||||
|
const start = performance.now();
|
||||||
|
for (let page = 0; page < 100; page++) {
|
||||||
|
clock += 1_000;
|
||||||
|
store.ingest([batch(clock, Array.from({ length: 1_000 }, (_, index) => connection(`${page}-${index}`, 100, 1_000, {
|
||||||
|
startedAt: new Date(clock).toISOString(), closedAt: new Date(clock).toISOString(),
|
||||||
|
})))], 'live');
|
||||||
|
}
|
||||||
|
const writeMs = performance.now() - start;
|
||||||
|
clock += 60_000;
|
||||||
|
const startQuery = performance.now();
|
||||||
|
assert.equal(store.query(query()).totals.downloadBytes, '100000000');
|
||||||
|
const queryMs = performance.now() - startQuery;
|
||||||
|
const db = new DatabaseSync(f.file);
|
||||||
|
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 100_000);
|
||||||
|
const bytes = db.prepare('PRAGMA page_count').get().page_count * db.prepare('PRAGMA page_size').get().page_size;
|
||||||
|
store.ingest([batch(clock, [], true, 'sing-box-next')], 'live');
|
||||||
|
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||||
|
const reusableBytes = db.prepare('PRAGMA freelist_count').get().freelist_count * db.prepare('PRAGMA page_size').get().page_size;
|
||||||
|
db.close();
|
||||||
|
t.diagnostic(JSON.stringify({ closedLifecycles: 100_000, writeMs, queryMs, bytes, reusableBytes }));
|
||||||
|
});
|
||||||
@@ -47,6 +47,9 @@ test('version paths map to the components actually shipped by this repository',
|
|||||||
'gatewayBackend',
|
'gatewayBackend',
|
||||||
]);
|
]);
|
||||||
assert.deepEqual(affectedComponents(['scripts/runtime-impact.mjs']), ['gatewayBackend']);
|
assert.deepEqual(affectedComponents(['scripts/runtime-impact.mjs']), ['gatewayBackend']);
|
||||||
|
for (const file of ['.node-version', 'scripts/check-sqlite-runtime.mjs']) {
|
||||||
|
assert.deepEqual(affectedComponents([file]), ['macClient', 'gatewayClient', 'gatewayBackend']);
|
||||||
|
}
|
||||||
assert.deepEqual(affectedComponents(['README.md', 'test/server/version.test.js']), []);
|
assert.deepEqual(affectedComponents(['README.md', 'test/server/version.test.js']), []);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -153,7 +153,7 @@ test('Mac and Gateway traffic drawers use one feature boundary and the cached re
|
|||||||
|
|
||||||
test('traffic polling runs every second only while the drawer is open and unpaused', () => {
|
test('traffic polling runs every second only while the drawer is open and unpaused', () => {
|
||||||
assert.match(feature, /const POLL_MS = 1_000/);
|
assert.match(feature, /const POLL_MS = 1_000/);
|
||||||
assert.match(feature, /if \(!enabled \|\| !isOpen \|\| paused\) return undefined/);
|
assert.match(feature, /if \(!enabled \|\| !isOpen \|\| paused \|\| view !== 'live'\) return undefined/);
|
||||||
assert.match(feature, /assertLiveTrafficSnapshot\(await loadLiveTraffic\(\)\)/);
|
assert.match(feature, /assertLiveTrafficSnapshot\(await loadLiveTraffic\(\)\)/);
|
||||||
assert.match(feature, /setSnapshot\(next\)[\s\S]*setRequestState\('ready'\)/);
|
assert.match(feature, /setSnapshot\(next\)[\s\S]*setRequestState\('ready'\)/);
|
||||||
assert.match(feature, /catch \{[\s\S]*setRequestState\('error'\)/);
|
assert.match(feature, /catch \{[\s\S]*setRequestState\('error'\)/);
|
||||||
@@ -175,7 +175,7 @@ test('traffic drawer exposes the requested truthful states and accessible contro
|
|||||||
'Только трафик через Harbor Connect. Приложения macOS недоступны внутри Docker.',
|
'Только трафик через Harbor Connect. Приложения macOS недоступны внутри Docker.',
|
||||||
'Только соединения, прошедшие через sing-box Gateway. Трафик, обходящий sing-box напрямую, здесь не виден.',
|
'Только соединения, прошедшие через sing-box Gateway. Трафик, обходящий sing-box напрямую, здесь не виден.',
|
||||||
]) assert.match(feature, new RegExp(copy.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
|
]) assert.match(feature, new RegExp(copy.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
|
||||||
assert.match(feature, /\{feature\.isGateway \? 'GATEWAY' : 'MAC'\} · \{snapshot\?\.summary\.active \|\| 0\} АКТИВНЫХ/);
|
assert.match(feature, /feature\.view === 'live' \? `\$\{snapshot\?\.summary\.active \|\| 0\} АКТИВНЫХ` : 'ИСТОРИЯ'/);
|
||||||
assert.match(feature, /feature\.isGateway[\s\S]*Инспектор трафика выключен в настройках Harbor Gateway\.[\s\S]*Инспектор трафика выключен в настройках Harbor Connect\./);
|
assert.match(feature, /feature\.isGateway[\s\S]*Инспектор трафика выключен в настройках Harbor Gateway\.[\s\S]*Инспектор трафика выключен в настройках Harbor Connect\./);
|
||||||
assert.match(feature, /feature\.isGateway[\s\S]*Только соединения, прошедшие через sing-box Gateway\. Трафик, обходящий sing-box напрямую, здесь не виден\.[\s\S]*Только трафик через Harbor Connect\. Приложения macOS недоступны внутри Docker\./);
|
assert.match(feature, /feature\.isGateway[\s\S]*Только соединения, прошедшие через sing-box Gateway\. Трафик, обходящий sing-box напрямую, здесь не виден\.[\s\S]*Только трафик через Harbor Connect\. Приложения macOS недоступны внутри Docker\./);
|
||||||
assert.match(feature, /aria-label="Найти устройство"/);
|
assert.match(feature, /aria-label="Найти устройство"/);
|
||||||
@@ -187,7 +187,8 @@ test('traffic drawer exposes the requested truthful states and accessible contro
|
|||||||
assert.match(feature, /const \[expandedId, setExpandedId\] = useState\(''\)/);
|
assert.match(feature, /const \[expandedId, setExpandedId\] = useState\(''\)/);
|
||||||
assert.match(feature, /aria-expanded=\{expanded\}[\s\S]*aria-controls=\{detailsId\}/);
|
assert.match(feature, /aria-expanded=\{expanded\}[\s\S]*aria-controls=\{detailsId\}/);
|
||||||
assert.match(feature, /Источник[\s\S]*Назначение[\s\S]*Правило[\s\S]*Цепочка/);
|
assert.match(feature, /Источник[\s\S]*Назначение[\s\S]*Правило[\s\S]*Цепочка/);
|
||||||
assert.doesNotMatch(feature, /closeConnection|reroute|history|sessionStorage/);
|
assert.doesNotMatch(feature, /closeConnection|reroute|sessionStorage/);
|
||||||
|
assert.match(feature, /TrafficHistoryPanel/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('traffic retention and grouping use canonical server settings and the frozen server observation clock', () => {
|
test('traffic retention and grouping use canonical server settings and the frozen server observation clock', () => {
|
||||||
|
|||||||
@@ -40,26 +40,26 @@ const expectedImports = [
|
|||||||
const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex');
|
const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex');
|
||||||
const acceptedLedger = {
|
const acceptedLedger = {
|
||||||
counts: {
|
counts: {
|
||||||
cascadeEdges: 1188,
|
cascadeEdges: 1190,
|
||||||
customProperties: 115,
|
customProperties: 115,
|
||||||
declarations: 4862,
|
declarations: 4872,
|
||||||
important: 0,
|
important: 0,
|
||||||
keyframes: 55,
|
keyframes: 55,
|
||||||
media: 23,
|
media: 23,
|
||||||
rules: 1329,
|
rules: 1332,
|
||||||
variableReferences: 1270,
|
variableReferences: 1274,
|
||||||
},
|
},
|
||||||
hashes: {
|
hashes: {
|
||||||
cascadeEdges: '3d02c10ab8fd95f7eed306048e9b59ec391a0febaa9e3e235badcff3b042eef7',
|
cascadeEdges: '07be381535d1cd8f78990a5b2eae27903471ec2c6cbe1589e36e8f1797f1e610',
|
||||||
customProperties: '81d1e70737ae5a8e4b20d4c1be1b24685975a404863444da4a8821be1f0d5097',
|
customProperties: '81d1e70737ae5a8e4b20d4c1be1b24685975a404863444da4a8821be1f0d5097',
|
||||||
declarations: '0f31e2cdeeab02b9e0a845a6b50ff7e3f035a441217a418e7b5465b5a296ae16',
|
declarations: 'bd186c9317ee7ff540b4eeddfc33d2c4ab64a1b2a9362f95480d180d4f644605',
|
||||||
duplicateKeyframes: '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945',
|
duplicateKeyframes: '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945',
|
||||||
duplicateSelectors: '8982145dba05b33bf1c93b2d54cfbe76305b276ff3c657aa020144016ada5848',
|
duplicateSelectors: '8982145dba05b33bf1c93b2d54cfbe76305b276ff3c657aa020144016ada5848',
|
||||||
keyframes: 'b9b0bf3fad92e69b93ec0c24f01da15e78bbe89a095597d64ce4f7ef5e272fdd',
|
keyframes: 'b9b0bf3fad92e69b93ec0c24f01da15e78bbe89a095597d64ce4f7ef5e272fdd',
|
||||||
ruleDeclarationSequences: '33a0dff2428a8e712776d40d796ea05d169544eb1af02ad350759061fb2a3c3c',
|
ruleDeclarationSequences: 'b066b7e20fd1195c67c23c38cbaf7b58edcd6356e62bd173371234206c74817e',
|
||||||
selectors: '5734c69a504c59025e2e6b1637cfb6d1905ca1ae358783d0916a694a973eea22',
|
selectors: '2e38ff14b0a7d581b090b520c92b6bff60c84082db4dc20a49c454ef468fbeb5',
|
||||||
variableReferences: '49ad724b2812128f5344fdd55b177aa96ea9b30eb4d878dc86f2e4f3fa898182',
|
variableReferences: '5fd2c93d2467be16976c102b595a5fb15c5692e2d98645022d7a9acf2f42d829',
|
||||||
witnesses: '4e357ee4f7310d60399baa3099bd8f4ed320debc1e3c9a09f55b892a1178049c',
|
witnesses: 'e9ae2a8416aa44a75a01452bc17884dcb133cccb939880cdecc761ea1043721c',
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -212,7 +212,7 @@ test('client typography uses the shared semantic scale outside the token owner',
|
|||||||
|
|
||||||
test('accepted stylesheet has pinned declaration, selector, keyframe, variable, and cascade ledgers', () => {
|
test('accepted stylesheet has pinned declaration, selector, keyframe, variable, and cascade ledgers', () => {
|
||||||
const witnesses = readStyleWitnesses(root);
|
const witnesses = readStyleWitnesses(root);
|
||||||
assert.equal(witnesses.length, 1375);
|
assert.equal(witnesses.length, 1486);
|
||||||
assert.equal(witnesses.filter((witness) => witness.unknown || witness.ancestorUnknown).length, 0);
|
assert.equal(witnesses.filter((witness) => witness.unknown || witness.ancestorUnknown).length, 0);
|
||||||
const ledger = createStyleLedger(readStyleSource(root), { witnesses });
|
const ledger = createStyleLedger(readStyleSource(root), { witnesses });
|
||||||
assert.deepEqual(ledger.counts, acceptedLedger.counts);
|
assert.deepEqual(ledger.counts, acceptedLedger.counts);
|
||||||
@@ -315,6 +315,12 @@ test('JSX witnesses keep real multi-class collisions and exclude impossible elem
|
|||||||
assert.equal(conservativeSibling.counts.cascadeEdges, 1);
|
assert.equal(conservativeSibling.counts.cascadeEdges, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('JSX witness recursion requires an explicit finite bound for the four-level history tree', () => {
|
||||||
|
const sources = [{ file: '/fixture/App.tsx', source: 'export function App() { return <Branch />; } function Branch() { return <div className="branch"><Branch /></div>; }' }];
|
||||||
|
assert.throws(() => createStyleWitnesses(sources), /Recursive JSX witness/);
|
||||||
|
assert.equal(createStyleWitnesses(sources, { recursionLimits: { Branch: 4 } }).length, 4);
|
||||||
|
});
|
||||||
|
|
||||||
test('selector proof uses the observed level-four grammar and exact specificity', () => {
|
test('selector proof uses the observed level-four grammar and exact specificity', () => {
|
||||||
const fixtures = [
|
const fixtures = [
|
||||||
['#root', [{ a: 1, b: 0, c: 0 }]],
|
['#root', [{ a: 1, b: 0, c: 0 }]],
|
||||||
@@ -405,8 +411,8 @@ test('main owns one public stylesheet and the regrouped production CSS is determ
|
|||||||
assert.equal((main.match(/import ['"][^'"]+\.css['"]/g) || []).length, 1);
|
assert.equal((main.match(/import ['"][^'"]+\.css['"]/g) || []).length, 1);
|
||||||
|
|
||||||
const assets = fs.readdirSync(path.join(root, 'dist/assets')).filter((file) => file.endsWith('.css'));
|
const assets = fs.readdirSync(path.join(root, 'dist/assets')).filter((file) => file.endsWith('.css'));
|
||||||
assert.deepEqual(assets, ['index-fllV-PfI.css']);
|
assert.deepEqual(assets, ['index-D6ACNk74.css']);
|
||||||
const built = fs.readFileSync(path.join(root, 'dist/assets', assets[0]));
|
const built = fs.readFileSync(path.join(root, 'dist/assets', assets[0]));
|
||||||
assert.equal(built.byteLength, 184958);
|
assert.equal(built.byteLength, 185399);
|
||||||
assert.equal(sha256(built), '3a9ed6fdf6c5d81234de9aa0abd995bc9db1cfe298194162654edcbfe3ae7c62');
|
assert.equal(sha256(built), '3327b4873e7dba63fd44c21d34c4fe19f78167ef6c5badcfd8dfe08eb2705c5d');
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -349,7 +349,7 @@ function bindCallParameters(callable, argumentsList, callerEnvironment, callerFi
|
|||||||
return environment;
|
return environment;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createStyleWitnesses(sources, { entry = 'App' } = {}) {
|
export function createStyleWitnesses(sources, { entry = 'App', recursionLimits = {} } = {}) {
|
||||||
const files = sources.map(({ file, source }) => ({ file, source, parsed: parseWitnessFile(file, source) }));
|
const files = sources.map(({ file, source }) => ({ file, source, parsed: parseWitnessFile(file, source) }));
|
||||||
const { definitions, definitionNodes } = componentDefinitions(files);
|
const { definitions, definitionNodes } = componentDefinitions(files);
|
||||||
const entryDefinition = definitions.get(entry);
|
const entryDefinition = definitions.get(entry);
|
||||||
@@ -361,7 +361,11 @@ export function createStyleWitnesses(sources, { entry = 'App' } = {}) {
|
|||||||
const dynamicByFile = dynamicClassBindings(files);
|
const dynamicByFile = dynamicClassBindings(files);
|
||||||
const witnesses = [];
|
const witnesses = [];
|
||||||
const expand = (definition, ancestors, ancestorUnknown, environment, stack) => {
|
const expand = (definition, ancestors, ancestorUnknown, environment, stack) => {
|
||||||
if (stack.includes(definition)) throw new TypeError(`Recursive JSX witness component: ${definition.node.id?.name || definition.file}`);
|
if (stack.includes(definition)) {
|
||||||
|
const limit = recursionLimits[definition.node.id?.name];
|
||||||
|
if (!Number.isInteger(limit) || limit < 1) throw new TypeError(`Recursive JSX witness component: ${definition.node.id?.name || definition.file}`);
|
||||||
|
if (stack.filter((item) => item === definition).length >= limit) return;
|
||||||
|
}
|
||||||
const nextStack = [...stack, definition];
|
const nextStack = [...stack, definition];
|
||||||
const entryEnvironment = withLocalJsxBindings(definition.node.body, environment, definitionNodes, definition.file);
|
const entryEnvironment = withLocalJsxBindings(definition.node.body, environment, definitionNodes, definition.file);
|
||||||
const visit = (
|
const visit = (
|
||||||
@@ -474,7 +478,8 @@ export function readStyleWitnesses(root) {
|
|||||||
file,
|
file,
|
||||||
source: fs.readFileSync(file, 'utf8'),
|
source: fs.readFileSync(file, 'utf8'),
|
||||||
}));
|
}));
|
||||||
return createStyleWitnesses(files);
|
// History has exactly service/domain/hostname/IP levels, not arbitrary JSX recursion.
|
||||||
|
return createStyleWitnesses(files, { recursionLimits: { HistoryRows: 4, HistoryBranch: 3 } });
|
||||||
}
|
}
|
||||||
|
|
||||||
const OBSERVED_PROPERTIES = new Set(`
|
const OBSERVED_PROPERTIES = new Set(`
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
"moduleResolution": "NodeNext",
|
"moduleResolution": "NodeNext",
|
||||||
"outDir": "dist",
|
"outDir": "dist",
|
||||||
"rootDir": "src",
|
"rootDir": "src",
|
||||||
"types": ["node18"]
|
"types": ["node"]
|
||||||
},
|
},
|
||||||
"include": ["src/server/**/*", "src/shared/**/*"],
|
"include": ["src/server/**/*", "src/shared/**/*"],
|
||||||
"exclude": ["dist", "node_modules", "src/web"]
|
"exclude": ["dist", "node_modules", "src/web"]
|
||||||
|
|||||||
+1
-1
@@ -7,7 +7,7 @@
|
|||||||
"moduleResolution": "NodeNext",
|
"moduleResolution": "NodeNext",
|
||||||
"outDir": ".test-dist",
|
"outDir": ".test-dist",
|
||||||
"rootDir": ".",
|
"rootDir": ".",
|
||||||
"types": ["node18", "vite/client"]
|
"types": ["node", "vite/client"]
|
||||||
},
|
},
|
||||||
"include": [
|
"include": [
|
||||||
"src/shared/**/*",
|
"src/shared/**/*",
|
||||||
|
|||||||
Reference in New Issue
Block a user