Update Harbor client and gateway integration workflows
This commit is contained in:
@@ -15,6 +15,8 @@ export const settings = {
|
||||
port: parsePort(process.env.PORT, 3456),
|
||||
proxyPort,
|
||||
diagnosticsProxyPort: parsePort(process.env.DIAGNOSTICS_PROXY_PORT, 18080),
|
||||
failoverPrimaryProxyPort: parsePort(process.env.FAILOVER_PRIMARY_PROXY_PORT, 18081),
|
||||
failoverReserveProxyPort: parsePort(process.env.FAILOVER_RESERVE_PROXY_PORT, 18082),
|
||||
singboxApiPort: parsePort(process.env.SING_BOX_API_PORT, 19090),
|
||||
tproxyPort: parsePort(process.env.TPROXY_PORT, 7895),
|
||||
tproxyMark: process.env.TPROXY_MARK || "1",
|
||||
@@ -40,6 +42,7 @@ export const settings = {
|
||||
cachePath: process.env.SING_BOX_CACHE || "/var/lib/sing-box/cache.db",
|
||||
statePath: path.join(dataDir, "state.json"),
|
||||
deviceStatePath: path.join(dataDir, "devices.json"),
|
||||
activityJournalPath: path.join(dataDir, "activity-journal.json"),
|
||||
subscriptionCachePath: path.join(dataDir, "subscription-cache.json"),
|
||||
sharedProxyHost: process.env.SHARED_PROXY_HOST || "",
|
||||
hostNetworkStatePath:
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
createDomainTrafficService,
|
||||
readSingboxConnections,
|
||||
} from './services/domainTrafficService.js';
|
||||
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
||||
|
||||
const socketPath = settings.dataplaneSocket;
|
||||
const runtime = createSingboxRuntime({
|
||||
@@ -40,6 +41,11 @@ const devicePolicy = createDevicePolicyService({
|
||||
const connectivityDiagnostics = createConnectivityDiagnosticsService({
|
||||
proxyPort: settings.diagnosticsProxyPort,
|
||||
});
|
||||
const failoverDiagnostics = {
|
||||
primary: createConnectivityDiagnosticsService({ proxyPort: settings.failoverPrimaryProxyPort }),
|
||||
reserve: createConnectivityDiagnosticsService({ proxyPort: settings.failoverReserveProxyPort }),
|
||||
};
|
||||
const selector = createSingboxSelectorService({ port: settings.singboxApiPort });
|
||||
const domainTraffic = createDomainTrafficService({
|
||||
observe: () => readSingboxConnections(settings.singboxApiPort),
|
||||
devices: () => traffic.snapshot().devices,
|
||||
@@ -126,6 +132,33 @@ const server = http.createServer(async (req: IncomingMessage, res: ServerRespons
|
||||
target,
|
||||
}));
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/failover/probe') {
|
||||
const { role, services = [], target = null, timeoutMs = 6_000 } = record(await readJson(req));
|
||||
if (role !== 'primary' && role !== 'reserve') throw new Error('Неизвестная failover role');
|
||||
return sendJson(res, 200, await failoverDiagnostics[role].runVpn({ services, target, timeoutMs: Number(timeoutMs) }));
|
||||
}
|
||||
if (req.method === 'GET' && req.url === '/failover/selector') {
|
||||
return sendJson(res, 200, await selector.read());
|
||||
}
|
||||
if (req.method === 'PUT' && req.url === '/failover/selector') {
|
||||
const { role } = record(await readJson(req));
|
||||
if (role !== 'primary' && role !== 'reserve') throw new Error('Неизвестная failover role');
|
||||
return sendJson(res, 200, await selector.select(role));
|
||||
}
|
||||
if (req.method === 'PUT' && req.url === '/failover/activity') {
|
||||
const { enabled } = record(await readJson(req));
|
||||
if (enabled === true) domainTraffic.enableActivity();
|
||||
else domainTraffic.disableActivity();
|
||||
return sendJson(res, 200, { enabled: enabled === true });
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/failover/activity/read') {
|
||||
const { thresholdBytesPerSecond = 0 } = record(await readJson(req));
|
||||
return sendJson(res, 200, { activity: domainTraffic.activitySnapshot(thresholdBytesPerSecond) });
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/config/check') {
|
||||
const { config } = record(await readJson(req));
|
||||
return sendJson(res, 200, runtime.checkConfig(config));
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/apply') {
|
||||
return sendJson(res, 200, await runtime.apply());
|
||||
}
|
||||
|
||||
@@ -82,6 +82,16 @@ export function createDataplaneClient(socketPath: string, send: SendDataplaneReq
|
||||
throw new HarborError('DIAGNOSTICS_FAILED', { cause });
|
||||
}
|
||||
},
|
||||
checkConfig: (config: unknown) => send(socketPath, '/config/check', 'POST', { config }, 15_000),
|
||||
runFailoverProbe: (role: 'primary' | 'reserve', services: unknown, target: unknown, timeoutMs: number) => (
|
||||
send(socketPath, '/failover/probe', 'POST', { role, services, target, timeoutMs }, timeoutMs + 10_000)
|
||||
),
|
||||
readFailoverSelector: () => send(socketPath, '/failover/selector', 'GET'),
|
||||
selectFailoverRole: (role: 'primary' | 'reserve') => send(socketPath, '/failover/selector', 'PUT', { role }),
|
||||
setFailoverActivityEnabled: (enabled: boolean) => send(socketPath, '/failover/activity', 'PUT', { enabled }),
|
||||
readFailoverActivity: (thresholdBytesPerSecond: number) => (
|
||||
send(socketPath, '/failover/activity/read', 'POST', { thresholdBytesPerSecond })
|
||||
),
|
||||
apply: () => update('/apply', 'POST'),
|
||||
restart: () => update('/restart', 'POST'),
|
||||
stop: () => update('/stop', 'POST'),
|
||||
|
||||
@@ -4,7 +4,9 @@ import {
|
||||
type StoredState,
|
||||
} from '../../../shared/contracts/state.js';
|
||||
import { HarborError } from '../../../shared/errors.js';
|
||||
import { finishRollback } from '../../services/rollback.js';
|
||||
import { finishRollback, type RollbackStep } from '../../services/rollback.js';
|
||||
import type { AppliedFailoverPolicy } from '../../../shared/failover.js';
|
||||
import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js';
|
||||
|
||||
interface ConnectionServiceDependencies {
|
||||
state: {
|
||||
@@ -26,6 +28,18 @@ interface ConnectionServiceDependencies {
|
||||
restartCommand(): Promise<RuntimeCommandResult>;
|
||||
};
|
||||
route?: { isGatewayDirect(): boolean };
|
||||
failover?: {
|
||||
build(state: StoredState, source?: 'desired' | 'applied'): {
|
||||
config: unknown;
|
||||
applied: AppliedFailoverPolicy;
|
||||
primaryProfile: StoredProfile;
|
||||
primaryServer: StoredProfile['servers'][number];
|
||||
};
|
||||
prepareActivation(role: 'primary' | 'reserve'): Promise<unknown>;
|
||||
restoreAppliedActivation(state: StoredState): Promise<unknown>;
|
||||
reconcile(): Promise<unknown>;
|
||||
};
|
||||
onEvent?: (event: ActivityJournalEventInput) => void;
|
||||
serialize<T>(operation: () => Promise<T>): Promise<T>;
|
||||
now(): Date;
|
||||
}
|
||||
@@ -72,6 +86,33 @@ function withDesiredServer(state: StoredState, profile: StoredProfile, serverId:
|
||||
}
|
||||
|
||||
export function createConnectionService(dependencies: ConnectionServiceDependencies) {
|
||||
const activationTarget = (
|
||||
state: StoredState,
|
||||
applied: AppliedFailoverPolicy,
|
||||
role: 'primary' | 'reserve',
|
||||
) => {
|
||||
const target = applied[role];
|
||||
const profile = profileById(state, target.profileId);
|
||||
const server = profile?.servers.find(({ id }) => id === target.serverId);
|
||||
if (!profile || !server) throw new HarborError('SERVER_NOT_FOUND');
|
||||
return { profile, server };
|
||||
};
|
||||
const finishConnectionRollback = async (error: unknown, steps: RollbackStep[], message: string) => {
|
||||
try {
|
||||
await finishRollback(error, steps, message);
|
||||
} catch (cause) {
|
||||
const code = cause && typeof cause === 'object' && 'code' in cause
|
||||
&& /^[A-Z0-9_]{1,50}$/.test(String(cause.code)) ? String(cause.code) : 'UNKNOWN';
|
||||
dependencies.onEvent?.({
|
||||
type: 'connection.failed',
|
||||
severity: 'error',
|
||||
source: 'connection',
|
||||
dedupeKey: `connection.failed:${dependencies.state.read().revision}:${code}`,
|
||||
data: { errorCode: code },
|
||||
});
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
const applyWithinQueue = async (
|
||||
previousState: StoredState,
|
||||
profile: StoredProfile,
|
||||
@@ -88,18 +129,26 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
if (!selectedServer) throw new HarborError('SERVER_NOT_FOUND');
|
||||
if (!profile.subscriptionConfig) throw new HarborError('CONFIG_INVALID');
|
||||
|
||||
const wasRunning = await dependencies.runtime.isRunning();
|
||||
if (wasRunning && previousState.failoverPolicy?.enabled) {
|
||||
dependencies.state.update((state) => withDesiredServer(state, profile, selectedServer.id));
|
||||
return { profileId: profile.id, serverId: selectedServer.id, selectedTag: selectedServer.label };
|
||||
}
|
||||
|
||||
if (dependencies.route?.isGatewayDirect()) {
|
||||
dependencies.state.update((state) => withDesiredServer(state, profile, selectedServer.id));
|
||||
return { profileId: profile.id, serverId: selectedServer.id, selectedTag: selectedServer.label };
|
||||
}
|
||||
|
||||
const nextConfig = dependencies.config.build(
|
||||
const failoverCandidate = previousState.failoverPolicy?.enabled
|
||||
? dependencies.failover?.build(previousState, wasRunning ? 'applied' : 'desired')
|
||||
: null;
|
||||
const nextConfig = failoverCandidate?.config || dependencies.config.build(
|
||||
profile.subscriptionConfig,
|
||||
selectedServer.id,
|
||||
previousState.routeRules,
|
||||
);
|
||||
const previousConfig = dependencies.config.read();
|
||||
const wasRunning = await dependencies.runtime.isRunning();
|
||||
let configMutationStarted = false;
|
||||
let runtimeMutationStarted = false;
|
||||
let stateCommitStarted = false;
|
||||
@@ -109,18 +158,20 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
dependencies.config.write(nextConfig);
|
||||
runtimeMutationStarted = true;
|
||||
await dependencies.runtime.start();
|
||||
if (failoverCandidate) await dependencies.failover!.prepareActivation('primary');
|
||||
stateCommitStarted = true;
|
||||
dependencies.state.update((state) => ({
|
||||
...withDesiredServer(state, profile, selectedServer.id),
|
||||
connectionDesired: 'running',
|
||||
appliedProfileId: profile.id,
|
||||
appliedServerId: selectedServer.id,
|
||||
appliedServerSnapshot: selectedServer,
|
||||
appliedProfileId: failoverCandidate?.primaryProfile.id || profile.id,
|
||||
appliedServerId: failoverCandidate?.primaryServer.id || selectedServer.id,
|
||||
appliedServerSnapshot: failoverCandidate?.primaryServer || selectedServer,
|
||||
appliedFailoverPolicy: failoverCandidate?.applied || null,
|
||||
appliedAt: dependencies.now().toISOString(),
|
||||
appliedRouteRules: state.routeRules,
|
||||
}));
|
||||
} catch (error) {
|
||||
await finishRollback(error, [
|
||||
await finishConnectionRollback(error, [
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
...(configMutationStarted ? [{
|
||||
run: () => previousConfig === null
|
||||
@@ -128,12 +179,28 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
: dependencies.config.restore(previousConfig),
|
||||
}] : []),
|
||||
...(runtimeMutationStarted ? [{
|
||||
run: () => wasRunning ? dependencies.runtime.start() : dependencies.runtime.stop(),
|
||||
run: async () => {
|
||||
if (!wasRunning) return dependencies.runtime.stop();
|
||||
await dependencies.runtime.start();
|
||||
await dependencies.failover?.restoreAppliedActivation(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
], 'Connection rollback failed');
|
||||
}
|
||||
|
||||
await dependencies.failover?.reconcile();
|
||||
dependencies.onEvent?.({
|
||||
type: 'connection.started',
|
||||
severity: 'info',
|
||||
source: 'connection',
|
||||
dedupeKey: `connection.started:${dependencies.state.read().revision}`,
|
||||
data: {
|
||||
profileLabel: failoverCandidate?.primaryProfile.label || profile.label,
|
||||
serverLabel: failoverCandidate?.primaryServer.label || selectedServer.label,
|
||||
},
|
||||
});
|
||||
|
||||
return { profileId: profile.id, serverId: selectedServer.id, selectedTag: selectedServer.label };
|
||||
};
|
||||
|
||||
@@ -186,16 +253,29 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
appliedProfileId: '',
|
||||
appliedServerId: '',
|
||||
appliedServerSnapshot: null,
|
||||
appliedFailoverPolicy: null,
|
||||
}));
|
||||
} catch (error) {
|
||||
await finishRollback(error, [
|
||||
await finishConnectionRollback(error, [
|
||||
...(runtimeMutationStarted && wasRunning !== null ? [{
|
||||
run: () => wasRunning ? dependencies.runtime.start() : dependencies.runtime.stop(),
|
||||
run: async () => {
|
||||
if (!wasRunning) return dependencies.runtime.stop();
|
||||
await dependencies.runtime.start();
|
||||
await dependencies.failover?.restoreAppliedActivation(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
], 'Connection rollback failed');
|
||||
}
|
||||
await dependencies.failover?.reconcile();
|
||||
dependencies.onEvent?.({
|
||||
type: 'connection.stopped',
|
||||
severity: 'info',
|
||||
source: 'connection',
|
||||
dedupeKey: `connection.stopped:${dependencies.state.read().revision}`,
|
||||
data: {},
|
||||
});
|
||||
});
|
||||
|
||||
const restart = () => dependencies.serialize(async () => {
|
||||
@@ -213,7 +293,18 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
? previousState.appliedServerSnapshot
|
||||
: null);
|
||||
if (!server || !profile.subscriptionConfig) throw new HarborError('CONFIG_INVALID');
|
||||
const candidateConfig = dependencies.config.build(
|
||||
const failoverCandidate = previousState.failoverPolicy?.enabled
|
||||
? dependencies.failover?.build(previousState, wasRunning ? 'applied' : 'desired')
|
||||
: null;
|
||||
const activationRole = failoverCandidate && wasRunning
|
||||
&& previousState.appliedProfileId === failoverCandidate.applied.reserve.profileId
|
||||
&& previousState.appliedServerId === failoverCandidate.applied.reserve.serverId
|
||||
? 'reserve' as const
|
||||
: 'primary' as const;
|
||||
const failoverTarget = failoverCandidate
|
||||
? activationTarget(previousState, failoverCandidate.applied, activationRole)
|
||||
: null;
|
||||
const candidateConfig = failoverCandidate?.config || dependencies.config.build(
|
||||
profile.subscriptionConfig,
|
||||
server.id,
|
||||
previousState.routeRules,
|
||||
@@ -229,30 +320,47 @@ export function createConnectionService(dependencies: ConnectionServiceDependenc
|
||||
const command = await dependencies.runtime.restartCommand();
|
||||
runtimeMutationStarted = command.mutationStarted;
|
||||
if (!command.ok) throw command.error;
|
||||
if (failoverCandidate) await dependencies.failover!.prepareActivation(activationRole);
|
||||
stateCommitStarted = true;
|
||||
dependencies.state.update((state) => ({
|
||||
...state,
|
||||
desiredProfileId: wasRunning ? state.desiredProfileId : profile.id,
|
||||
appliedProfileId: profile.id,
|
||||
appliedServerId: server.id,
|
||||
appliedServerSnapshot: server,
|
||||
appliedProfileId: failoverTarget?.profile.id || profile.id,
|
||||
appliedServerId: failoverTarget?.server.id || server.id,
|
||||
appliedServerSnapshot: failoverTarget?.server || server,
|
||||
appliedFailoverPolicy: failoverCandidate?.applied || null,
|
||||
connectionDesired: 'running',
|
||||
appliedRouteRules: dependencies.route?.isGatewayDirect() ? [] : state.routeRules,
|
||||
}));
|
||||
} catch (error) {
|
||||
await finishRollback(error, [
|
||||
await finishConnectionRollback(error, [
|
||||
...(configMutationStarted ? [{
|
||||
run: () => previousConfig === null
|
||||
? dependencies.config.remove()
|
||||
: dependencies.config.restore(previousConfig),
|
||||
}] : []),
|
||||
...(runtimeMutationStarted ? [{
|
||||
run: () => wasRunning ? dependencies.runtime.start() : dependencies.runtime.stop(),
|
||||
run: async () => {
|
||||
if (!wasRunning) return dependencies.runtime.stop();
|
||||
await dependencies.runtime.start();
|
||||
await dependencies.failover?.restoreAppliedActivation(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
], 'Connection rollback failed');
|
||||
}
|
||||
await dependencies.failover?.reconcile();
|
||||
dependencies.onEvent?.({
|
||||
type: 'connection.started',
|
||||
severity: 'info',
|
||||
source: 'connection',
|
||||
dedupeKey: `connection.started:${dependencies.state.read().revision}`,
|
||||
data: {
|
||||
profileLabel: failoverTarget?.profile.label || profile.label,
|
||||
serverLabel: failoverTarget?.server.label || server.label,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
return { apply, activate, stop, restart };
|
||||
|
||||
@@ -0,0 +1,671 @@
|
||||
import crypto from 'node:crypto';
|
||||
import {
|
||||
createIdleFailoverSnapshot,
|
||||
isFailoverConfigured,
|
||||
nextFailoverDecision,
|
||||
normalizeFailoverPolicy,
|
||||
type AppliedFailoverPolicy,
|
||||
type FailoverDecisionMemory,
|
||||
type FailoverHealth,
|
||||
type FailoverPolicy,
|
||||
type FailoverRole,
|
||||
type FailoverSnapshot,
|
||||
} from '../../../shared/failover.js';
|
||||
import type { HarborServer, StoredState } from '../../../shared/contracts/state.js';
|
||||
import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js';
|
||||
import { HarborError } from '../../../shared/errors.js';
|
||||
|
||||
interface Candidate {
|
||||
config: unknown;
|
||||
applied: AppliedFailoverPolicy;
|
||||
}
|
||||
|
||||
interface FailoverServiceDependencies {
|
||||
state: {
|
||||
read(): StoredState;
|
||||
update(mutator: (state: StoredState) => Record<string, unknown>): StoredState;
|
||||
};
|
||||
runtime: { isRunning(): Promise<boolean> };
|
||||
dataplane: {
|
||||
checkConfig(config: unknown): Promise<unknown>;
|
||||
runFailoverProbe(role: FailoverRole, services: unknown, target: string, timeoutMs: number): Promise<unknown>;
|
||||
readFailoverSelector(): Promise<unknown>;
|
||||
selectFailoverRole(role: FailoverRole): Promise<unknown>;
|
||||
setFailoverActivityEnabled(enabled: boolean): Promise<unknown>;
|
||||
readFailoverActivity(thresholdBytesPerSecond: number): Promise<unknown>;
|
||||
};
|
||||
buildCandidate(state: StoredState): Candidate;
|
||||
serialize<T>(operation: () => Promise<T>): Promise<T>;
|
||||
scheduler?: {
|
||||
setTimeout(callback: () => void, intervalMs: number): NodeJS.Timeout;
|
||||
clearTimeout(timer: NodeJS.Timeout): void;
|
||||
};
|
||||
now?: () => Date;
|
||||
onWarning?: (error: unknown) => void;
|
||||
onSwitch?: (from: FailoverRole, to: FailoverRole, reason: string) => void;
|
||||
onEvent?: (event: ActivityJournalEventInput) => void;
|
||||
}
|
||||
|
||||
const record = (value: unknown): Record<string, unknown> => (
|
||||
value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
||||
);
|
||||
|
||||
function targetServer(state: StoredState, role: FailoverRole): HarborServer | null {
|
||||
const target = state.appliedFailoverPolicy?.[role] || state.failoverPolicy[role];
|
||||
return state.profiles.find(({ id }) => id === target.profileId)
|
||||
?.servers.find(({ id }) => id === target.serverId) || null;
|
||||
}
|
||||
|
||||
function currentRole(state: StoredState): FailoverRole | null {
|
||||
const applied = state.appliedFailoverPolicy;
|
||||
if (!applied) return null;
|
||||
for (const role of ['primary', 'reserve'] as const) {
|
||||
if (
|
||||
state.appliedProfileId === applied[role].profileId
|
||||
&& state.appliedServerId === applied[role].serverId
|
||||
) return role;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function probeHealth(value: unknown): boolean {
|
||||
const vpn = record(record(value).vpn);
|
||||
const sites = Array.isArray(vpn.sites) ? vpn.sites.map(record) : [];
|
||||
return sites.length === 1 && sites[0].status === 'available';
|
||||
}
|
||||
|
||||
function safeErrorCode(error: unknown) {
|
||||
const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
|
||||
return /^[A-Z0-9_]{1,50}$/.test(code) ? code : 'UNKNOWN';
|
||||
}
|
||||
|
||||
export function createFailoverService(dependencies: FailoverServiceDependencies) {
|
||||
const scheduler = dependencies.scheduler || {
|
||||
setTimeout: (callback: () => void, intervalMs: number) => setTimeout(callback, intervalMs),
|
||||
clearTimeout: (timer: NodeJS.Timeout) => clearTimeout(timer),
|
||||
};
|
||||
const now = dependencies.now || (() => new Date());
|
||||
const epoch = crypto.randomUUID();
|
||||
let sequence = 0;
|
||||
let generation = 0;
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
let collectorEnabled: boolean | null = null;
|
||||
let roundPromise: Promise<void> | null = null;
|
||||
let decisionMemory: FailoverDecisionMemory | undefined;
|
||||
let snapshot = createIdleFailoverSnapshot(dependencies.state.read().failoverPolicy, epoch, sequence);
|
||||
|
||||
function appliedMatchesDesired(state: StoredState) {
|
||||
if (!state.appliedFailoverPolicy) return false;
|
||||
try {
|
||||
return JSON.stringify(dependencies.buildCandidate(state).applied)
|
||||
=== JSON.stringify(state.appliedFailoverPolicy);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function publish(next: FailoverSnapshot) {
|
||||
sequence += 1;
|
||||
snapshot = { ...next, observationEpoch: epoch, observationSequence: sequence };
|
||||
}
|
||||
|
||||
function clearTimer() {
|
||||
if (timer) scheduler.clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
|
||||
function schedule(delay: number) {
|
||||
clearTimer();
|
||||
timer = scheduler.setTimeout(() => {
|
||||
timer = null;
|
||||
void runRound().catch(dependencies.onWarning);
|
||||
}, delay);
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
async function disableCollector() {
|
||||
if (collectorEnabled === false) return;
|
||||
await dependencies.dataplane.setFailoverActivityEnabled(false);
|
||||
collectorEnabled = false;
|
||||
}
|
||||
|
||||
async function deactivate(policy: FailoverPolicy, passiveRole: FailoverRole | null = null) {
|
||||
generation += 1;
|
||||
clearTimer();
|
||||
decisionMemory = undefined;
|
||||
await disableCollector();
|
||||
const idle = createIdleFailoverSnapshot(policy, epoch, sequence);
|
||||
if (passiveRole) {
|
||||
idle.activation = 'passive-loaded';
|
||||
idle.currentRole = passiveRole;
|
||||
idle.reason = 'disabled';
|
||||
}
|
||||
publish(idle);
|
||||
}
|
||||
|
||||
function activeSnapshot(state: StoredState, status: FailoverSnapshot['status'] = 'observing'): FailoverSnapshot {
|
||||
const role = state.failoverRuntimeState.reasonCode === 'selector-unknown' ? null : currentRole(state);
|
||||
const channel = (target: typeof state.failoverPolicy.primary) => ({
|
||||
target,
|
||||
health: 'unknown' as FailoverHealth,
|
||||
failingServiceIds: [],
|
||||
checkedAt: null,
|
||||
stateSince: null,
|
||||
});
|
||||
return {
|
||||
observationEpoch: epoch,
|
||||
observationSequence: sequence,
|
||||
configured: isFailoverConfigured(state.failoverPolicy),
|
||||
enabled: state.failoverPolicy.enabled,
|
||||
paused: state.failoverPolicy.paused,
|
||||
activation: appliedMatchesDesired(state) ? 'active' : 'pending',
|
||||
currentRole: role || 'other',
|
||||
status,
|
||||
primary: channel(state.appliedFailoverPolicy?.primary || state.failoverPolicy.primary),
|
||||
reserve: channel(state.appliedFailoverPolicy?.reserve || state.failoverPolicy.reserve),
|
||||
nextDecisionAt: null,
|
||||
reason: null,
|
||||
trafficActivity: null,
|
||||
policy: state.failoverPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
async function reconcile() {
|
||||
let state = dependencies.state.read();
|
||||
const policy = state.failoverPolicy;
|
||||
if (!policy.enabled) {
|
||||
const role = currentRole(state);
|
||||
const passiveRole = role && await dependencies.runtime.isRunning() ? role : null;
|
||||
return deactivate(policy, passiveRole);
|
||||
}
|
||||
const running = await dependencies.runtime.isRunning();
|
||||
if (!running || !state.appliedFailoverPolicy || !currentRole(state)) {
|
||||
generation += 1;
|
||||
clearTimer();
|
||||
await disableCollector();
|
||||
const pending = activeSnapshot(state, 'idle');
|
||||
pending.activation = running ? 'pending' : 'inactive';
|
||||
pending.reason = running ? 'pending-activation' : 'vpn-stopped';
|
||||
publish(pending);
|
||||
return;
|
||||
}
|
||||
const role = currentRole(state)!;
|
||||
const selected = record(await dependencies.dataplane.readFailoverSelector());
|
||||
if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
|
||||
if (state.failoverRuntimeState.reasonCode === 'selector-unknown') {
|
||||
state = dependencies.state.update((current) => ({
|
||||
...current,
|
||||
failoverRuntimeState: { ...current.failoverRuntimeState, reasonCode: null },
|
||||
}));
|
||||
}
|
||||
if (collectorEnabled !== true) {
|
||||
await dependencies.dataplane.setFailoverActivityEnabled(true);
|
||||
collectorEnabled = true;
|
||||
}
|
||||
generation += 1;
|
||||
const active = activeSnapshot(state);
|
||||
if (active.activation === 'pending') active.reason = 'pending-activation';
|
||||
publish(active);
|
||||
schedule(0);
|
||||
}
|
||||
|
||||
async function reconcileAfterCommit() {
|
||||
try {
|
||||
await reconcile();
|
||||
} catch (error) {
|
||||
dependencies.onWarning?.(error);
|
||||
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
||||
failed.reason = 'reconcile-failed';
|
||||
publish(failed);
|
||||
}
|
||||
}
|
||||
|
||||
async function assessRole(role: FailoverRole, policy: FailoverPolicy) {
|
||||
const custom = dependencies.state.read().diagnostics.customServices;
|
||||
const results = await Promise.all(policy.checks.map(async (check) => {
|
||||
try {
|
||||
return {
|
||||
id: check.serviceId,
|
||||
ok: probeHealth(await dependencies.dataplane.runFailoverProbe(
|
||||
role,
|
||||
custom,
|
||||
`site:${check.serviceId}`,
|
||||
check.timeoutMs,
|
||||
)),
|
||||
};
|
||||
} catch {
|
||||
return { id: check.serviceId, ok: null };
|
||||
}
|
||||
}));
|
||||
return {
|
||||
health: results.some(({ ok }) => ok === null)
|
||||
? 'unknown' as const
|
||||
: results.every(({ ok }) => ok) ? 'healthy' as const : 'unhealthy' as const,
|
||||
failingServiceIds: results.filter(({ ok }) => ok === false).map(({ id }) => id),
|
||||
};
|
||||
}
|
||||
|
||||
async function switchWithinQueue(role: FailoverRole, reason: string) {
|
||||
const before = dependencies.state.read();
|
||||
const from = currentRole(before);
|
||||
if (!from || from === role) return;
|
||||
try {
|
||||
await dependencies.dataplane.selectFailoverRole(role);
|
||||
const server = targetServer(before, role);
|
||||
if (!server) throw new HarborError('SERVER_NOT_FOUND');
|
||||
const target = before.appliedFailoverPolicy![role];
|
||||
const switchedAt = now().toISOString();
|
||||
const cutoff = now().getTime() - before.failoverPolicy.flapProtection.windowMs;
|
||||
const history = role === 'reserve'
|
||||
? [...before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff), switchedAt]
|
||||
: before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff);
|
||||
const quarantine = history.length >= before.failoverPolicy.flapProtection.count
|
||||
? new Date(now().getTime() + before.failoverPolicy.flapProtection.quarantineMs).toISOString()
|
||||
: before.failoverRuntimeState.primaryQuarantineUntil;
|
||||
dependencies.state.update((state) => ({
|
||||
...state,
|
||||
appliedProfileId: target.profileId,
|
||||
appliedServerId: target.serverId,
|
||||
appliedServerSnapshot: server,
|
||||
failoverPolicy: reason === 'manual'
|
||||
? { ...state.failoverPolicy, paused: true }
|
||||
: role === 'primary' ? { ...state.failoverPolicy, paused: false } : state.failoverPolicy,
|
||||
failoverRuntimeState: {
|
||||
...state.failoverRuntimeState,
|
||||
lastSwitchAt: switchedAt,
|
||||
holdUntil: role === 'reserve'
|
||||
? new Date(now().getTime() + state.failoverPolicy.minimumReserveMs).toISOString()
|
||||
: null,
|
||||
primaryQuarantineUntil: quarantine,
|
||||
failoverHistory: history,
|
||||
reasonCode: reason,
|
||||
},
|
||||
}));
|
||||
} catch (error) {
|
||||
try {
|
||||
await dependencies.dataplane.selectFailoverRole(from);
|
||||
} catch (rollback) {
|
||||
generation += 1;
|
||||
clearTimer();
|
||||
decisionMemory = undefined;
|
||||
dependencies.state.update((state) => ({
|
||||
...state,
|
||||
failoverPolicy: { ...state.failoverPolicy, paused: true },
|
||||
failoverRuntimeState: { ...state.failoverRuntimeState, reasonCode: 'selector-unknown' },
|
||||
}));
|
||||
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
||||
failed.reason = 'selector-unknown';
|
||||
publish(failed);
|
||||
throw new AggregateError([error, rollback], 'Failover selector rollback failed');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
dependencies.onSwitch?.(from, role, reason);
|
||||
dependencies.onEvent?.({
|
||||
type: 'failover.switched',
|
||||
severity: 'info',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.switched:${dependencies.state.read().revision}`,
|
||||
data: {
|
||||
fromRole: from,
|
||||
toRole: role,
|
||||
primaryLabel: targetServer(dependencies.state.read(), 'primary')?.label || 'Primary',
|
||||
reserveLabel: targetServer(dependencies.state.read(), 'reserve')?.label || 'Reserve',
|
||||
reason,
|
||||
manual: reason === 'manual',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function performRound(capturedGeneration: number) {
|
||||
const prepared = await dependencies.serialize(async () => {
|
||||
const state = dependencies.state.read();
|
||||
const role = currentRole(state);
|
||||
if (
|
||||
capturedGeneration !== generation
|
||||
|| !state.failoverPolicy.enabled
|
||||
|| !state.appliedFailoverPolicy
|
||||
|| !role
|
||||
) return null;
|
||||
const selected = record(await dependencies.dataplane.readFailoverSelector());
|
||||
if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
|
||||
await dependencies.dataplane.setFailoverActivityEnabled(true);
|
||||
collectorEnabled = true;
|
||||
const latest = dependencies.state.read();
|
||||
return capturedGeneration === generation
|
||||
&& latest.failoverPolicy.enabled
|
||||
&& currentRole(latest) === role
|
||||
? { state: latest, policy: latest.failoverPolicy, role }
|
||||
: null;
|
||||
});
|
||||
if (!prepared) return;
|
||||
const { state, policy, role } = prepared;
|
||||
const checkedAt = now().toISOString();
|
||||
let primary;
|
||||
let reserve;
|
||||
try {
|
||||
[primary, reserve] = await Promise.all([
|
||||
assessRole('primary', policy),
|
||||
assessRole('reserve', policy),
|
||||
]);
|
||||
} catch {
|
||||
primary = { health: 'unknown' as const, failingServiceIds: [] };
|
||||
reserve = { health: 'unknown' as const, failingServiceIds: [] };
|
||||
}
|
||||
if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
|
||||
const activityResponse = record(await dependencies.dataplane.readFailoverActivity(
|
||||
policy.trafficGuard.thresholdBytesPerSecond,
|
||||
));
|
||||
if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
|
||||
const activity = record(activityResponse.activity);
|
||||
const observedAt = typeof activity.observedAt === 'string' ? Date.parse(activity.observedAt) : NaN;
|
||||
const activityState = Number.isFinite(observedAt) && now().getTime() - observedAt <= 4_000
|
||||
&& (activity.state === 'active' || activity.state === 'quiet')
|
||||
? activity.state
|
||||
: 'unknown';
|
||||
const decision = nextFailoverDecision({
|
||||
now: now().getTime(),
|
||||
policy,
|
||||
currentRole: role,
|
||||
primaryHealth: primary.health,
|
||||
reserveHealth: reserve.health,
|
||||
activity: activityState,
|
||||
holdUntil: Date.parse(state.failoverRuntimeState.holdUntil || '') || null,
|
||||
primaryQuarantineUntil: Date.parse(state.failoverRuntimeState.primaryQuarantineUntil || '') || null,
|
||||
memory: decisionMemory,
|
||||
});
|
||||
decisionMemory = decision.memory;
|
||||
const previousSnapshot = snapshot;
|
||||
const next = activeSnapshot(state, decision.status);
|
||||
next.currentRole = role;
|
||||
next.primary = {
|
||||
...next.primary,
|
||||
...primary,
|
||||
checkedAt,
|
||||
stateSince: previousSnapshot.primary.health === primary.health
|
||||
? previousSnapshot.primary.stateSince || checkedAt
|
||||
: checkedAt,
|
||||
};
|
||||
next.reserve = {
|
||||
...next.reserve,
|
||||
...reserve,
|
||||
checkedAt,
|
||||
stateSince: previousSnapshot.reserve.health === reserve.health
|
||||
? previousSnapshot.reserve.stateSince || checkedAt
|
||||
: checkedAt,
|
||||
};
|
||||
next.reason = decision.reason;
|
||||
next.nextDecisionAt = decision.nextDecisionAt ? new Date(decision.nextDecisionAt).toISOString() : null;
|
||||
next.trafficActivity = activityState === 'unknown' ? {
|
||||
state: 'unknown',
|
||||
observedAt: Number.isFinite(observedAt) ? new Date(observedAt).toISOString() : checkedAt,
|
||||
windowMs: 10_000,
|
||||
thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
|
||||
totalBytesPerSecond: 0,
|
||||
transmittingConnections: 0,
|
||||
quietSince: null,
|
||||
switchTarget: decision.switchTo,
|
||||
blockers: [],
|
||||
} : {
|
||||
state: activityState,
|
||||
observedAt: String(activity.observedAt),
|
||||
windowMs: Number(activity.windowMs) || 10_000,
|
||||
thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
|
||||
totalBytesPerSecond: Number(activity.totalBytesPerSecond) || 0,
|
||||
transmittingConnections: Number(activity.transmittingConnections) || 0,
|
||||
quietSince: typeof activity.quietSince === 'string' ? activity.quietSince : null,
|
||||
switchTarget: decision.switchTo,
|
||||
blockers: (Array.isArray(activity.blockers) ? activity.blockers : []).slice(0, 3) as FailoverSnapshot['trafficActivity'] extends infer T ? T extends { blockers: infer B } ? B : never : never,
|
||||
};
|
||||
publish(next);
|
||||
if (decision.status === 'waiting-for-idle' && previousSnapshot.status !== 'waiting-for-idle') {
|
||||
dependencies.onEvent?.({
|
||||
type: 'failover.waiting_for_idle',
|
||||
severity: 'info',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.waiting_for_idle:${state.revision}:${role}:${decision.reason}`,
|
||||
data: { fromRole: role, toRole: decision.switchTo || (role === 'primary' ? 'reserve' : 'primary'), reason: decision.reason },
|
||||
});
|
||||
}
|
||||
if (decision.reason === 'both-unhealthy' && previousSnapshot.reason !== 'both-unhealthy') {
|
||||
dependencies.onEvent?.({
|
||||
type: 'failover.both_unhealthy',
|
||||
severity: 'warning',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.both_unhealthy:${state.revision}`,
|
||||
data: { reason: decision.reason },
|
||||
});
|
||||
}
|
||||
if (primary.health === 'healthy' && previousSnapshot.primary.health === 'unhealthy') {
|
||||
dependencies.onEvent?.({
|
||||
type: 'failover.recovered',
|
||||
severity: 'info',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.recovered:${state.revision}:primary`,
|
||||
data: { role: 'primary', reason: decision.reason },
|
||||
});
|
||||
}
|
||||
if (decision.switchTo) {
|
||||
try {
|
||||
const switched = await dependencies.serialize(async () => {
|
||||
const current = dependencies.state.read();
|
||||
if (
|
||||
capturedGeneration !== generation
|
||||
|| !current.failoverPolicy.enabled
|
||||
|| current.failoverPolicy.paused
|
||||
|| currentRole(current) !== role
|
||||
|| !current.appliedFailoverPolicy
|
||||
) return false;
|
||||
let freshPrimary;
|
||||
let freshReserve;
|
||||
try {
|
||||
[freshPrimary, freshReserve] = await Promise.all([
|
||||
assessRole('primary', current.failoverPolicy),
|
||||
assessRole('reserve', current.failoverPolicy),
|
||||
]);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const healthStillAllowsSwitch = decision.switchTo === 'reserve'
|
||||
? freshPrimary.health === 'unhealthy' && freshReserve.health === 'healthy'
|
||||
: freshPrimary.health === 'healthy';
|
||||
if (!healthStillAllowsSwitch || capturedGeneration !== generation) return false;
|
||||
if (current.failoverPolicy.trafficGuard.enabled) {
|
||||
const freshResponse = record(await dependencies.dataplane.readFailoverActivity(
|
||||
current.failoverPolicy.trafficGuard.thresholdBytesPerSecond,
|
||||
));
|
||||
const freshActivity = record(freshResponse.activity);
|
||||
const freshObservedAt = typeof freshActivity.observedAt === 'string'
|
||||
? Date.parse(freshActivity.observedAt)
|
||||
: NaN;
|
||||
const freshQuietSince = typeof freshActivity.quietSince === 'string'
|
||||
? Date.parse(freshActivity.quietSince)
|
||||
: NaN;
|
||||
if (
|
||||
capturedGeneration !== generation
|
||||
|| freshActivity.state !== 'quiet'
|
||||
|| !Number.isFinite(freshObservedAt)
|
||||
|| now().getTime() - freshObservedAt > 4_000
|
||||
|| !Number.isFinite(freshQuietSince)
|
||||
|| now().getTime() - freshQuietSince < current.failoverPolicy.trafficGuard.quietWindowMs
|
||||
) return false;
|
||||
}
|
||||
await switchWithinQueue(decision.switchTo!, decision.reason);
|
||||
return true;
|
||||
});
|
||||
if (!switched) {
|
||||
const cancelled = activeSnapshot(dependencies.state.read(), 'observing');
|
||||
cancelled.reason = 'revalidation-required';
|
||||
publish(cancelled);
|
||||
decisionMemory = undefined;
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
||||
failed.reason = dependencies.state.read().failoverRuntimeState.reasonCode === 'selector-unknown'
|
||||
? 'selector-unknown'
|
||||
: 'switch-failed';
|
||||
publish(failed);
|
||||
dependencies.onEvent?.({
|
||||
type: 'failover.switch_failed',
|
||||
severity: 'error',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.switch_failed:${state.revision}:${role}:${decision.switchTo}`,
|
||||
data: { fromRole: role, toRole: decision.switchTo, reason: decision.reason, errorCode: safeErrorCode(error) },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
if (capturedGeneration !== generation) return;
|
||||
publish(activeSnapshot(dependencies.state.read(), decision.switchTo === 'reserve' ? 'reserve' : 'primary'));
|
||||
decisionMemory = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function runRound() {
|
||||
if (roundPromise) return roundPromise;
|
||||
const capturedGeneration = generation;
|
||||
roundPromise = performRound(capturedGeneration).finally(() => {
|
||||
roundPromise = null;
|
||||
const policy = dependencies.state.read().failoverPolicy;
|
||||
if (capturedGeneration === generation && policy.enabled && dependencies.state.read().appliedFailoverPolicy) {
|
||||
const decisionAt = snapshot.nextDecisionAt ? Date.parse(snapshot.nextDecisionAt) : NaN;
|
||||
const decisionDelay = Number.isFinite(decisionAt)
|
||||
? Math.max(250, decisionAt - now().getTime())
|
||||
: policy.intervalMs;
|
||||
schedule(Math.min(policy.intervalMs, decisionDelay));
|
||||
}
|
||||
});
|
||||
return roundPromise;
|
||||
}
|
||||
|
||||
function save(value: unknown) {
|
||||
return dependencies.serialize(async () => {
|
||||
let policy: FailoverPolicy;
|
||||
try {
|
||||
policy = normalizeFailoverPolicy(value, { strict: true });
|
||||
} catch (cause) {
|
||||
throw new HarborError('REQUEST_INVALID', { cause });
|
||||
}
|
||||
if (policy.enabled && !isFailoverConfigured(policy)) throw new HarborError('REQUEST_INVALID');
|
||||
const before = dependencies.state.read();
|
||||
const candidateState = { ...before, failoverPolicy: policy };
|
||||
if (policy.enabled) {
|
||||
const candidate = dependencies.buildCandidate(candidateState);
|
||||
await dependencies.dataplane.checkConfig(candidate.config);
|
||||
}
|
||||
dependencies.state.update((state) => {
|
||||
const role = before.failoverPolicy.enabled && !policy.enabled ? currentRole(state) : null;
|
||||
const target = role ? state.appliedFailoverPolicy?.[role] : null;
|
||||
return {
|
||||
...state,
|
||||
failoverPolicy: policy,
|
||||
...(target ? {
|
||||
desiredProfileId: target.profileId,
|
||||
profiles: state.profiles.map((profile) => profile.id === target.profileId
|
||||
? { ...profile, desiredServerId: target.serverId }
|
||||
: profile),
|
||||
} : {}),
|
||||
};
|
||||
});
|
||||
decisionMemory = undefined;
|
||||
if (before.failoverPolicy.enabled !== policy.enabled) {
|
||||
dependencies.onEvent?.({
|
||||
type: policy.enabled ? 'failover.enabled' : 'failover.disabled',
|
||||
severity: 'info',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.${policy.enabled ? 'enabled' : 'disabled'}:${dependencies.state.read().revision}`,
|
||||
data: {},
|
||||
});
|
||||
}
|
||||
await reconcileAfterCommit();
|
||||
});
|
||||
}
|
||||
|
||||
function pause(paused: boolean) {
|
||||
return dependencies.serialize(async () => {
|
||||
const before = dependencies.state.read();
|
||||
if (!paused && (
|
||||
!before.appliedFailoverPolicy
|
||||
|| !targetServer(before, 'primary')
|
||||
|| !targetServer(before, 'reserve')
|
||||
)) throw new HarborError('REQUEST_INVALID');
|
||||
dependencies.state.update((state) => ({
|
||||
...state,
|
||||
failoverPolicy: { ...state.failoverPolicy, paused },
|
||||
}));
|
||||
decisionMemory = undefined;
|
||||
dependencies.onEvent?.({
|
||||
type: paused ? 'failover.paused' : 'failover.resumed',
|
||||
severity: 'info',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.${paused ? 'paused' : 'resumed'}:${dependencies.state.read().revision}`,
|
||||
data: {},
|
||||
});
|
||||
await reconcileAfterCommit();
|
||||
});
|
||||
}
|
||||
|
||||
async function manualSwitch(role: FailoverRole) {
|
||||
await dependencies.serialize(async () => {
|
||||
const state = dependencies.state.read();
|
||||
if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
|
||||
throw new HarborError('REQUEST_INVALID');
|
||||
}
|
||||
await switchWithinQueue(role, 'manual');
|
||||
});
|
||||
await reconcileAfterCommit();
|
||||
}
|
||||
|
||||
const prepareActivation = (role: FailoverRole) => dependencies.dataplane.selectFailoverRole(role);
|
||||
|
||||
async function restoreAppliedActivation(state: StoredState) {
|
||||
if (!state.appliedFailoverPolicy) return;
|
||||
const role = currentRole(state);
|
||||
if (!role) throw new HarborError('CONFIG_INVALID');
|
||||
await prepareActivation(role);
|
||||
}
|
||||
|
||||
function checkNow() {
|
||||
return dependencies.serialize(async () => {
|
||||
const state = dependencies.state.read();
|
||||
if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
|
||||
throw new HarborError('REQUEST_INVALID');
|
||||
}
|
||||
generation += 1;
|
||||
clearTimer();
|
||||
const checkedAt = now().toISOString();
|
||||
let primary;
|
||||
let reserve;
|
||||
try {
|
||||
[primary, reserve] = await Promise.all([
|
||||
assessRole('primary', state.failoverPolicy),
|
||||
assessRole('reserve', state.failoverPolicy),
|
||||
]);
|
||||
} catch {
|
||||
primary = { health: 'unknown' as const, failingServiceIds: [] };
|
||||
reserve = { health: 'unknown' as const, failingServiceIds: [] };
|
||||
}
|
||||
const next = activeSnapshot(dependencies.state.read(), 'observing');
|
||||
next.primary = { ...next.primary, ...primary, checkedAt, stateSince: checkedAt };
|
||||
next.reserve = { ...next.reserve, ...reserve, checkedAt, stateSince: checkedAt };
|
||||
next.reason = 'manual-check';
|
||||
publish(next);
|
||||
schedule(state.failoverPolicy.intervalMs);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
snapshot: () => snapshot,
|
||||
save,
|
||||
pause,
|
||||
manualSwitch,
|
||||
checkNow,
|
||||
prepareActivation,
|
||||
restoreAppliedActivation,
|
||||
reconcile,
|
||||
runRound,
|
||||
shutdown: () => deactivate(dependencies.state.read().failoverPolicy),
|
||||
};
|
||||
}
|
||||
|
||||
export type FailoverService = ReturnType<typeof createFailoverService>;
|
||||
@@ -35,6 +35,8 @@ interface RouteRulesDependencies {
|
||||
route?: { isGatewayDirect(): boolean };
|
||||
serialize<T>(operation: () => Promise<T>): Promise<T>;
|
||||
runOperation<T>(operation: () => Promise<T>): Promise<T>;
|
||||
afterApply?: () => Promise<unknown>;
|
||||
restoreAppliedActivation?: (state: StoredState) => Promise<unknown>;
|
||||
}
|
||||
|
||||
export function createRouteRulesService(dependencies: RouteRulesDependencies) {
|
||||
@@ -96,6 +98,7 @@ export function createRouteRulesService(dependencies: RouteRulesDependencies) {
|
||||
...(wasRunning ? { appliedRouteRules: routeRules } : {}),
|
||||
routeRulesRevision: state.routeRulesRevision + 1,
|
||||
}));
|
||||
await dependencies.afterApply?.();
|
||||
} catch (error) {
|
||||
await finishRollback(error, [
|
||||
...(configMutationStarted ? [{
|
||||
@@ -104,7 +107,10 @@ export function createRouteRulesService(dependencies: RouteRulesDependencies) {
|
||||
: dependencies.config.restore(previousConfig),
|
||||
}] : []),
|
||||
...(wasRunning && runtimeMutationStarted ? [{
|
||||
run: () => dependencies.runtime.restoreRunning(),
|
||||
run: async () => {
|
||||
await dependencies.runtime.restoreRunning();
|
||||
await dependencies.restoreAppliedActivation?.(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type StateSnapshot,
|
||||
type StoredState,
|
||||
} from '../../../shared/contracts/state.js';
|
||||
import type { FailoverSnapshot } from '../../../shared/failover.js';
|
||||
|
||||
interface RuntimeState {
|
||||
running?: boolean;
|
||||
@@ -26,6 +27,7 @@ interface StateServiceDependencies {
|
||||
getGatewayAutoState: () => GatewayAutoState;
|
||||
getOperationState: () => OperationState;
|
||||
configExists: () => boolean;
|
||||
getFailoverSnapshot?: () => FailoverSnapshot;
|
||||
}
|
||||
|
||||
function subscriptionHost(value: unknown) {
|
||||
@@ -51,6 +53,7 @@ export function createStateService(dependencies: StateServiceDependencies) {
|
||||
configExists,
|
||||
subscriptionHost: subscriptionHost(storedState.subscriptionUrl),
|
||||
operation: dependencies.getOperationState(),
|
||||
failoverSnapshot: dependencies.getFailoverSnapshot?.(),
|
||||
});
|
||||
return { snapshot, storedState, gatewayAuto, configExists };
|
||||
},
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type StoredState,
|
||||
} from '../../../shared/contracts/state.js';
|
||||
import { HarborError } from '../../../shared/errors.js';
|
||||
import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js';
|
||||
import { finishRollback } from '../../services/rollback.js';
|
||||
|
||||
interface ParsedSubscription {
|
||||
@@ -55,6 +56,11 @@ interface SubscriptionServiceDependencies {
|
||||
clearInterval(handle: TimerHandle): void;
|
||||
};
|
||||
onRefreshError(error: unknown): void;
|
||||
onEvent?: (event: ActivityJournalEventInput) => void;
|
||||
failover?: {
|
||||
reconcile(): Promise<unknown>;
|
||||
restoreAppliedActivation(state: StoredState): Promise<unknown>;
|
||||
};
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
@@ -106,6 +112,10 @@ function mutationResult(profile: Pick<StoredProfile, 'id' | 'label'>): ProfileMu
|
||||
return { success: true, profileId: profile.id, label: profile.label };
|
||||
}
|
||||
|
||||
function publicHost(url: string) {
|
||||
try { return new URL(url).hostname; } catch { return ''; }
|
||||
}
|
||||
|
||||
export function createSubscriptionService(dependencies: SubscriptionServiceDependencies) {
|
||||
const refreshPromises = new Map<string, Promise<ProfileMutationResult>>();
|
||||
let refreshTimer: TimerHandle | null = null;
|
||||
@@ -166,6 +176,13 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
profiles: [...current.profiles, profile],
|
||||
desiredProfileId: current.profiles.length ? current.desiredProfileId : profile.id,
|
||||
}));
|
||||
dependencies.onEvent?.({
|
||||
type: 'subscription.added',
|
||||
severity: 'info',
|
||||
source: 'subscription',
|
||||
dedupeKey: `subscription.added:${dependencies.state.read().revision}`,
|
||||
data: { profileId: profile.id, profileLabel: profile.label, host: publicHost(profile.subscriptionUrl), serverCount: profile.servers.length },
|
||||
});
|
||||
return mutationResult(profile);
|
||||
});
|
||||
};
|
||||
@@ -217,6 +234,7 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
profileId: string,
|
||||
subscriptionUrl: string,
|
||||
error: unknown,
|
||||
origin: 'manual' | 'scheduled',
|
||||
) => dependencies.serialize(async () => {
|
||||
const state = dependencies.state.read();
|
||||
const profile = requireProfile(state, profileId);
|
||||
@@ -230,12 +248,27 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
...current,
|
||||
profiles: replaceProfile(current, failed),
|
||||
}));
|
||||
dependencies.onEvent?.({
|
||||
type: 'subscription.refresh_failed',
|
||||
severity: 'warning',
|
||||
source: 'subscription',
|
||||
dedupeKey: origin === 'scheduled'
|
||||
? `subscription.refresh_failed:${failed.id}:${failed.fetchedAt || 'never'}:${failed.lastRefreshErrorCode}`
|
||||
: `subscription.refresh_failed:${dependencies.state.read().revision}`,
|
||||
data: {
|
||||
profileId: failed.id,
|
||||
profileLabel: failed.label,
|
||||
host: publicHost(failed.subscriptionUrl),
|
||||
errorCode: failed.lastRefreshErrorCode || 'UNKNOWN',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const commitRefresh = (
|
||||
profileId: string,
|
||||
subscriptionUrl: string,
|
||||
parsed: ParsedSubscription,
|
||||
origin: 'manual' | 'scheduled',
|
||||
) => dependencies.serialize(async () => {
|
||||
// Re-read the profile after provider I/O and guard its owner instead of rejecting background-only revisions.
|
||||
const previousState = dependencies.state.read();
|
||||
@@ -257,8 +290,46 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
lastRefreshAttemptAt: parsed.fetchedAt,
|
||||
lastRefreshErrorCode: null,
|
||||
};
|
||||
const contentChanged = JSON.stringify({
|
||||
subscriptionConfig: previousProfile.subscriptionConfig,
|
||||
servers: previousProfile.servers,
|
||||
userInfo: previousProfile.userInfo,
|
||||
}) !== JSON.stringify({
|
||||
subscriptionConfig: refreshedProfile.subscriptionConfig,
|
||||
servers: refreshedProfile.servers,
|
||||
userInfo: refreshedProfile.userInfo,
|
||||
});
|
||||
const appendRefreshEvent = () => {
|
||||
if (origin === 'scheduled' && !contentChanged && !previousProfile.lastRefreshErrorCode) return;
|
||||
dependencies.onEvent?.({
|
||||
type: 'subscription.refreshed',
|
||||
severity: 'info',
|
||||
source: 'subscription',
|
||||
dedupeKey: `subscription.refreshed:${dependencies.state.read().revision}`,
|
||||
data: {
|
||||
profileId: refreshedProfile.id,
|
||||
profileLabel: refreshedProfile.label,
|
||||
host: publicHost(refreshedProfile.subscriptionUrl),
|
||||
serverCount: refreshedProfile.servers.length,
|
||||
added: refreshedProfile.servers.filter(({ id }) => !previousProfile.servers.some((server) => server.id === id)).length,
|
||||
removed: previousProfile.servers.filter(({ id }) => !refreshedProfile.servers.some((server) => server.id === id)).length,
|
||||
},
|
||||
});
|
||||
};
|
||||
const loadedTargets = previousState.appliedFailoverPolicy
|
||||
? [previousState.appliedFailoverPolicy.primary, previousState.appliedFailoverPolicy.reserve]
|
||||
: [];
|
||||
const missingLoadedTarget = loadedTargets.some((target) => (
|
||||
target.profileId === profileId
|
||||
&& !refreshedProfile.servers.some(({ id }) => id === target.serverId)
|
||||
));
|
||||
const pauseFailover = previousState.failoverPolicy?.enabled
|
||||
&& !previousState.failoverPolicy.paused
|
||||
&& missingLoadedTarget;
|
||||
const running = await dependencies.runtime.isRunning();
|
||||
const refreshesApplied = running && previousState.appliedProfileId === profileId;
|
||||
const refreshesApplied = running
|
||||
&& previousState.appliedProfileId === profileId
|
||||
&& !previousState.appliedFailoverPolicy;
|
||||
const nextAppliedServerId = refreshesApplied
|
||||
? dependencies.provider.selectRefreshedServer(
|
||||
previousState.appliedServerId,
|
||||
@@ -271,7 +342,17 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
dependencies.state.update((current) => ({
|
||||
...current,
|
||||
profiles: replaceProfile(current, refreshedProfile),
|
||||
...(pauseFailover ? { failoverPolicy: { ...current.failoverPolicy, paused: true } } : {}),
|
||||
}));
|
||||
if (pauseFailover) dependencies.onEvent?.({
|
||||
type: 'failover.paused',
|
||||
severity: 'warning',
|
||||
source: 'failover',
|
||||
dedupeKey: `failover.paused:missing-target:${profileId}:${dependencies.state.read().revision}`,
|
||||
data: {},
|
||||
});
|
||||
await dependencies.failover?.reconcile();
|
||||
appendRefreshEvent();
|
||||
return mutationResult(refreshedProfile);
|
||||
}
|
||||
|
||||
@@ -305,13 +386,25 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
await finishRollback(error, [
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
...(configMutationStarted ? [{ run: () => restoreConfig(previousConfig) }] : []),
|
||||
...(runtimeMutationStarted ? [{ run: () => dependencies.runtime.start(), runtime: true }] : []),
|
||||
...(runtimeMutationStarted ? [{
|
||||
run: async () => {
|
||||
await dependencies.runtime.start();
|
||||
await dependencies.failover?.restoreAppliedActivation(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
], 'Subscription refresh rollback failed');
|
||||
}
|
||||
await dependencies.failover?.reconcile();
|
||||
appendRefreshEvent();
|
||||
return mutationResult(refreshedProfile);
|
||||
});
|
||||
|
||||
const refreshProfile = (profileIdValue: unknown, expectedRevision?: unknown) => {
|
||||
const refreshProfile = (
|
||||
profileIdValue: unknown,
|
||||
expectedRevision?: unknown,
|
||||
origin: 'manual' | 'scheduled' = 'manual',
|
||||
) => {
|
||||
const profileId = String(profileIdValue || '').trim();
|
||||
const existing = refreshPromises.get(profileId);
|
||||
if (existing) return existing;
|
||||
@@ -328,6 +421,7 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
profileId,
|
||||
initialProfile.subscriptionUrl,
|
||||
error,
|
||||
origin,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
@@ -336,6 +430,7 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
profileId,
|
||||
initialProfile.subscriptionUrl,
|
||||
parsed,
|
||||
origin,
|
||||
);
|
||||
})().finally(() => refreshPromises.delete(profileId));
|
||||
refreshPromises.set(profileId, operation);
|
||||
@@ -353,12 +448,18 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
const mode = String(modeValue || 'delete');
|
||||
if (!['delete', 'stop-and-delete'].includes(mode)) throw new HarborError('REQUEST_INVALID');
|
||||
const running = await dependencies.runtime.isRunning();
|
||||
const applied = running && previousState.appliedProfileId === profile.id;
|
||||
const failoverReferencesProfile = Boolean(previousState.appliedFailoverPolicy && (
|
||||
previousState.appliedFailoverPolicy.primary.profileId === profile.id
|
||||
|| previousState.appliedFailoverPolicy.reserve.profileId === profile.id
|
||||
));
|
||||
const desiredFailoverReferencesProfile = previousState.failoverPolicy?.primary.profileId === profile.id
|
||||
|| previousState.failoverPolicy?.reserve.profileId === profile.id;
|
||||
const applied = running && (previousState.appliedProfileId === profile.id || failoverReferencesProfile);
|
||||
if (applied && mode !== 'stop-and-delete') throw new HarborError('PROFILE_IN_USE');
|
||||
|
||||
const previousConfig = dependencies.config.read();
|
||||
const previousGatewayAuto = dependencies.gatewayAuto.read();
|
||||
const removesAppliedTarget = previousState.appliedProfileId === profile.id;
|
||||
const removesAppliedTarget = previousState.appliedProfileId === profile.id || failoverReferencesProfile;
|
||||
let runtimeMutationStarted = false;
|
||||
let configMutationStarted = false;
|
||||
let gatewayMutationStarted = false;
|
||||
@@ -390,16 +491,49 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
appliedServerSnapshot: current.appliedProfileId === profile.id
|
||||
? null
|
||||
: current.appliedServerSnapshot,
|
||||
...(current.appliedProfileId === profile.id ? { connectionDesired: 'stopped' } : {}),
|
||||
...(removesAppliedTarget ? {
|
||||
connectionDesired: 'stopped',
|
||||
appliedProfileId: '',
|
||||
appliedServerId: '',
|
||||
appliedServerSnapshot: null,
|
||||
appliedFailoverPolicy: null,
|
||||
} : {}),
|
||||
...(failoverReferencesProfile || desiredFailoverReferencesProfile ? {
|
||||
failoverPolicy: {
|
||||
...current.failoverPolicy,
|
||||
enabled: false,
|
||||
paused: false,
|
||||
primary: current.failoverPolicy.primary.profileId === profile.id
|
||||
? { profileId: '', serverId: '' }
|
||||
: current.failoverPolicy.primary,
|
||||
reserve: current.failoverPolicy.reserve.profileId === profile.id
|
||||
? { profileId: '', serverId: '' }
|
||||
: current.failoverPolicy.reserve,
|
||||
},
|
||||
} : {}),
|
||||
}));
|
||||
} catch (error) {
|
||||
await finishRollback(error, [
|
||||
...(stateCommitStarted ? [{ run: () => dependencies.state.update(() => previousState) }] : []),
|
||||
...(gatewayMutationStarted ? [{ run: () => dependencies.gatewayAuto.set(previousGatewayAuto) }] : []),
|
||||
...(configMutationStarted ? [{ run: () => restoreConfig(previousConfig) }] : []),
|
||||
...(runtimeMutationStarted ? [{ run: () => dependencies.runtime.start(), runtime: true }] : []),
|
||||
...(runtimeMutationStarted ? [{
|
||||
run: async () => {
|
||||
await dependencies.runtime.start();
|
||||
await dependencies.failover?.restoreAppliedActivation(previousState);
|
||||
},
|
||||
runtime: true,
|
||||
}] : []),
|
||||
], 'Subscription delete rollback failed');
|
||||
}
|
||||
await dependencies.failover?.reconcile();
|
||||
dependencies.onEvent?.({
|
||||
type: 'subscription.deleted',
|
||||
severity: 'info',
|
||||
source: 'subscription',
|
||||
dedupeKey: `subscription.deleted:${dependencies.state.read().revision}`,
|
||||
data: { profileId: profile.id, profileLabel: profile.label },
|
||||
});
|
||||
return mutationResult(profile);
|
||||
});
|
||||
|
||||
@@ -436,7 +570,7 @@ export function createSubscriptionService(dependencies: SubscriptionServiceDepen
|
||||
void (async () => {
|
||||
for (const { id } of dependencies.state.read().profiles) {
|
||||
try {
|
||||
await refreshProfile(id);
|
||||
await refreshProfile(id, undefined, 'scheduled');
|
||||
} catch (error) {
|
||||
dependencies.onRefreshError(error);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import type { ActivityJournalService } from '../../services/activityJournalService.js';
|
||||
import { sendJson } from '../response.js';
|
||||
|
||||
export function createActivityJournalRoute({ journal }: { journal: ActivityJournalService }) {
|
||||
return {
|
||||
async handle(req: IncomingMessage, res: ServerResponse) {
|
||||
const url = new URL(req.url || '/', 'http://localhost');
|
||||
if (url.pathname !== '/api/activity-journal' || req.method !== 'GET') return false;
|
||||
sendJson(res, 200, journal.page(
|
||||
Number(url.searchParams.get('limit')) || 50,
|
||||
url.searchParams.get('cursor'),
|
||||
));
|
||||
return true;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import type { FailoverService } from '../../features/failover/failoverService.js';
|
||||
import { HarborError } from '../../../shared/errors.js';
|
||||
|
||||
interface FailoverRouteDependencies {
|
||||
appMode: string;
|
||||
failover: Pick<FailoverService, 'save' | 'pause' | 'manualSwitch' | 'checkNow'>;
|
||||
readBody(req: IncomingMessage): Promise<Record<string, unknown>>;
|
||||
withOperation<T>(kind: string, operation: () => Promise<T>, options?: { expectedRevision?: unknown }): Promise<T>;
|
||||
sendState(res: ServerResponse): Promise<void>;
|
||||
}
|
||||
|
||||
export function createFailoverRoute(dependencies: FailoverRouteDependencies) {
|
||||
return {
|
||||
async handle(req: IncomingMessage, res: ServerResponse) {
|
||||
const pathname = new URL(req.url || '/', 'http://localhost').pathname;
|
||||
if (!pathname.startsWith('/api/failover')) return false;
|
||||
if (dependencies.appMode !== 'gateway') throw new HarborError('ENDPOINT_NOT_FOUND');
|
||||
const body = await dependencies.readBody(req);
|
||||
if (pathname === '/api/failover' && req.method === 'PUT') {
|
||||
await dependencies.withOperation(
|
||||
'failover-save',
|
||||
() => dependencies.failover.save(body.policy),
|
||||
{ expectedRevision: body.expectedRevision },
|
||||
);
|
||||
} else if (pathname === '/api/failover/pause' && req.method === 'POST') {
|
||||
if (typeof body.paused !== 'boolean') throw new HarborError('REQUEST_INVALID');
|
||||
await dependencies.withOperation(
|
||||
body.paused ? 'failover-pause' : 'failover-resume',
|
||||
() => dependencies.failover.pause(body.paused as boolean),
|
||||
{ expectedRevision: body.expectedRevision },
|
||||
);
|
||||
} else if (pathname === '/api/failover/switch' && req.method === 'POST') {
|
||||
if (body.role !== 'primary' && body.role !== 'reserve') throw new HarborError('REQUEST_INVALID');
|
||||
await dependencies.withOperation(
|
||||
'failover-switch',
|
||||
() => dependencies.failover.manualSwitch(body.role as 'primary' | 'reserve'),
|
||||
{ expectedRevision: body.expectedRevision },
|
||||
);
|
||||
} else if (pathname === '/api/failover/check' && req.method === 'POST') {
|
||||
await dependencies.failover.checkNow();
|
||||
} else {
|
||||
throw new HarborError('ENDPOINT_NOT_FOUND');
|
||||
}
|
||||
await dependencies.sendState(res);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
}
|
||||
+204
-10
@@ -16,7 +16,10 @@ import {
|
||||
import { createSingboxRuntime } from './singboxRuntime.js';
|
||||
import { tcpPing } from './ping.js';
|
||||
import {
|
||||
buildDualChannelGatewayConfig,
|
||||
buildGatewayConfig,
|
||||
dualChannelConfigMatchesApplied,
|
||||
fingerprintSelectedOutbound,
|
||||
removeSingboxConfig,
|
||||
restoreSingboxConfig,
|
||||
writeSingboxConfig,
|
||||
@@ -81,6 +84,13 @@ import { createConnectivityDiagnosticsRoute } from './http/routes/connectivityDi
|
||||
import { createGatewayPresenceRoute } from './http/routes/gatewayPresenceRoute.js';
|
||||
import { createSharedProxyRoute } from './http/routes/sharedProxyRoute.js';
|
||||
import { createVersionRoute } from './http/routes/versionRoute.js';
|
||||
import { createSingboxSelectorService } from './services/singboxSelectorService.js';
|
||||
import { createFailoverService } from './features/failover/failoverService.js';
|
||||
import { createFailoverRoute } from './http/routes/failoverRoute.js';
|
||||
import { createActivityJournalService } from './services/activityJournalService.js';
|
||||
import { createActivityJournalRoute } from './http/routes/activityJournalRoute.js';
|
||||
import { createDomainTrafficService, readSingboxConnections } from './services/domainTrafficService.js';
|
||||
import type { ActivityJournalEventInput } from '../shared/activityJournal.js';
|
||||
|
||||
const MAX_BODY_BYTES = 1_000_000;
|
||||
const SUBSCRIPTION_REFRESH_INTERVAL_MS = 15 * 60 * 1000;
|
||||
@@ -142,6 +152,14 @@ if (legacyCacheOwnerMismatch) {
|
||||
}
|
||||
}
|
||||
const stateStore = createStateStore(settings.statePath, { legacySubscriptionCache });
|
||||
const activityJournal = createActivityJournalService({ filePath: settings.activityJournalPath });
|
||||
const appendJournal = (event: ActivityJournalEventInput) => {
|
||||
try {
|
||||
activityJournal.append(event);
|
||||
} catch (error) {
|
||||
console.warn(`[journal] событие не сохранено: ${errorMessage(error)}`);
|
||||
}
|
||||
};
|
||||
const deviceStore = createJsonStore<InventoryState>({
|
||||
filePath: settings.deviceStatePath,
|
||||
defaultValue: migrateDeviceInventoryState({}),
|
||||
@@ -283,6 +301,37 @@ const deviceInventory = settings.appMode === 'gateway'
|
||||
const localConnectivityDiagnostics = !remoteDataplane
|
||||
? createConnectivityDiagnosticsService({ proxyPort: settings.diagnosticsProxyPort })
|
||||
: null;
|
||||
const localFailoverDiagnostics = !remoteDataplane ? {
|
||||
primary: createConnectivityDiagnosticsService({ proxyPort: settings.failoverPrimaryProxyPort }),
|
||||
reserve: createConnectivityDiagnosticsService({ proxyPort: settings.failoverReserveProxyPort }),
|
||||
} : null;
|
||||
const localSelector = !remoteDataplane && settings.appMode === 'gateway'
|
||||
? createSingboxSelectorService({ port: settings.singboxApiPort })
|
||||
: null;
|
||||
const localFailoverTraffic = !remoteDataplane && settings.appMode === 'gateway'
|
||||
? createDomainTrafficService({
|
||||
observe: () => readSingboxConnections(settings.singboxApiPort),
|
||||
devices: () => record(deviceInventory?.snapshot()).devices,
|
||||
})
|
||||
: null;
|
||||
let localFailoverTrafficTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
function setLocalFailoverActivityEnabled(enabled: boolean) {
|
||||
if (!localFailoverTraffic) throw new Error('Failover activity недоступна');
|
||||
if (!enabled) {
|
||||
if (localFailoverTrafficTimer) clearInterval(localFailoverTrafficTimer);
|
||||
localFailoverTrafficTimer = null;
|
||||
localFailoverTraffic.disableActivity();
|
||||
return;
|
||||
}
|
||||
localFailoverTraffic.enableActivity();
|
||||
if (localFailoverTrafficTimer) return;
|
||||
const refresh = () => localFailoverTraffic.refresh()
|
||||
.catch((error: unknown) => console.warn(`[control] failover activity: ${errorMessage(error)}`));
|
||||
void refresh();
|
||||
localFailoverTrafficTimer = setInterval(refresh, 2_000);
|
||||
localFailoverTrafficTimer.unref();
|
||||
}
|
||||
|
||||
function requireLocalConnectivityDiagnostics() {
|
||||
if (!localConnectivityDiagnostics) throw new Error('Harbor local diagnostics are not configured');
|
||||
@@ -355,6 +404,48 @@ const gatewayAutoService = createGatewayAutoService({
|
||||
onDiscoveryWarning: (reason) => console.warn(`[control] Gateway не используется: ${reason}`),
|
||||
onTimerError: (error) => console.warn(`[control] Gateway detection failed: ${errorMessage(error)}`),
|
||||
});
|
||||
const failoverDataplane = remoteDataplane ? {
|
||||
checkConfig: (config: unknown) => requireRemoteRuntime().checkConfig(config),
|
||||
runFailoverProbe: (role: 'primary' | 'reserve', services: unknown, target: unknown, timeoutMs: number) => (
|
||||
requireRemoteRuntime().runFailoverProbe(role, services, target, timeoutMs)
|
||||
),
|
||||
readFailoverSelector: () => requireRemoteRuntime().readFailoverSelector(),
|
||||
selectFailoverRole: (role: 'primary' | 'reserve') => requireRemoteRuntime().selectFailoverRole(role),
|
||||
setFailoverActivityEnabled: (enabled: boolean) => requireRemoteRuntime().setFailoverActivityEnabled(enabled),
|
||||
readFailoverActivity: (threshold: number) => requireRemoteRuntime().readFailoverActivity(threshold),
|
||||
} : {
|
||||
checkConfig: async (config: unknown) => singboxRuntime.checkConfig(config),
|
||||
runFailoverProbe: async (role: 'primary' | 'reserve', services: unknown, target: unknown, timeoutMs: number) => {
|
||||
if (!localFailoverDiagnostics) throw new Error('Failover diagnostics недоступна');
|
||||
return localFailoverDiagnostics[role].runVpn({ services, target, timeoutMs });
|
||||
},
|
||||
readFailoverSelector: async () => {
|
||||
if (!localSelector) throw new Error('Failover selector недоступен');
|
||||
return localSelector.read();
|
||||
},
|
||||
selectFailoverRole: async (role: 'primary' | 'reserve') => {
|
||||
if (!localSelector) throw new Error('Failover selector недоступен');
|
||||
return localSelector.select(role);
|
||||
},
|
||||
setFailoverActivityEnabled: async (enabled: boolean) => setLocalFailoverActivityEnabled(enabled),
|
||||
readFailoverActivity: async (threshold: number) => ({
|
||||
activity: localFailoverTraffic?.activitySnapshot(threshold) || null,
|
||||
}),
|
||||
};
|
||||
const failoverService = createFailoverService({
|
||||
state: {
|
||||
read: () => normalizeStoredState(stateStore.read()),
|
||||
update: updateStoredState,
|
||||
},
|
||||
runtime: { isRunning: async () => Boolean((await singboxRuntime.refresh()).running) },
|
||||
dataplane: failoverDataplane,
|
||||
buildCandidate: buildFailoverCandidate,
|
||||
serialize: serializeControl,
|
||||
onWarning: (error) => console.warn(`[control] failover: ${errorMessage(error)}`),
|
||||
onSwitch: (from, to, reason) => console.log(`[control] failover ${from} -> ${to}: ${reason}`),
|
||||
onEvent: appendJournal,
|
||||
});
|
||||
const gatewayFailover = settings.appMode === 'gateway' ? failoverService : null;
|
||||
const stateService = createStateService({
|
||||
appMode: settings.appMode,
|
||||
readStoredState: () => stateStore.read(),
|
||||
@@ -362,6 +453,7 @@ const stateService = createStateService({
|
||||
getGatewayAutoState: gatewayAutoService.read,
|
||||
getOperationState: () => operationState,
|
||||
configExists: () => fs.existsSync(settings.configPath),
|
||||
getFailoverSnapshot: failoverService.snapshot,
|
||||
});
|
||||
const stateRoute = createStateRoute({
|
||||
stateService,
|
||||
@@ -375,6 +467,14 @@ const gatewayAutoRoute = createGatewayAutoRoute({
|
||||
withOperation,
|
||||
readStatePayload: stateRoute.readPayload,
|
||||
});
|
||||
const failoverRoute = createFailoverRoute({
|
||||
appMode: settings.appMode,
|
||||
failover: failoverService,
|
||||
readBody,
|
||||
withOperation,
|
||||
sendState: (res) => stateRoute.send(res),
|
||||
});
|
||||
const activityJournalRoute = createActivityJournalRoute({ journal: activityJournal });
|
||||
const deviceInventoryRoute = createDeviceInventoryRoute({
|
||||
deviceInventory,
|
||||
readBody,
|
||||
@@ -427,9 +527,12 @@ const subscriptionService = createSubscriptionService({
|
||||
update: updateStoredState,
|
||||
},
|
||||
config: {
|
||||
build: (subscriptionConfig, selectedServerId, routeRules) => (
|
||||
buildActiveConfig(subscriptionConfig, selectedServerId, routeRules)
|
||||
),
|
||||
build: (subscriptionConfig, selectedServerId, routeRules) => {
|
||||
const state = normalizeStoredState(stateStore.read());
|
||||
return state.appliedFailoverPolicy
|
||||
? buildFailoverCandidate({ ...state, routeRules }, 'applied').config
|
||||
: buildActiveConfig(subscriptionConfig, selectedServerId, routeRules);
|
||||
},
|
||||
read: () => fs.existsSync(settings.configPath)
|
||||
? fs.readFileSync(settings.configPath, 'utf8')
|
||||
: null,
|
||||
@@ -453,6 +556,12 @@ const subscriptionService = createSubscriptionService({
|
||||
clearInterval: (timer) => clearInterval(timer),
|
||||
},
|
||||
onRefreshError: (error) => console.warn(`[control] подписка не обновлена: ${errorMessage(error)}`),
|
||||
onEvent: appendJournal,
|
||||
failover: gatewayFailover ? {
|
||||
reconcile: () => gatewayFailover.reconcile()
|
||||
.catch((error) => console.warn(`[control] failover reconcile: ${errorMessage(error)}`)),
|
||||
restoreAppliedActivation: gatewayFailover.restoreAppliedActivation,
|
||||
} : undefined,
|
||||
now: () => new Date(),
|
||||
});
|
||||
const serverHealthRoute = createServerHealthRoute({
|
||||
@@ -484,6 +593,14 @@ const connectionService = createConnectionService({
|
||||
isGatewayDirect: () => settings.appMode === 'client'
|
||||
&& gatewayAutoService.read().mode === 'gateway-direct',
|
||||
},
|
||||
failover: gatewayFailover ? {
|
||||
build: buildFailoverCandidate,
|
||||
prepareActivation: gatewayFailover.prepareActivation,
|
||||
restoreAppliedActivation: gatewayFailover.restoreAppliedActivation,
|
||||
reconcile: () => gatewayFailover.reconcile()
|
||||
.catch((error) => console.warn(`[control] failover reconcile: ${errorMessage(error)}`)),
|
||||
} : undefined,
|
||||
onEvent: appendJournal,
|
||||
runtime: {
|
||||
isRunning: async () => Boolean((await singboxRuntime.refresh()).running),
|
||||
start: () => startSingbox(),
|
||||
@@ -524,9 +641,12 @@ const routeRulesService = createRouteRulesService({
|
||||
readConfig: (profileId) => readProfileConfig(profileId),
|
||||
},
|
||||
config: {
|
||||
build: (subscriptionConfig, selectedServerId, routeRules) => (
|
||||
buildActiveConfig(subscriptionConfig, selectedServerId, routeRules)
|
||||
),
|
||||
build: (subscriptionConfig, selectedServerId, routeRules) => {
|
||||
const state = normalizeStoredState(stateStore.read());
|
||||
return state.appliedFailoverPolicy
|
||||
? buildFailoverCandidate({ ...state, routeRules }, 'applied').config
|
||||
: buildActiveConfig(subscriptionConfig, selectedServerId, routeRules);
|
||||
},
|
||||
read: () => fs.existsSync(settings.configPath)
|
||||
? fs.readFileSync(settings.configPath, 'utf8')
|
||||
: null,
|
||||
@@ -548,6 +668,9 @@ const routeRulesService = createRouteRulesService({
|
||||
},
|
||||
serialize: serializeControl,
|
||||
runOperation: (operation) => withOperation('route-rules', operation),
|
||||
afterApply: gatewayFailover ? () => gatewayFailover.reconcile()
|
||||
.catch((error) => console.warn(`[control] failover reconcile: ${errorMessage(error)}`)) : undefined,
|
||||
restoreAppliedActivation: gatewayFailover?.restoreAppliedActivation,
|
||||
});
|
||||
const routeRulesRoute = createRouteRulesRoute({
|
||||
routeRules: routeRulesService,
|
||||
@@ -676,6 +799,47 @@ function buildActiveConfig(
|
||||
});
|
||||
}
|
||||
|
||||
function buildFailoverCandidate(state: StoredState, source: 'desired' | 'applied' = 'desired') {
|
||||
const policy = source === 'applied' ? state.appliedFailoverPolicy : state.failoverPolicy;
|
||||
if (!policy) throw new HarborError('CONFIG_INVALID');
|
||||
if (
|
||||
policy.primary.profileId === policy.reserve.profileId
|
||||
&& policy.primary.serverId === policy.reserve.serverId
|
||||
) throw new HarborError('REQUEST_INVALID');
|
||||
const channel = (role: 'primary' | 'reserve') => {
|
||||
const target = policy[role];
|
||||
const profile = state.profiles.find(({ id }) => id === target.profileId);
|
||||
const server = profile?.servers.find(({ id }) => id === target.serverId);
|
||||
if (!profile || !server || !profile.subscriptionConfig) throw new HarborError('SERVER_NOT_FOUND');
|
||||
return { profile, server };
|
||||
};
|
||||
const primary = channel('primary');
|
||||
const reserve = channel('reserve');
|
||||
const applied = {
|
||||
primary: policy.primary,
|
||||
reserve: policy.reserve,
|
||||
primaryConfigFingerprint: fingerprintSelectedOutbound(primary.profile.subscriptionConfig, primary.server.id),
|
||||
reserveConfigFingerprint: fingerprintSelectedOutbound(reserve.profile.subscriptionConfig, reserve.server.id),
|
||||
};
|
||||
if (source === 'applied' && JSON.stringify(applied) !== JSON.stringify(state.appliedFailoverPolicy)) {
|
||||
throw new HarborError('CONFIG_INVALID');
|
||||
}
|
||||
const defaultRole = source === 'applied'
|
||||
&& state.appliedProfileId === policy.reserve.profileId
|
||||
&& state.appliedServerId === policy.reserve.serverId
|
||||
? 'reserve'
|
||||
: 'primary';
|
||||
return {
|
||||
config: buildDualChannelGatewayConfig({
|
||||
primary: { subscriptionConfig: primary.profile.subscriptionConfig, selectedServerId: primary.server.id },
|
||||
reserve: { subscriptionConfig: reserve.profile.subscriptionConfig, selectedServerId: reserve.server.id },
|
||||
}, { routeRules: state.routeRules, defaultRole }),
|
||||
applied,
|
||||
primaryProfile: primary.profile,
|
||||
primaryServer: primary.server,
|
||||
};
|
||||
}
|
||||
|
||||
const stopSingbox = () => singboxRuntime.stop();
|
||||
const startSingbox = () => singboxRuntime.apply();
|
||||
|
||||
@@ -689,7 +853,13 @@ function writeCurrentConfig() {
|
||||
const server = profile?.servers.find((candidate) => candidate.id === serverId);
|
||||
const subscriptionConfig = profile ? readProfileConfig(profile.id) : null;
|
||||
if (!profile || !server || !subscriptionConfig) return null;
|
||||
const activeConfig = buildActiveConfig(subscriptionConfig, server.id);
|
||||
let activeConfig: unknown;
|
||||
if (state.appliedFailoverPolicy) {
|
||||
const candidate = buildFailoverCandidate(state, 'applied');
|
||||
activeConfig = candidate.config;
|
||||
} else {
|
||||
activeConfig = buildActiveConfig(subscriptionConfig, server.id);
|
||||
}
|
||||
const previousConfig = fs.existsSync(settings.configPath)
|
||||
? fs.readFileSync(settings.configPath, 'utf8')
|
||||
: null;
|
||||
@@ -704,7 +874,7 @@ function writeCurrentConfig() {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { profile, server };
|
||||
return { profile, server, failoverApplied: state.appliedFailoverPolicy };
|
||||
}
|
||||
|
||||
const CONFIG_PROXY_TYPES = new Set(['vless', 'vmess', 'trojan', 'shadowsocks', 'hysteria2']);
|
||||
@@ -717,6 +887,13 @@ function currentConfigMatchesAppliedTarget(state: StoredState) {
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (state.appliedFailoverPolicy) {
|
||||
const expectedRole = state.appliedProfileId === state.appliedFailoverPolicy.reserve.profileId
|
||||
&& state.appliedServerId === state.appliedFailoverPolicy.reserve.serverId
|
||||
? 'reserve'
|
||||
: 'primary';
|
||||
return dualChannelConfigMatchesApplied(config, state.appliedFailoverPolicy, expectedRole);
|
||||
}
|
||||
const proxyOutbounds = (Array.isArray(config.outbounds) ? config.outbounds : [])
|
||||
.map(record)
|
||||
.filter((outbound) => CONFIG_PROXY_TYPES.has(String(outbound.type || '')));
|
||||
@@ -750,6 +927,7 @@ async function reconcileStoppedBoot({ removeConfig = false } = {}) {
|
||||
|| state.appliedProfileId
|
||||
|| state.appliedServerId
|
||||
|| state.appliedServerSnapshot
|
||||
|| state.appliedFailoverPolicy
|
||||
) {
|
||||
updateStoredState((current) => ({
|
||||
...current,
|
||||
@@ -757,6 +935,7 @@ async function reconcileStoppedBoot({ removeConfig = false } = {}) {
|
||||
appliedProfileId: '',
|
||||
appliedServerId: '',
|
||||
appliedServerSnapshot: null,
|
||||
appliedFailoverPolicy: null,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -770,6 +949,8 @@ async function handleApi(req: IncomingMessage, res: ServerResponse) {
|
||||
if (await connectionRuntimeRoute.handle(req, res)) return;
|
||||
if (await routeRulesRoute.handle(req, res)) return;
|
||||
if (await gatewayAutoRoute.handle(req, res)) return;
|
||||
if (await failoverRoute.handle(req, res)) return;
|
||||
if (await activityJournalRoute.handle(req, res)) return;
|
||||
if (await connectivityDiagnosticsRoute.handle(req, res)) return;
|
||||
|
||||
if (await versionRoute.handle(req, res)) return;
|
||||
@@ -822,6 +1003,7 @@ async function shutdown() {
|
||||
subscriptionService.stopAutoRefresh();
|
||||
gatewayAutoService.stopDiscovery();
|
||||
if (deviceDiscoveryTimer) clearInterval(deviceDiscoveryTimer);
|
||||
await gatewayFailover?.shutdown().catch((error) => console.warn(`[control] failover shutdown: ${errorMessage(error)}`));
|
||||
await serializeControl(() => singboxRuntime.shutdown());
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -846,8 +1028,14 @@ if (bootWantsRunning) {
|
||||
&& currentConfigMatchesAppliedTarget(normalizeStoredState(stateStore.read()));
|
||||
if (target || canReuseCurrentConfig) {
|
||||
await startSingbox()
|
||||
.then(() => {
|
||||
.then(async () => {
|
||||
const current = normalizeStoredState(stateStore.read());
|
||||
const bootRole = current.appliedFailoverPolicy
|
||||
&& current.appliedProfileId === current.appliedFailoverPolicy.reserve.profileId
|
||||
&& current.appliedServerId === current.appliedFailoverPolicy.reserve.serverId
|
||||
? 'reserve'
|
||||
: current.appliedFailoverPolicy ? 'primary' : null;
|
||||
if (bootRole) await failoverDataplane.selectFailoverRole(bootRole);
|
||||
const appliedProfile = target?.profile
|
||||
|| current.profiles.find((profile) => profile.id === current.appliedProfileId);
|
||||
const appliedServer = target?.server
|
||||
@@ -866,7 +1054,10 @@ if (bootWantsRunning) {
|
||||
}));
|
||||
}
|
||||
})
|
||||
.catch((error: unknown) => console.warn(`[control] sing-box не запущен: ${errorMessage(error)}`));
|
||||
.catch(async (error: unknown) => {
|
||||
console.warn(`[control] sing-box не запущен: ${errorMessage(error)}`);
|
||||
await reconcileStoppedBoot();
|
||||
});
|
||||
} else {
|
||||
await reconcileStoppedBoot({ removeConfig: true });
|
||||
}
|
||||
@@ -874,6 +1065,9 @@ if (bootWantsRunning) {
|
||||
await reconcileStoppedBoot();
|
||||
}
|
||||
|
||||
await gatewayFailover?.reconcile()
|
||||
.catch((error) => console.warn(`[control] failover reconcile: ${errorMessage(error)}`));
|
||||
|
||||
if (deviceInventory) {
|
||||
await deviceInventory.reconcilePolicies()
|
||||
.catch((error: unknown) => console.warn(`[control] device policy не применена: ${errorMessage(error)}`));
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import {
|
||||
ACTIVITY_JOURNAL_MAX_EVENTS,
|
||||
ACTIVITY_JOURNAL_RETENTION_DAYS,
|
||||
normalizeActivityEventInput,
|
||||
normalizeStoredActivityEvent,
|
||||
type ActivityJournalEvent,
|
||||
type ActivityJournalEventInput,
|
||||
type ActivityJournalPage,
|
||||
} from '../../shared/activityJournal.js';
|
||||
import { createJsonStore } from './stateStore.js';
|
||||
|
||||
interface JournalState {
|
||||
schemaVersion: 1;
|
||||
events: ActivityJournalEvent[];
|
||||
}
|
||||
|
||||
const migrateJournal = (value: unknown): JournalState => {
|
||||
const candidate = value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
events: (Array.isArray(candidate.events) ? candidate.events : [])
|
||||
.map(normalizeStoredActivityEvent)
|
||||
.filter((event): event is ActivityJournalEvent => Boolean(event)),
|
||||
};
|
||||
};
|
||||
|
||||
export function createActivityJournalService({
|
||||
filePath,
|
||||
now = () => new Date(),
|
||||
}: {
|
||||
filePath: string;
|
||||
now?: () => Date;
|
||||
}) {
|
||||
const store = createJsonStore<JournalState>({
|
||||
filePath,
|
||||
defaultValue: { schemaVersion: 1, events: [] },
|
||||
migrate: migrateJournal,
|
||||
});
|
||||
let recoveryRecorded = false;
|
||||
let writeFailed = false;
|
||||
|
||||
function retained(events: ActivityJournalEvent[]) {
|
||||
const cutoff = now().getTime() - ACTIVITY_JOURNAL_RETENTION_DAYS * 86_400_000;
|
||||
return events
|
||||
.filter(({ occurredAt }) => Date.parse(occurredAt) >= cutoff)
|
||||
.slice(-ACTIVITY_JOURNAL_MAX_EVENTS);
|
||||
}
|
||||
|
||||
function append(value: ActivityJournalEventInput) {
|
||||
const input = normalizeActivityEventInput(value);
|
||||
const storedInput = input.dedupeKey ? {
|
||||
...input,
|
||||
dedupeKey: `${input.type}:sha256:${crypto.createHash('sha256').update(input.dedupeKey).digest('hex')}`,
|
||||
} : input;
|
||||
let appended: ActivityJournalEvent | null = null;
|
||||
try {
|
||||
store.update((state) => {
|
||||
const events = retained(state.events);
|
||||
if (storedInput.dedupeKey && events.some(({ dedupeKey }) => dedupeKey === storedInput.dedupeKey)) {
|
||||
return { schemaVersion: 1, events };
|
||||
}
|
||||
appended = {
|
||||
id: crypto.randomUUID(),
|
||||
occurredAt: now().toISOString(),
|
||||
...storedInput,
|
||||
};
|
||||
return { schemaVersion: 1, events: retained([...events, appended]) };
|
||||
});
|
||||
writeFailed = false;
|
||||
} catch (error) {
|
||||
writeFailed = true;
|
||||
throw error;
|
||||
}
|
||||
return appended;
|
||||
}
|
||||
|
||||
function ensureRecoveryEvent() {
|
||||
if (!store.recovery || recoveryRecorded) return;
|
||||
append({
|
||||
type: 'journal.recovered',
|
||||
severity: 'warning',
|
||||
source: 'storage',
|
||||
dedupeKey: `journal.recovered:${store.recovery.recoveredAt}`,
|
||||
data: {},
|
||||
});
|
||||
recoveryRecorded = true;
|
||||
}
|
||||
|
||||
function page(limitValue: unknown = 50, cursorValue: unknown = null): ActivityJournalPage {
|
||||
try {
|
||||
let state = store.read();
|
||||
ensureRecoveryEvent();
|
||||
if (store.recovery) state = store.read();
|
||||
const retainedEvents = retained(state.events);
|
||||
if (retainedEvents.length !== state.events.length) {
|
||||
state = store.update(() => ({ schemaVersion: 1, events: retainedEvents }));
|
||||
}
|
||||
const events = [...state.events].reverse();
|
||||
const limit = Math.min(100, Math.max(1, Number.isSafeInteger(limitValue) ? Number(limitValue) : 50));
|
||||
const cursor = typeof cursorValue === 'string' ? cursorValue : '';
|
||||
const cursorIndex = cursor ? events.findIndex(({ id }) => id === cursor) : -1;
|
||||
if (cursor && cursorIndex < 0) return {
|
||||
events: [],
|
||||
nextCursor: null,
|
||||
retentionDays: 30,
|
||||
generatedAt: now().toISOString(),
|
||||
storage: writeFailed
|
||||
? { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' }
|
||||
: { status: 'ready', errorCode: null },
|
||||
};
|
||||
const safeStart = cursorIndex + 1;
|
||||
const selected = events.slice(safeStart, safeStart + limit);
|
||||
return {
|
||||
events: selected.map((event) => ({ ...event, dedupeKey: null })),
|
||||
nextCursor: safeStart + selected.length < events.length ? selected.at(-1)?.id || null : null,
|
||||
retentionDays: 30,
|
||||
generatedAt: now().toISOString(),
|
||||
storage: writeFailed
|
||||
? { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' }
|
||||
: { status: 'ready', errorCode: null },
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
events: [],
|
||||
nextCursor: null,
|
||||
retentionDays: 30,
|
||||
generatedAt: now().toISOString(),
|
||||
storage: { status: 'error', errorCode: 'JOURNAL_UNAVAILABLE' },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.existsSync(filePath)) {
|
||||
try {
|
||||
const state = store.read();
|
||||
const retainedEvents = retained(state.events);
|
||||
if (retainedEvents.length !== state.events.length) {
|
||||
store.update(() => ({ schemaVersion: 1, events: retainedEvents }));
|
||||
}
|
||||
} catch {
|
||||
writeFailed = true;
|
||||
}
|
||||
}
|
||||
|
||||
return { append, page };
|
||||
}
|
||||
|
||||
export type ActivityJournalService = ReturnType<typeof createActivityJournalService>;
|
||||
@@ -44,6 +44,7 @@ interface RequestOptions {
|
||||
ipv4?: boolean;
|
||||
follow?: boolean;
|
||||
resolve?: string | null;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface RequestResult {
|
||||
@@ -179,7 +180,9 @@ async function request(probe: BaseProbe, path: PathKind, proxyPort: number, exec
|
||||
ipv4 = false,
|
||||
follow = true,
|
||||
resolve = null,
|
||||
timeoutMs = 6_000,
|
||||
}: RequestOptions = {}): Promise<RequestResult> {
|
||||
const boundedTimeoutMs = Math.min(30_000, Math.max(1_000, Math.round(timeoutMs)));
|
||||
const args = [
|
||||
'--silent',
|
||||
'--show-error',
|
||||
@@ -189,9 +192,9 @@ async function request(probe: BaseProbe, path: PathKind, proxyPort: number, exec
|
||||
'--proto-redir',
|
||||
'=https',
|
||||
'--connect-timeout',
|
||||
'3',
|
||||
String(Math.min(3_000, boundedTimeoutMs) / 1_000),
|
||||
'--max-time',
|
||||
'6',
|
||||
String(boundedTimeoutMs / 1_000),
|
||||
'--user-agent',
|
||||
'Harbor-Diagnostics/1',
|
||||
'--output',
|
||||
@@ -379,6 +382,8 @@ async function siteProbe(
|
||||
proxyPort: number,
|
||||
execute: CurlExecutor,
|
||||
sampleCount = 1,
|
||||
timeoutMs = 6_000,
|
||||
retryFailure = true,
|
||||
): Promise<SiteProbeResult> {
|
||||
if (probe.validationError) return {
|
||||
id: probe.id,
|
||||
@@ -391,12 +396,12 @@ async function siteProbe(
|
||||
stage: 'validation',
|
||||
error: probe.validationError,
|
||||
};
|
||||
const options = { follow: probe.follow !== false, resolve: probe.resolve };
|
||||
const options = { follow: probe.follow !== false, resolve: probe.resolve, timeoutMs };
|
||||
const samples = [];
|
||||
for (let attempt = 0; attempt < sampleCount; attempt += 1) {
|
||||
samples.push(await request(probe, path, proxyPort, execute, options));
|
||||
}
|
||||
if (sampleCount === 1 && samples[0] && !samples[0].ok) {
|
||||
if (retryFailure && sampleCount === 1 && samples[0] && !samples[0].ok) {
|
||||
samples.push(await request(probe, path, proxyPort, execute, options));
|
||||
}
|
||||
const status = mostCommon(samples.map(siteStatus)) || 'unavailable';
|
||||
@@ -469,15 +474,18 @@ async function probeTarget(
|
||||
path: PathKind,
|
||||
proxyPort: number,
|
||||
execute: CurlExecutor,
|
||||
timeoutMs = 6_000,
|
||||
sampleCount = TARGET_SAMPLE_COUNT,
|
||||
retryFailure = true,
|
||||
): Promise<ConnectivityPathResult> {
|
||||
const network = target.kind === 'network'
|
||||
? await networkProbe(path, proxyPort, execute, TARGET_SAMPLE_COUNT)
|
||||
? await networkProbe(path, proxyPort, execute, sampleCount)
|
||||
: null;
|
||||
const ip = target.kind === 'ip'
|
||||
? await ipProbe(target.probe, path, proxyPort, execute, TARGET_SAMPLE_COUNT)
|
||||
? await ipProbe(target.probe, path, proxyPort, execute, sampleCount)
|
||||
: null;
|
||||
const site = target.kind === 'site'
|
||||
? await siteProbe(target.probe, path, proxyPort, execute, TARGET_SAMPLE_COUNT)
|
||||
? await siteProbe(target.probe, path, proxyPort, execute, sampleCount, timeoutMs, retryFailure)
|
||||
: null;
|
||||
const ipv4Sources = ip?.family === 4 ? [ip] : [];
|
||||
const ipv6Source = ip?.family === 6 ? ip : null;
|
||||
@@ -537,7 +545,25 @@ export function createConnectivityDiagnosticsService({
|
||||
assessment: assessConnectivity(direct, vpn),
|
||||
};
|
||||
}
|
||||
async function runVpn({ services = [], target: targetId = null, timeoutMs = 6_000 }: {
|
||||
services?: unknown;
|
||||
target?: unknown;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const requestedServices = typeof targetId === 'string' && targetId.startsWith('site:custom-')
|
||||
? (Array.isArray(services) ? services : []).filter((service) => `site:${String(record(service).id || '')}` === targetId)
|
||||
: [];
|
||||
const customProbes = await prepareCustomProbes(requestedServices, lookup);
|
||||
const siteProbes = [...SITE_PROBES, ...customProbes];
|
||||
const target = resolveTarget(targetId, siteProbes);
|
||||
if (!target || target.kind !== 'site') throw new Error('Failover check ожидает site target');
|
||||
return {
|
||||
checkedAt: now(),
|
||||
vpn: await probeTarget(target, 'vpn', proxyPort, execute, timeoutMs, TARGET_SAMPLE_COUNT, false),
|
||||
};
|
||||
}
|
||||
return {
|
||||
run: runOnce,
|
||||
runVpn,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -84,12 +84,33 @@ interface DomainTrafficSnapshot {
|
||||
}>;
|
||||
}
|
||||
|
||||
interface ActivityEntry {
|
||||
device: string;
|
||||
service: string;
|
||||
upload: bigint;
|
||||
download: bigint;
|
||||
}
|
||||
|
||||
interface ActivitySample {
|
||||
at: number;
|
||||
entries: ActivityEntry[];
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
function publicDeviceLabel(value: unknown) {
|
||||
const device = record(value);
|
||||
for (const candidate of [device.alias, device.hostname]) {
|
||||
const label = typeof candidate === 'string' ? candidate.trim() : '';
|
||||
if (label && label.length <= 64 && !/[\/?#@\\]/.test(label) && !net.isIP(label)) return label;
|
||||
}
|
||||
return 'Устройство';
|
||||
}
|
||||
|
||||
const matchesDomain = (domain: string, suffix: string) => domain === suffix || domain.endsWith(`.${suffix}`);
|
||||
|
||||
export function classifyDomain(value: unknown): { domain: string; service: string } | null {
|
||||
@@ -209,6 +230,10 @@ export function createDomainTrafficService({
|
||||
unsupported_source: 0n,
|
||||
};
|
||||
let refreshPromise: Promise<DomainTrafficSnapshot> | null = null;
|
||||
let activityEnabled = false;
|
||||
let activityStartedAt = 0;
|
||||
let activitySamples: ActivitySample[] = [];
|
||||
let quietSince: string | null = null;
|
||||
let current: DomainTrafficSnapshot = {
|
||||
epoch,
|
||||
observedAt: null,
|
||||
@@ -270,6 +295,7 @@ export function createDomainTrafficService({
|
||||
const response = record(await observe());
|
||||
if (!Array.isArray(response.connections)) throw new Error('Sing-box не вернул connections array');
|
||||
const devicesByIp = new Map<string, string | null>();
|
||||
const deviceLabels = new Map<string, string>();
|
||||
const observedDevices = devices();
|
||||
for (const value of Array.isArray(observedDevices) ? observedDevices : []) {
|
||||
const device = record(value);
|
||||
@@ -277,9 +303,11 @@ export function createDomainTrafficService({
|
||||
const id = typeof device.mac === 'string' ? deviceId(device.mac.toLowerCase()) : null;
|
||||
if (!net.isIPv4(ip) || !id) continue;
|
||||
devicesByIp.set(ip, devicesByIp.has(ip) ? null : id);
|
||||
deviceLabels.set(id, publicDeviceLabel(device));
|
||||
}
|
||||
const connections = response.connections.map((connection) => parseConnection(connection, devicesByIp));
|
||||
const activeConnections = new Map<string, PreviousConnection>();
|
||||
const activityEntries: ActivityEntry[] = [];
|
||||
for (const connection of connections) {
|
||||
const previous = previousConnections.get(connection.id);
|
||||
if (connection.outcome !== 'classified' && previous?.outcome !== connection.outcome) {
|
||||
@@ -302,6 +330,16 @@ export function createDomainTrafficService({
|
||||
tracked.uploadBytes += uploadDelta;
|
||||
tracked.downloadBytes += downloadDelta;
|
||||
trackedTotals.set(trackedKey, tracked);
|
||||
if (activityEnabled && connection.outbound === 'vpn' && uploadDelta + downloadDelta > 0n) {
|
||||
activityEntries.push({
|
||||
device: 'deviceId' in connection
|
||||
? deviceLabels.get(connection.deviceId) || 'Устройство'
|
||||
: 'Неизвестное устройство',
|
||||
service: 'service' in connection ? connection.service : 'Не распознано',
|
||||
upload: uploadDelta,
|
||||
download: downloadDelta,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (connection.outcome === 'unknown_device' || connection.outcome === 'unsupported_source') {
|
||||
activeConnections.set(connection.id, {
|
||||
@@ -372,7 +410,12 @@ export function createDomainTrafficService({
|
||||
});
|
||||
}
|
||||
previousConnections = activeConnections;
|
||||
current = { ...current, observedAt: now().toISOString() };
|
||||
const observed = now();
|
||||
if (activityEnabled) {
|
||||
activitySamples.push({ at: observed.getTime(), entries: activityEntries });
|
||||
activitySamples = activitySamples.filter(({ at }) => at >= observed.getTime() - 10_000);
|
||||
}
|
||||
current = { ...current, observedAt: observed.toISOString() };
|
||||
current = buildSnapshot();
|
||||
return current;
|
||||
} catch (error) {
|
||||
@@ -390,5 +433,64 @@ export function createDomainTrafficService({
|
||||
return refreshPromise;
|
||||
}
|
||||
|
||||
return { snapshot: () => current, refresh };
|
||||
function enableActivity() {
|
||||
if (activityEnabled) return;
|
||||
activityEnabled = true;
|
||||
activityStartedAt = now().getTime();
|
||||
activitySamples = [];
|
||||
quietSince = null;
|
||||
}
|
||||
|
||||
function disableActivity() {
|
||||
activityEnabled = false;
|
||||
activityStartedAt = 0;
|
||||
activitySamples = [];
|
||||
quietSince = null;
|
||||
}
|
||||
|
||||
function activitySnapshot(thresholdBytesPerSecond: unknown = 0) {
|
||||
if (!activityEnabled || !current.observedAt) return null;
|
||||
const observedAt = Date.parse(current.observedAt);
|
||||
const threshold = Math.max(0, Number(thresholdBytesPerSecond) || 0);
|
||||
const divisorMs = Math.max(1_000, Math.min(10_000, observedAt - activityStartedAt || 1_000));
|
||||
const totals = new Map<string, ActivityEntry>();
|
||||
let bytes = 0n;
|
||||
for (const sample of activitySamples) {
|
||||
for (const entry of sample.entries) {
|
||||
bytes += entry.upload + entry.download;
|
||||
const key = `${entry.device}\0${entry.service}`;
|
||||
const total = totals.get(key) || { ...entry, upload: 0n, download: 0n };
|
||||
total.upload += entry.upload;
|
||||
total.download += entry.download;
|
||||
totals.set(key, total);
|
||||
}
|
||||
}
|
||||
const totalBytesPerSecond = Number(bytes * 1_000n / BigInt(divisorMs));
|
||||
const active = totalBytesPerSecond > threshold;
|
||||
quietSince = active ? null : quietSince || current.observedAt;
|
||||
const latest = activitySamples.at(-1);
|
||||
return {
|
||||
state: active ? 'active' : 'quiet',
|
||||
observedAt: current.observedAt,
|
||||
windowMs: 10_000,
|
||||
thresholdBytesPerSecond: threshold,
|
||||
totalBytesPerSecond,
|
||||
transmittingConnections: latest?.entries.length || 0,
|
||||
quietSince,
|
||||
blockers: [...totals.values()]
|
||||
.map((entry) => ({
|
||||
device: entry.device,
|
||||
service: entry.service,
|
||||
uploadBytesPerSecond: Number(entry.upload * 1_000n / BigInt(divisorMs)),
|
||||
downloadBytesPerSecond: Number(entry.download * 1_000n / BigInt(divisorMs)),
|
||||
}))
|
||||
.sort((left, right) => (
|
||||
right.uploadBytesPerSecond + right.downloadBytesPerSecond
|
||||
- left.uploadBytesPerSecond - left.downloadBytesPerSecond
|
||||
))
|
||||
.slice(0, 3),
|
||||
};
|
||||
}
|
||||
|
||||
return { snapshot: () => current, refresh, enableActivity, disableActivity, activitySnapshot };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import http from 'node:http';
|
||||
import {
|
||||
FAILOVER_PRIMARY_TAG,
|
||||
FAILOVER_RESERVE_TAG,
|
||||
FAILOVER_SELECTOR_TAG,
|
||||
} from '../singbox.js';
|
||||
|
||||
type Role = 'primary' | 'reserve';
|
||||
|
||||
function request(port: number, method: string, body?: unknown): Promise<unknown> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const encoded = body === undefined ? null : JSON.stringify(body);
|
||||
const req = http.request({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path: `/proxies/${encodeURIComponent(FAILOVER_SELECTOR_TAG)}`,
|
||||
method,
|
||||
headers: encoded ? {
|
||||
'content-type': 'application/json',
|
||||
'content-length': Buffer.byteLength(encoded),
|
||||
} : {},
|
||||
}, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
if ((res.statusCode || 500) >= 400) return reject(new Error(`Sing-box selector HTTP ${res.statusCode}`));
|
||||
if (!chunks.length) return resolve({});
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
||||
} catch (cause) {
|
||||
reject(new Error('Sing-box selector вернул невалидный JSON', { cause }));
|
||||
}
|
||||
});
|
||||
});
|
||||
req.setTimeout(2_000, () => req.destroy(new Error('Sing-box selector timeout')));
|
||||
req.on('error', reject);
|
||||
req.end(encoded);
|
||||
});
|
||||
}
|
||||
|
||||
const tagFor = (role: Role) => role === 'primary' ? FAILOVER_PRIMARY_TAG : FAILOVER_RESERVE_TAG;
|
||||
const roleFor = (tag: unknown): Role | null => (
|
||||
tag === FAILOVER_PRIMARY_TAG ? 'primary' : tag === FAILOVER_RESERVE_TAG ? 'reserve' : null
|
||||
);
|
||||
|
||||
export function createSingboxSelectorService({
|
||||
port,
|
||||
send = (method: string, body?: unknown) => request(port, method, body),
|
||||
}: {
|
||||
port: number;
|
||||
send?: (method: string, body?: unknown) => Promise<unknown>;
|
||||
}) {
|
||||
async function read() {
|
||||
const value = await send('GET') as Record<string, unknown>;
|
||||
const role = roleFor(value.now);
|
||||
if (!role) throw new Error('Sing-box selector вернул неизвестный outbound');
|
||||
return { role };
|
||||
}
|
||||
async function select(role: Role) {
|
||||
await send('PUT', { name: tagFor(role) });
|
||||
const selected = await read();
|
||||
if (selected.role !== role) throw new Error('Sing-box selector не подтвердил переключение');
|
||||
return selected;
|
||||
}
|
||||
return { read, select };
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
type NormalizedServer,
|
||||
} from '../../shared/serverIdentity.js';
|
||||
|
||||
export const STATE_SCHEMA_VERSION = 7;
|
||||
export const STATE_SCHEMA_VERSION = 8;
|
||||
|
||||
export interface AtomicWriteOptions {
|
||||
beforeRename?: (temporaryPath: string, filePath: string) => void;
|
||||
|
||||
+131
-6
@@ -1,13 +1,20 @@
|
||||
import fs from 'node:fs';
|
||||
import crypto from 'node:crypto';
|
||||
import { settings } from './config.js';
|
||||
import { HarborError } from '../shared/errors.js';
|
||||
import { normalizeRouteRules } from '../shared/routingRules.js';
|
||||
import type { AppliedFailoverPolicy } from '../shared/failover.js';
|
||||
import { atomicWriteFile, atomicWriteJson } from './services/stateStore.js';
|
||||
|
||||
const PROXY_TYPES = new Set(['vless', 'vmess', 'trojan', 'shadowsocks', 'hysteria2']);
|
||||
const MIXED_INBOUND = 'mixed-in';
|
||||
const TPROXY_INBOUND = 'tproxy-in';
|
||||
const DIAGNOSTICS_INBOUND = 'diagnostics-vpn-in';
|
||||
const DIAGNOSTICS_PRIMARY_INBOUND = 'diagnostics-primary-in';
|
||||
const DIAGNOSTICS_RESERVE_INBOUND = 'diagnostics-reserve-in';
|
||||
export const FAILOVER_SELECTOR_TAG = 'channel-selector';
|
||||
export const FAILOVER_PRIMARY_TAG = 'channel-primary';
|
||||
export const FAILOVER_RESERVE_TAG = 'channel-reserve';
|
||||
const SNIFF_TIMEOUT = '1s';
|
||||
const SNIFFERS = ['http', 'tls', 'quic'];
|
||||
|
||||
@@ -34,18 +41,66 @@ function findOutbound(subscriptionConfig: unknown, selectedTag: unknown): ProxyO
|
||||
));
|
||||
}
|
||||
|
||||
function selectedOutbound(subscriptionConfig: unknown, selectedTag: unknown, tag?: string) {
|
||||
const outbound = structuredClone(findOutbound(subscriptionConfig, selectedTag));
|
||||
if (!outbound) throw new HarborError('SERVER_NOT_FOUND');
|
||||
if (tag) outbound.tag = tag;
|
||||
else if (!outbound.tag) outbound.tag = 'vpn-out';
|
||||
if (outbound.type === 'vless' && !outbound.packet_encoding) outbound.packet_encoding = 'xudp';
|
||||
return outbound;
|
||||
}
|
||||
|
||||
export interface DualChannelConfig {
|
||||
primary: { subscriptionConfig: unknown; selectedServerId: string };
|
||||
reserve: { subscriptionConfig: unknown; selectedServerId: string };
|
||||
}
|
||||
|
||||
export function fingerprintConfiguredOutbound(value: unknown, selectedServerId: string) {
|
||||
const outbound = structuredClone(record(value)) as ProxyOutbound;
|
||||
if (!PROXY_TYPES.has(String(outbound.type || ''))) throw new HarborError('CONFIG_INVALID');
|
||||
outbound.tag = selectedServerId;
|
||||
if (outbound.type === 'vless' && !outbound.packet_encoding) outbound.packet_encoding = 'xudp';
|
||||
return crypto.createHash('sha256').update(JSON.stringify(outbound)).digest('hex');
|
||||
}
|
||||
|
||||
export function fingerprintSelectedOutbound(subscriptionConfig: unknown, selectedServerId: string) {
|
||||
const outbound = findOutbound(subscriptionConfig, selectedServerId);
|
||||
if (!outbound) throw new HarborError('SERVER_NOT_FOUND');
|
||||
return fingerprintConfiguredOutbound(outbound, selectedServerId);
|
||||
}
|
||||
|
||||
export function dualChannelConfigMatchesApplied(
|
||||
configValue: unknown,
|
||||
applied: AppliedFailoverPolicy,
|
||||
expectedRole: 'primary' | 'reserve',
|
||||
) {
|
||||
const config = record(configValue);
|
||||
const outbounds = (Array.isArray(config.outbounds) ? config.outbounds : []).map(record);
|
||||
const primary = outbounds.filter(({ tag }) => tag === FAILOVER_PRIMARY_TAG);
|
||||
const reserve = outbounds.filter(({ tag }) => tag === FAILOVER_RESERVE_TAG);
|
||||
const selector = outbounds.find(({ tag }) => tag === FAILOVER_SELECTOR_TAG);
|
||||
try {
|
||||
return primary.length === 1
|
||||
&& reserve.length === 1
|
||||
&& fingerprintConfiguredOutbound(primary[0], applied.primary.serverId) === applied.primaryConfigFingerprint
|
||||
&& fingerprintConfiguredOutbound(reserve[0], applied.reserve.serverId) === applied.reserveConfigFingerprint
|
||||
&& selector?.type === 'selector'
|
||||
&& JSON.stringify(selector.outbounds) === JSON.stringify([FAILOVER_PRIMARY_TAG, FAILOVER_RESERVE_TAG])
|
||||
&& selector.default === (expectedRole === 'reserve' ? FAILOVER_RESERVE_TAG : FAILOVER_PRIMARY_TAG)
|
||||
&& selector.interrupt_exist_connections === false
|
||||
&& record(config.route).final === FAILOVER_SELECTOR_TAG;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildGatewayConfig(subscriptionConfig: unknown, selectedTag: unknown, {
|
||||
clientDirect = false,
|
||||
routeRules = [],
|
||||
}: { clientDirect?: boolean; routeRules?: unknown } = {}) {
|
||||
const clientMode = settings.appMode === 'client';
|
||||
const directClient = clientMode && clientDirect;
|
||||
const vpnOutbound = structuredClone(findOutbound(subscriptionConfig, selectedTag));
|
||||
if (!vpnOutbound) throw new HarborError('SERVER_NOT_FOUND');
|
||||
if (!vpnOutbound.tag) vpnOutbound.tag = 'vpn-out';
|
||||
if (vpnOutbound.type === 'vless' && !vpnOutbound.packet_encoding) {
|
||||
vpnOutbound.packet_encoding = 'xudp';
|
||||
}
|
||||
const vpnOutbound = selectedOutbound(subscriptionConfig, selectedTag);
|
||||
const outboundTag = directClient ? 'direct' : vpnOutbound.tag;
|
||||
|
||||
const inbounds = [
|
||||
@@ -124,6 +179,76 @@ export function buildGatewayConfig(subscriptionConfig: unknown, selectedTag: unk
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
export function buildDualChannelGatewayConfig(
|
||||
channels: DualChannelConfig,
|
||||
{ routeRules = [], defaultRole = 'primary' }: { routeRules?: unknown; defaultRole?: 'primary' | 'reserve' } = {},
|
||||
) {
|
||||
if (settings.appMode === 'client') throw new Error('Dual-channel config доступен только Gateway');
|
||||
const primary = selectedOutbound(
|
||||
channels.primary.subscriptionConfig,
|
||||
channels.primary.selectedServerId,
|
||||
FAILOVER_PRIMARY_TAG,
|
||||
);
|
||||
const reserve = selectedOutbound(
|
||||
channels.reserve.subscriptionConfig,
|
||||
channels.reserve.selectedServerId,
|
||||
FAILOVER_RESERVE_TAG,
|
||||
);
|
||||
const userRules = normalizeRouteRules(routeRules)
|
||||
.filter((rule) => rule.enabled)
|
||||
.map((rule) => ({
|
||||
[rule.type]: [rule.value],
|
||||
outbound: rule.outbound === 'vpn' ? FAILOVER_SELECTOR_TAG : 'direct',
|
||||
}));
|
||||
const userInbounds = [TPROXY_INBOUND, MIXED_INBOUND];
|
||||
return {
|
||||
log: { level: settings.logLevel, timestamp: true },
|
||||
experimental: {
|
||||
cache_file: { enabled: true, path: settings.cachePath },
|
||||
clash_api: { external_controller: `127.0.0.1:${settings.singboxApiPort}` },
|
||||
},
|
||||
dns: { independent_cache: true },
|
||||
inbounds: [
|
||||
{ type: 'tproxy', tag: TPROXY_INBOUND, listen: '::', listen_port: settings.tproxyPort },
|
||||
{ type: 'mixed', tag: MIXED_INBOUND, listen: settings.bindIp, listen_port: settings.proxyPort, set_system_proxy: false },
|
||||
{ type: 'mixed', tag: DIAGNOSTICS_INBOUND, listen: '127.0.0.1', listen_port: settings.diagnosticsProxyPort, set_system_proxy: false },
|
||||
{ type: 'mixed', tag: DIAGNOSTICS_PRIMARY_INBOUND, listen: '127.0.0.1', listen_port: settings.failoverPrimaryProxyPort, set_system_proxy: false },
|
||||
{ type: 'mixed', tag: DIAGNOSTICS_RESERVE_INBOUND, listen: '127.0.0.1', listen_port: settings.failoverReserveProxyPort, set_system_proxy: false },
|
||||
],
|
||||
outbounds: [
|
||||
primary,
|
||||
reserve,
|
||||
{
|
||||
type: 'selector',
|
||||
tag: FAILOVER_SELECTOR_TAG,
|
||||
outbounds: [FAILOVER_PRIMARY_TAG, FAILOVER_RESERVE_TAG],
|
||||
default: defaultRole === 'reserve' ? FAILOVER_RESERVE_TAG : FAILOVER_PRIMARY_TAG,
|
||||
interrupt_exist_connections: false,
|
||||
},
|
||||
{ type: 'direct', tag: 'direct' },
|
||||
],
|
||||
route: {
|
||||
rule_set: [],
|
||||
rules: [
|
||||
{
|
||||
inbound: [TPROXY_INBOUND, MIXED_INBOUND, DIAGNOSTICS_INBOUND, DIAGNOSTICS_PRIMARY_INBOUND, DIAGNOSTICS_RESERVE_INBOUND],
|
||||
action: 'sniff',
|
||||
sniffer: SNIFFERS,
|
||||
timeout: SNIFF_TIMEOUT,
|
||||
},
|
||||
{ inbound: [DIAGNOSTICS_PRIMARY_INBOUND], outbound: FAILOVER_PRIMARY_TAG },
|
||||
{ inbound: [DIAGNOSTICS_RESERVE_INBOUND], outbound: FAILOVER_RESERVE_TAG },
|
||||
{ inbound: [DIAGNOSTICS_INBOUND], outbound: FAILOVER_SELECTOR_TAG },
|
||||
...userRules,
|
||||
{ inbound: userInbounds, outbound: FAILOVER_SELECTOR_TAG },
|
||||
],
|
||||
final: FAILOVER_SELECTOR_TAG,
|
||||
auto_detect_interface: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function writeSingboxConfig(config: unknown) {
|
||||
atomicWriteJson(settings.configPath, config);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,23 @@ export function createSingboxRuntime({
|
||||
|
||||
const state = () => ({ running: Boolean(child), startedAt });
|
||||
|
||||
function checkConfig(config: unknown) {
|
||||
const directory = fs.mkdtempSync(`${configPath}.check-`);
|
||||
const candidatePath = `${directory}/config.json`;
|
||||
try {
|
||||
fs.writeFileSync(candidatePath, JSON.stringify(config));
|
||||
const check = spawnSync('sing-box', ['check', '-c', candidatePath], { encoding: 'utf8' });
|
||||
if (check.status !== 0) {
|
||||
throw new HarborError('CONFIG_INVALID', {
|
||||
cause: new Error((check.stderr || check.stdout || check.error?.message || 'sing-box check failed').trim()),
|
||||
});
|
||||
}
|
||||
return { valid: true };
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function stop() {
|
||||
if (gateway) setGatewayInterception(false, tproxyChain);
|
||||
if (!child) {
|
||||
@@ -100,6 +117,7 @@ export function createSingboxRuntime({
|
||||
get running() { return Boolean(child); },
|
||||
get startedAt() { return startedAt; },
|
||||
refresh: async () => state(),
|
||||
checkConfig,
|
||||
apply,
|
||||
restart: () => apply({ force: true }),
|
||||
stop,
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
export const ACTIVITY_JOURNAL_RETENTION_DAYS = 30;
|
||||
export const ACTIVITY_JOURNAL_MAX_EVENTS = 10_000;
|
||||
|
||||
export const ACTIVITY_EVENT_TYPES = [
|
||||
'connection.started', 'connection.stopped', 'connection.failed',
|
||||
'subscription.added', 'subscription.refreshed', 'subscription.refresh_failed', 'subscription.deleted',
|
||||
'failover.enabled', 'failover.disabled', 'failover.paused', 'failover.resumed',
|
||||
'failover.waiting_for_idle', 'failover.switched', 'failover.switch_failed',
|
||||
'failover.both_unhealthy', 'failover.recovered', 'journal.recovered',
|
||||
] as const;
|
||||
|
||||
export type ActivityEventType = typeof ACTIVITY_EVENT_TYPES[number];
|
||||
export type ActivityEventSeverity = 'info' | 'warning' | 'error';
|
||||
export type ActivityEventSource = 'connection' | 'subscription' | 'failover' | 'storage';
|
||||
|
||||
export interface ActivityJournalEvent {
|
||||
id: string;
|
||||
occurredAt: string;
|
||||
type: ActivityEventType | 'unknown';
|
||||
severity: ActivityEventSeverity;
|
||||
source: ActivityEventSource;
|
||||
dedupeKey: string | null;
|
||||
data: Record<string, string | number | boolean | null>;
|
||||
}
|
||||
|
||||
export type ActivityJournalEventInput = Omit<ActivityJournalEvent, 'id' | 'occurredAt'>;
|
||||
|
||||
export interface ActivityJournalPage {
|
||||
events: ActivityJournalEvent[];
|
||||
nextCursor: string | null;
|
||||
retentionDays: 30;
|
||||
generatedAt: string;
|
||||
storage: { status: 'ready' | 'error'; errorCode: string | null };
|
||||
}
|
||||
|
||||
const ALLOWED_DATA_KEYS: Record<ActivityEventType, readonly string[]> = {
|
||||
'connection.started': ['profileLabel', 'serverLabel'],
|
||||
'connection.stopped': [],
|
||||
'connection.failed': ['errorCode'],
|
||||
'subscription.added': ['profileId', 'profileLabel', 'host', 'serverCount'],
|
||||
'subscription.refreshed': ['profileId', 'profileLabel', 'host', 'serverCount', 'added', 'removed'],
|
||||
'subscription.refresh_failed': ['profileId', 'profileLabel', 'host', 'errorCode'],
|
||||
'subscription.deleted': ['profileId', 'profileLabel'],
|
||||
'failover.enabled': ['primaryLabel', 'reserveLabel'],
|
||||
'failover.disabled': [],
|
||||
'failover.paused': [],
|
||||
'failover.resumed': [],
|
||||
'failover.waiting_for_idle': ['fromRole', 'toRole', 'reason'],
|
||||
'failover.switched': ['fromRole', 'toRole', 'primaryLabel', 'reserveLabel', 'reason', 'manual'],
|
||||
'failover.switch_failed': ['fromRole', 'toRole', 'reason', 'errorCode'],
|
||||
'failover.both_unhealthy': ['reason'],
|
||||
'failover.recovered': ['role', 'reason'],
|
||||
'journal.recovered': [],
|
||||
};
|
||||
|
||||
const typeSet = new Set<string>(ACTIVITY_EVENT_TYPES);
|
||||
const severitySet = new Set(['info', 'warning', 'error']);
|
||||
const sourceSet = new Set(['connection', 'subscription', 'failover', 'storage']);
|
||||
const record = (value: unknown): Record<string, unknown> => (
|
||||
value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
||||
);
|
||||
|
||||
const LABEL_KEYS = new Set(['profileLabel', 'serverLabel', 'primaryLabel', 'reserveLabel']);
|
||||
const ROLE_KEYS = new Set(['fromRole', 'toRole', 'role']);
|
||||
const SAFE_LABEL_FALLBACKS: Record<string, string> = {
|
||||
profileLabel: 'Подписка',
|
||||
serverLabel: 'Сервер',
|
||||
primaryLabel: 'Основной канал',
|
||||
reserveLabel: 'Резервный канал',
|
||||
};
|
||||
|
||||
function safeScalar(key: string, value: unknown) {
|
||||
if (value === null || typeof value === 'boolean') return value;
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) return value;
|
||||
if (typeof value !== 'string' || value.length > 120 || /[\r\n]/.test(value)) {
|
||||
throw new TypeError('Unsafe journal value');
|
||||
}
|
||||
if (LABEL_KEYS.has(key) && (
|
||||
!value.trim()
|
||||
|| /(?:[a-z][a-z0-9+.-]*:\/\/)|[\/?#@\\]/i.test(value)
|
||||
|| /(?:^|\D)(?:\d{1,3}\.){3}\d{1,3}(?:\D|$)/.test(value)
|
||||
|| /(?:^|[^0-9a-f])(?:[0-9a-f]{0,4}:){2,}[0-9a-f]{0,4}(?:[^0-9a-f]|$)/i.test(value)
|
||||
)) return SAFE_LABEL_FALLBACKS[key];
|
||||
if (ROLE_KEYS.has(key) && !['primary', 'reserve'].includes(value)) throw new TypeError('Unsafe journal role');
|
||||
if (key === 'errorCode' && !/^[A-Z0-9_]{1,50}$/.test(value)) throw new TypeError('Unsafe journal error code');
|
||||
if (key === 'reason' && !/^[a-z0-9-]{1,80}$/.test(value)) throw new TypeError('Unsafe journal reason');
|
||||
if (key === 'profileId' && !/^[a-zA-Z0-9_-]{1,80}$/.test(value)) throw new TypeError('Unsafe journal profile id');
|
||||
if (key === 'host' && (
|
||||
!/^[a-z0-9.-]{1,120}$/i.test(value)
|
||||
|| /^(?:\d{1,3}\.){3}\d{1,3}$/.test(value)
|
||||
|| value.includes('..')
|
||||
)) return 'Провайдер';
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeActivityEventInput(value: unknown): ActivityJournalEventInput {
|
||||
const candidate = record(value);
|
||||
const type = String(candidate.type || '') as ActivityEventType;
|
||||
const severity = String(candidate.severity || '') as ActivityEventSeverity;
|
||||
const source = String(candidate.source || '') as ActivityEventSource;
|
||||
if (!typeSet.has(type) || !severitySet.has(severity) || !sourceSet.has(source)) {
|
||||
throw new TypeError('Unknown journal event');
|
||||
}
|
||||
const inputData = record(candidate.data);
|
||||
const allowed = new Set(ALLOWED_DATA_KEYS[type]);
|
||||
if (Object.keys(inputData).some((key) => !allowed.has(key))) throw new TypeError('Unsafe journal data key');
|
||||
const data = Object.fromEntries(Object.entries(inputData).map(([key, item]) => [key, safeScalar(key, item)]));
|
||||
const dedupeKey = candidate.dedupeKey == null ? null : String(candidate.dedupeKey).trim();
|
||||
if (dedupeKey !== null && (
|
||||
!dedupeKey.startsWith(`${type}:`)
|
||||
|| !/^[a-zA-Z0-9_.:-]{1,160}$/.test(dedupeKey)
|
||||
)) {
|
||||
throw new TypeError('Invalid journal dedupe key');
|
||||
}
|
||||
return { type, severity, source, dedupeKey, data };
|
||||
}
|
||||
|
||||
export function normalizeStoredActivityEvent(value: unknown): ActivityJournalEvent | null {
|
||||
const candidate = record(value);
|
||||
const id = typeof candidate.id === 'string' && /^[a-f0-9-]{20,50}$/i.test(candidate.id) ? candidate.id : '';
|
||||
const occurredAt = typeof candidate.occurredAt === 'string' && Number.isFinite(Date.parse(candidate.occurredAt))
|
||||
? candidate.occurredAt
|
||||
: '';
|
||||
if (!id || !occurredAt) return null;
|
||||
try {
|
||||
return { id, occurredAt, ...normalizeActivityEventInput(candidate) };
|
||||
} catch {
|
||||
const severity = String(candidate.severity || '') as ActivityEventSeverity;
|
||||
const source = String(candidate.source || '') as ActivityEventSource;
|
||||
return severitySet.has(severity) && sourceSet.has(source) && typeof candidate.type === 'string'
|
||||
&& /^[a-z][a-z0-9_.-]{0,79}$/.test(candidate.type)
|
||||
? { id, occurredAt, type: 'unknown', severity, source, dedupeKey: null, data: {} }
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
export function assertActivityJournalPage(value: unknown): ActivityJournalPage {
|
||||
const candidate = record(value);
|
||||
const events = Array.isArray(candidate.events) ? candidate.events.map(normalizeStoredActivityEvent) : [];
|
||||
const storage = record(candidate.storage);
|
||||
if (
|
||||
!Array.isArray(candidate.events) || events.some((event) => event === null)
|
||||
|| !(candidate.nextCursor === null || typeof candidate.nextCursor === 'string')
|
||||
|| candidate.retentionDays !== ACTIVITY_JOURNAL_RETENTION_DAYS
|
||||
|| typeof candidate.generatedAt !== 'string' || !Number.isFinite(Date.parse(candidate.generatedAt))
|
||||
|| !['ready', 'error'].includes(String(storage.status || ''))
|
||||
|| !(storage.errorCode === null || typeof storage.errorCode === 'string')
|
||||
) throw new TypeError('Invalid activity journal page');
|
||||
return {
|
||||
events: events as ActivityJournalEvent[],
|
||||
nextCursor: candidate.nextCursor as string | null,
|
||||
retentionDays: 30,
|
||||
generatedAt: candidate.generatedAt,
|
||||
storage: { status: storage.status as 'ready' | 'error', errorCode: storage.errorCode as string | null },
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,16 @@ import {
|
||||
normalizeDiagnosticSettings,
|
||||
type DiagnosticSettings,
|
||||
} from '../connectivityDiagnostics.js';
|
||||
import {
|
||||
createIdleFailoverSnapshot,
|
||||
normalizeAppliedFailoverPolicy,
|
||||
normalizeFailoverPolicy,
|
||||
normalizeFailoverRuntimeState,
|
||||
type AppliedFailoverPolicy,
|
||||
type FailoverPolicy,
|
||||
type FailoverRuntimeState,
|
||||
type FailoverSnapshot,
|
||||
} from '../failover.js';
|
||||
|
||||
export type HarborMode = 'client' | 'gateway';
|
||||
export type ConnectionState = 'running' | 'stopped';
|
||||
@@ -83,6 +93,7 @@ export interface StateSnapshot {
|
||||
lastError: string | null;
|
||||
};
|
||||
diagnostics: DiagnosticSettings;
|
||||
failover: FailoverSnapshot;
|
||||
route: {
|
||||
rulesContractVersion?: typeof ROUTE_RULES_CONTRACT_VERSION;
|
||||
mode: string;
|
||||
@@ -122,6 +133,9 @@ export interface PersistedState extends Record<string, unknown> {
|
||||
connectionDesired?: ConnectionState;
|
||||
gatewayAutoEnabled?: boolean;
|
||||
diagnostics: DiagnosticSettings;
|
||||
failoverPolicy: FailoverPolicy;
|
||||
failoverRuntimeState: FailoverRuntimeState;
|
||||
appliedFailoverPolicy: AppliedFailoverPolicy | null;
|
||||
}
|
||||
|
||||
// Legacy fields are derived in memory for bounded callers during the v5 cutover.
|
||||
@@ -293,6 +307,9 @@ export function normalizeStoredState(value: unknown): StoredState {
|
||||
? state.routeRulesRevision
|
||||
: 0,
|
||||
diagnostics: normalizeDiagnosticSettings(state.diagnostics),
|
||||
failoverPolicy: normalizeFailoverPolicy(state.failoverPolicy),
|
||||
failoverRuntimeState: normalizeFailoverRuntimeState(state.failoverRuntimeState),
|
||||
appliedFailoverPolicy: normalizeAppliedFailoverPolicy(state.appliedFailoverPolicy),
|
||||
subscriptionUrl: selectedProfile?.subscriptionUrl || '',
|
||||
selectedServerId,
|
||||
selectedTag: selectedServer?.label || '',
|
||||
@@ -327,6 +344,7 @@ export function createStateSnapshot({
|
||||
appMode,
|
||||
configExists,
|
||||
operation = { kind: null, status: 'idle', startedAt: null, error: null },
|
||||
failoverSnapshot,
|
||||
now = new Date(),
|
||||
}: {
|
||||
storedState: unknown;
|
||||
@@ -336,6 +354,7 @@ export function createStateSnapshot({
|
||||
configExists: boolean;
|
||||
subscriptionHost?: string;
|
||||
operation?: OperationState;
|
||||
failoverSnapshot?: FailoverSnapshot | null;
|
||||
now?: Date;
|
||||
}): StateSnapshot {
|
||||
const stored = normalizeStoredState(storedState);
|
||||
@@ -391,6 +410,7 @@ export function createStateSnapshot({
|
||||
lastError: null,
|
||||
},
|
||||
diagnostics: stored.diagnostics,
|
||||
failover: failoverSnapshot || createIdleFailoverSnapshot(stored.failoverPolicy),
|
||||
route: {
|
||||
rulesContractVersion: ROUTE_RULES_CONTRACT_VERSION,
|
||||
mode: routeMode,
|
||||
@@ -429,18 +449,24 @@ export function assertStateSnapshot(snapshot: unknown): StateSnapshot {
|
||||
const candidateWithDiagnostics = rawCandidate && rawCandidate.diagnostics === undefined
|
||||
? { ...rawCandidate, diagnostics: normalizeDiagnosticSettings(null) }
|
||||
: rawCandidate;
|
||||
const candidate = candidateWithDiagnostics?.route?.rulesContractVersion === undefined
|
||||
&& Array.isArray(candidateWithDiagnostics?.route?.localRules)
|
||||
&& Array.isArray(candidateWithDiagnostics?.route?.activeLocalRules)
|
||||
const candidateWithFailover = candidateWithDiagnostics && candidateWithDiagnostics.failover === undefined
|
||||
? {
|
||||
...candidateWithDiagnostics,
|
||||
route: {
|
||||
...candidateWithDiagnostics.route,
|
||||
localRules: candidateWithDiagnostics.route.localRules.map(legacyRule),
|
||||
activeLocalRules: candidateWithDiagnostics.route.activeLocalRules.map(legacyRule),
|
||||
},
|
||||
failover: createIdleFailoverSnapshot(normalizeFailoverPolicy(null)),
|
||||
}
|
||||
: candidateWithDiagnostics;
|
||||
const candidate = candidateWithFailover?.route?.rulesContractVersion === undefined
|
||||
&& Array.isArray(candidateWithFailover?.route?.localRules)
|
||||
&& Array.isArray(candidateWithFailover?.route?.activeLocalRules)
|
||||
? {
|
||||
...candidateWithFailover,
|
||||
route: {
|
||||
...candidateWithFailover.route,
|
||||
localRules: candidateWithFailover.route.localRules.map(legacyRule),
|
||||
activeLocalRules: candidateWithFailover.route.activeLocalRules.map(legacyRule),
|
||||
},
|
||||
}
|
||||
: candidateWithFailover;
|
||||
const validDate = (value: unknown) => typeof value === 'string' && Number.isFinite(Date.parse(value));
|
||||
const nullableDate = (value: unknown) => value === null || validDate(value);
|
||||
const nullableString = (value: unknown) => value === null || typeof value === 'string';
|
||||
@@ -486,6 +512,37 @@ export function assertStateSnapshot(snapshot: unknown): StateSnapshot {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const validFailover = (value: FailoverSnapshot) => {
|
||||
const channel = (item: FailoverSnapshot['primary']) => (
|
||||
item && typeof item.target?.profileId === 'string' && typeof item.target?.serverId === 'string'
|
||||
&& ['healthy', 'unhealthy', 'unknown', 'not-monitoring'].includes(item.health)
|
||||
&& Array.isArray(item.failingServiceIds) && item.failingServiceIds.every((id) => typeof id === 'string')
|
||||
&& nullableDate(item.checkedAt) && nullableDate(item.stateSince)
|
||||
);
|
||||
const activity = value.trafficActivity;
|
||||
try {
|
||||
normalizeFailoverPolicy(value.policy, { strict: true });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return channel(value.primary) && channel(value.reserve)
|
||||
&& nullableDate(value.nextDecisionAt) && nullableString(value.reason)
|
||||
&& (activity === null || (
|
||||
['active', 'quiet', 'unknown'].includes(activity.state)
|
||||
&& validDate(activity.observedAt)
|
||||
&& [activity.windowMs, activity.thresholdBytesPerSecond, activity.totalBytesPerSecond, activity.transmittingConnections]
|
||||
.every((number) => Number.isFinite(number) && number >= 0)
|
||||
&& nullableDate(activity.quietSince)
|
||||
&& (activity.switchTarget === null || ['primary', 'reserve'].includes(activity.switchTarget))
|
||||
&& Array.isArray(activity.blockers)
|
||||
&& activity.blockers.length <= 3
|
||||
&& activity.blockers.every((blocker) => (
|
||||
typeof blocker.device === 'string' && typeof blocker.service === 'string'
|
||||
&& Number.isFinite(blocker.uploadBytesPerSecond) && blocker.uploadBytesPerSecond >= 0
|
||||
&& Number.isFinite(blocker.downloadBytesPerSecond) && blocker.downloadBytesPerSecond >= 0
|
||||
))
|
||||
));
|
||||
};
|
||||
|
||||
if (
|
||||
!snapshot ||
|
||||
@@ -516,6 +573,18 @@ export function assertStateSnapshot(snapshot: unknown): StateSnapshot {
|
||||
!nullableDate(candidate.connection.startedAt) ||
|
||||
!nullableString(candidate.connection.lastError) ||
|
||||
!validDiagnostics(candidate.diagnostics) ||
|
||||
!candidate.failover ||
|
||||
typeof candidate.failover.observationEpoch !== 'string' ||
|
||||
!Number.isSafeInteger(candidate.failover.observationSequence) ||
|
||||
candidate.failover.observationSequence < 0 ||
|
||||
typeof candidate.failover.enabled !== 'boolean' ||
|
||||
typeof candidate.failover.paused !== 'boolean' ||
|
||||
typeof candidate.failover.configured !== 'boolean' ||
|
||||
!candidate.failover.policy ||
|
||||
!['inactive', 'active', 'pending', 'passive-loaded'].includes(candidate.failover.activation) ||
|
||||
!['primary', 'reserve', 'other', 'none'].includes(candidate.failover.currentRole) ||
|
||||
!['idle', 'observing', 'primary', 'reserve', 'waiting-for-idle', 'blocked', 'switching', 'error'].includes(candidate.failover.status) ||
|
||||
!validFailover(candidate.failover) ||
|
||||
!candidate.route ||
|
||||
![undefined, ROUTE_RULES_CONTRACT_VERSION].includes(candidate.route.rulesContractVersion) ||
|
||||
typeof candidate.route.mode !== 'string' ||
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
export type FailoverRole = 'primary' | 'reserve';
|
||||
export type FailoverHealth = 'healthy' | 'unhealthy' | 'unknown' | 'not-monitoring';
|
||||
|
||||
export interface FailoverTarget {
|
||||
profileId: string;
|
||||
serverId: string;
|
||||
}
|
||||
|
||||
export interface FailoverCheck {
|
||||
serviceId: string;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface FailoverPolicy {
|
||||
version: 1;
|
||||
enabled: boolean;
|
||||
paused: boolean;
|
||||
primary: FailoverTarget;
|
||||
reserve: FailoverTarget;
|
||||
checks: FailoverCheck[];
|
||||
intervalMs: number;
|
||||
failureWindowMs: number;
|
||||
recoveryWindowMs: number;
|
||||
trafficGuard: {
|
||||
enabled: boolean;
|
||||
thresholdBytesPerSecond: number;
|
||||
quietWindowMs: number;
|
||||
};
|
||||
minimumReserveMs: number;
|
||||
flapProtection: {
|
||||
count: number;
|
||||
windowMs: number;
|
||||
quarantineMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface FailoverRuntimeState {
|
||||
lastSwitchAt: string | null;
|
||||
holdUntil: string | null;
|
||||
primaryQuarantineUntil: string | null;
|
||||
failoverHistory: string[];
|
||||
reasonCode: string | null;
|
||||
}
|
||||
|
||||
export interface AppliedFailoverPolicy {
|
||||
primary: FailoverTarget;
|
||||
reserve: FailoverTarget;
|
||||
primaryConfigFingerprint: string;
|
||||
reserveConfigFingerprint: string;
|
||||
}
|
||||
|
||||
export interface FailoverActivity {
|
||||
state: 'active' | 'quiet' | 'unknown';
|
||||
observedAt: string;
|
||||
windowMs: number;
|
||||
thresholdBytesPerSecond: number;
|
||||
totalBytesPerSecond: number;
|
||||
transmittingConnections: number;
|
||||
quietSince: string | null;
|
||||
switchTarget: FailoverRole | null;
|
||||
blockers: Array<{
|
||||
device: string;
|
||||
service: string;
|
||||
uploadBytesPerSecond: number;
|
||||
downloadBytesPerSecond: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface FailoverSnapshot {
|
||||
observationEpoch: string;
|
||||
observationSequence: number;
|
||||
configured: boolean;
|
||||
enabled: boolean;
|
||||
paused: boolean;
|
||||
activation: 'inactive' | 'active' | 'pending' | 'passive-loaded';
|
||||
currentRole: FailoverRole | 'other' | 'none';
|
||||
status: 'idle' | 'observing' | 'primary' | 'reserve' | 'waiting-for-idle' | 'blocked' | 'switching' | 'error';
|
||||
primary: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null };
|
||||
reserve: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null };
|
||||
nextDecisionAt: string | null;
|
||||
reason: string | null;
|
||||
trafficActivity: FailoverActivity | null;
|
||||
policy: FailoverPolicy;
|
||||
}
|
||||
|
||||
export interface FailoverDecisionMemory {
|
||||
primaryFailedSince: number | null;
|
||||
primaryRecoveredSince: number | null;
|
||||
quietSince: number | null;
|
||||
}
|
||||
|
||||
export interface FailoverDecisionInput {
|
||||
now: number;
|
||||
policy: FailoverPolicy;
|
||||
currentRole: FailoverRole;
|
||||
primaryHealth: Exclude<FailoverHealth, 'not-monitoring'>;
|
||||
reserveHealth: Exclude<FailoverHealth, 'not-monitoring'>;
|
||||
activity: 'active' | 'quiet' | 'unknown';
|
||||
holdUntil?: number | null;
|
||||
primaryQuarantineUntil?: number | null;
|
||||
memory?: FailoverDecisionMemory;
|
||||
}
|
||||
|
||||
export interface FailoverDecision {
|
||||
status: FailoverSnapshot['status'];
|
||||
reason: string;
|
||||
switchTo: FailoverRole | null;
|
||||
memory: FailoverDecisionMemory;
|
||||
nextDecisionAt: number | null;
|
||||
}
|
||||
|
||||
const text = (value: unknown) => typeof value === 'string' ? value.trim() : '';
|
||||
const record = (value: unknown): Record<string, unknown> => (
|
||||
value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
||||
);
|
||||
const bounded = (value: unknown, fallback: number, minimum: number, maximum: number) => {
|
||||
const candidate = Number(value);
|
||||
return Number.isFinite(candidate) ? Math.min(maximum, Math.max(minimum, Math.round(candidate))) : fallback;
|
||||
};
|
||||
const equivalent = (left: unknown, right: unknown): boolean => {
|
||||
if (Array.isArray(left) || Array.isArray(right)) {
|
||||
return Array.isArray(left) && Array.isArray(right)
|
||||
&& left.length === right.length
|
||||
&& left.every((value, index) => equivalent(value, right[index]));
|
||||
}
|
||||
if (left && right && typeof left === 'object' && typeof right === 'object') {
|
||||
const leftRecord = left as Record<string, unknown>;
|
||||
const rightRecord = right as Record<string, unknown>;
|
||||
const leftKeys = Object.keys(leftRecord).sort();
|
||||
const rightKeys = Object.keys(rightRecord).sort();
|
||||
return leftKeys.length === rightKeys.length
|
||||
&& leftKeys.every((key, index) => key === rightKeys[index] && equivalent(leftRecord[key], rightRecord[key]));
|
||||
}
|
||||
return Object.is(left, right);
|
||||
};
|
||||
|
||||
export const DEFAULT_FAILOVER_POLICY: FailoverPolicy = Object.freeze({
|
||||
version: 1,
|
||||
enabled: false,
|
||||
paused: false,
|
||||
primary: { profileId: '', serverId: '' },
|
||||
reserve: { profileId: '', serverId: '' },
|
||||
checks: [{ serviceId: 'youtube', timeoutMs: 6_000 }, { serviceId: 'google', timeoutMs: 6_000 }],
|
||||
intervalMs: 60_000,
|
||||
failureWindowMs: 120_000,
|
||||
recoveryWindowMs: 900_000,
|
||||
trafficGuard: { enabled: true, thresholdBytesPerSecond: 32 * 1024, quietWindowMs: 30_000 },
|
||||
minimumReserveMs: 600_000,
|
||||
flapProtection: { count: 3, windowMs: 86_400_000, quarantineMs: 21_600_000 },
|
||||
});
|
||||
|
||||
export const DEFAULT_FAILOVER_RUNTIME_STATE: FailoverRuntimeState = Object.freeze({
|
||||
lastSwitchAt: null,
|
||||
holdUntil: null,
|
||||
primaryQuarantineUntil: null,
|
||||
failoverHistory: [],
|
||||
reasonCode: null,
|
||||
});
|
||||
|
||||
function target(value: unknown): FailoverTarget {
|
||||
const candidate = record(value);
|
||||
return { profileId: text(candidate.profileId), serverId: text(candidate.serverId) };
|
||||
}
|
||||
|
||||
export function normalizeFailoverPolicy(value: unknown, { strict = false } = {}): FailoverPolicy {
|
||||
const candidate = record(value);
|
||||
const requestedChecks = Array.isArray(candidate.checks) ? candidate.checks : DEFAULT_FAILOVER_POLICY.checks;
|
||||
const checks = requestedChecks.map((value) => {
|
||||
const check = record(value);
|
||||
return {
|
||||
serviceId: text(check.serviceId).slice(0, 100),
|
||||
timeoutMs: bounded(check.timeoutMs, 6_000, 2_000, 30_000),
|
||||
};
|
||||
}).filter(({ serviceId }, index, all) => serviceId && all.findIndex((item) => item.serviceId === serviceId) === index).slice(0, 10);
|
||||
const trafficGuard = record(candidate.trafficGuard);
|
||||
const flapProtection = record(candidate.flapProtection);
|
||||
const policy: FailoverPolicy = {
|
||||
version: 1,
|
||||
enabled: candidate.enabled === true,
|
||||
paused: candidate.paused === true,
|
||||
primary: target(candidate.primary),
|
||||
reserve: target(candidate.reserve),
|
||||
checks,
|
||||
intervalMs: bounded(candidate.intervalMs, 60_000, 15_000, 900_000),
|
||||
failureWindowMs: 0,
|
||||
recoveryWindowMs: bounded(candidate.recoveryWindowMs, 900_000, 60_000, 86_400_000),
|
||||
trafficGuard: {
|
||||
enabled: trafficGuard.enabled !== false,
|
||||
thresholdBytesPerSecond: bounded(trafficGuard.thresholdBytesPerSecond, 32 * 1024, 1024, 100 * 1024 * 1024),
|
||||
quietWindowMs: bounded(trafficGuard.quietWindowMs, 30_000, 5_000, 600_000),
|
||||
},
|
||||
minimumReserveMs: bounded(candidate.minimumReserveMs, 600_000, 60_000, 86_400_000),
|
||||
flapProtection: {
|
||||
count: bounded(flapProtection.count, 3, 2, 10),
|
||||
windowMs: bounded(flapProtection.windowMs, 86_400_000, 3_600_000, 72 * 3_600_000),
|
||||
quarantineMs: bounded(flapProtection.quarantineMs, 21_600_000, 600_000, 7 * 86_400_000),
|
||||
},
|
||||
};
|
||||
policy.failureWindowMs = bounded(
|
||||
candidate.failureWindowMs,
|
||||
120_000,
|
||||
policy.intervalMs * 2,
|
||||
1_800_000,
|
||||
);
|
||||
if (strict && !equivalent(policy, value)) {
|
||||
throw new TypeError('Invalid failover policy');
|
||||
}
|
||||
return policy;
|
||||
}
|
||||
|
||||
export function normalizeFailoverRuntimeState(value: unknown): FailoverRuntimeState {
|
||||
const candidate = record(value);
|
||||
const date = (value: unknown) => typeof value === 'string' && Number.isFinite(Date.parse(value)) ? value : null;
|
||||
return {
|
||||
lastSwitchAt: date(candidate.lastSwitchAt),
|
||||
holdUntil: date(candidate.holdUntil),
|
||||
primaryQuarantineUntil: date(candidate.primaryQuarantineUntil),
|
||||
failoverHistory: (Array.isArray(candidate.failoverHistory) ? candidate.failoverHistory : [])
|
||||
.map(date).filter((item): item is string => Boolean(item)).slice(-20),
|
||||
reasonCode: text(candidate.reasonCode) || null,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeAppliedFailoverPolicy(value: unknown): AppliedFailoverPolicy | null {
|
||||
if (!value) return null;
|
||||
const candidate = record(value);
|
||||
const primary = target(candidate.primary);
|
||||
const reserve = target(candidate.reserve);
|
||||
const primaryConfigFingerprint = text(candidate.primaryConfigFingerprint);
|
||||
const reserveConfigFingerprint = text(candidate.reserveConfigFingerprint);
|
||||
return primary.profileId && primary.serverId && reserve.profileId && reserve.serverId
|
||||
&& /^[a-f0-9]{64}$/.test(primaryConfigFingerprint)
|
||||
&& /^[a-f0-9]{64}$/.test(reserveConfigFingerprint)
|
||||
? { primary, reserve, primaryConfigFingerprint, reserveConfigFingerprint }
|
||||
: null;
|
||||
}
|
||||
|
||||
export function isFailoverConfigured(policy: FailoverPolicy) {
|
||||
return Boolean(
|
||||
policy.primary.profileId && policy.primary.serverId
|
||||
&& policy.reserve.profileId && policy.reserve.serverId
|
||||
&& policy.checks.length
|
||||
&& (policy.primary.profileId !== policy.reserve.profileId
|
||||
|| policy.primary.serverId !== policy.reserve.serverId)
|
||||
);
|
||||
}
|
||||
|
||||
export function createIdleFailoverSnapshot(
|
||||
policy: FailoverPolicy,
|
||||
observationEpoch = '',
|
||||
observationSequence = 0,
|
||||
): FailoverSnapshot {
|
||||
const channel = (target: FailoverTarget) => ({
|
||||
target,
|
||||
health: 'not-monitoring' as const,
|
||||
failingServiceIds: [],
|
||||
checkedAt: null,
|
||||
stateSince: null,
|
||||
});
|
||||
return {
|
||||
observationEpoch,
|
||||
observationSequence,
|
||||
configured: isFailoverConfigured(policy),
|
||||
enabled: policy.enabled,
|
||||
paused: policy.paused,
|
||||
activation: 'inactive',
|
||||
currentRole: 'none',
|
||||
status: 'idle',
|
||||
primary: channel(policy.primary),
|
||||
reserve: channel(policy.reserve),
|
||||
nextDecisionAt: null,
|
||||
reason: null,
|
||||
trafficActivity: null,
|
||||
policy,
|
||||
};
|
||||
}
|
||||
|
||||
export function nextFailoverDecision(input: FailoverDecisionInput): FailoverDecision {
|
||||
const memory = input.memory || {
|
||||
primaryFailedSince: null,
|
||||
primaryRecoveredSince: null,
|
||||
quietSince: null,
|
||||
};
|
||||
const next = { ...memory };
|
||||
if (!input.policy.enabled || input.policy.paused) {
|
||||
return { status: 'idle', reason: input.policy.paused ? 'paused' : 'disabled', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
if (input.primaryHealth === 'unhealthy' && input.reserveHealth === 'unhealthy') {
|
||||
next.quietSince = null;
|
||||
return { status: 'blocked', reason: 'both-unhealthy', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
|
||||
let target: FailoverRole | null = null;
|
||||
let readyAt: number | null = null;
|
||||
if (input.currentRole === 'primary') {
|
||||
next.primaryRecoveredSince = null;
|
||||
if (input.primaryHealth !== 'unhealthy') {
|
||||
next.primaryFailedSince = null;
|
||||
next.quietSince = null;
|
||||
return { status: 'primary', reason: input.primaryHealth === 'healthy' ? 'primary-healthy' : 'health-unknown', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
next.primaryFailedSince ??= input.now;
|
||||
readyAt = next.primaryFailedSince + input.policy.failureWindowMs;
|
||||
if (input.now < readyAt || input.reserveHealth !== 'healthy') {
|
||||
next.quietSince = null;
|
||||
return { status: 'observing', reason: input.reserveHealth === 'healthy' ? 'failure-window' : 'reserve-not-healthy', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
||||
}
|
||||
target = 'reserve';
|
||||
} else {
|
||||
next.primaryFailedSince = null;
|
||||
if (input.primaryHealth !== 'healthy') {
|
||||
next.primaryRecoveredSince = null;
|
||||
next.quietSince = null;
|
||||
return { status: 'reserve', reason: 'primary-not-recovered', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
if ((input.primaryQuarantineUntil || 0) > input.now) {
|
||||
next.primaryRecoveredSince = null;
|
||||
next.quietSince = null;
|
||||
return {
|
||||
status: 'reserve',
|
||||
reason: 'recovery-hold',
|
||||
switchTo: null,
|
||||
memory: next,
|
||||
nextDecisionAt: input.primaryQuarantineUntil || null,
|
||||
};
|
||||
}
|
||||
next.primaryRecoveredSince ??= input.now;
|
||||
const recoveredAt = next.primaryRecoveredSince + input.policy.recoveryWindowMs;
|
||||
readyAt = input.reserveHealth === 'unhealthy'
|
||||
? recoveredAt
|
||||
: Math.max(recoveredAt, input.holdUntil || 0, input.primaryQuarantineUntil || 0);
|
||||
if (input.now < readyAt) {
|
||||
next.quietSince = null;
|
||||
return { status: 'reserve', reason: 'recovery-hold', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
||||
}
|
||||
target = 'primary';
|
||||
}
|
||||
|
||||
if (input.policy.trafficGuard.enabled) {
|
||||
if (input.activity === 'unknown') {
|
||||
next.quietSince = null;
|
||||
return { status: 'blocked', reason: 'activity-unknown', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
if (input.activity === 'active') {
|
||||
next.quietSince = null;
|
||||
return { status: 'waiting-for-idle', reason: 'active-traffic', switchTo: null, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
next.quietSince ??= input.now;
|
||||
readyAt = next.quietSince + input.policy.trafficGuard.quietWindowMs;
|
||||
if (input.now < readyAt) {
|
||||
return { status: 'waiting-for-idle', reason: 'quiet-window', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
||||
}
|
||||
}
|
||||
return { status: 'switching', reason: target === 'reserve' ? 'primary-failed' : 'primary-recovered', switchTo: target, memory: next, nextDecisionAt: null };
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
export const HARBOR_VERSIONS = Object.freeze({
|
||||
macClient: '0.27.0',
|
||||
gatewayClient: '0.28.0',
|
||||
gatewayBackend: '0.28.0',
|
||||
macClient: '0.28.0',
|
||||
gatewayClient: '0.29.0',
|
||||
gatewayBackend: '0.29.0',
|
||||
});
|
||||
|
||||
export interface ParsedVersion {
|
||||
|
||||
@@ -25,6 +25,7 @@ const componentActions = {
|
||||
setDevicePolicy: api.devices.setPolicy,
|
||||
pingServers: api.servers.ping,
|
||||
runConnectivityDiagnostics: api.diagnostics.connectivity,
|
||||
loadActivityJournal: api.activityJournal.page,
|
||||
};
|
||||
|
||||
interface UiError {
|
||||
@@ -51,6 +52,10 @@ const operationErrorContext: Record<string, string> = {
|
||||
'subscription-refresh': 'subscription',
|
||||
'subscription-forget': 'subscription',
|
||||
'route-rules': 'routing',
|
||||
'failover-save': 'failover',
|
||||
'failover-pause': 'failover',
|
||||
'failover-resume': 'failover',
|
||||
'failover-switch': 'failover',
|
||||
};
|
||||
|
||||
function asHarborApiError(error: unknown) {
|
||||
@@ -304,6 +309,26 @@ export function App() {
|
||||
() => api.diagnostics.updateSettings(settings, revisionRef.current),
|
||||
'diagnostics',
|
||||
)}
|
||||
onSaveFailover={(policy: unknown) => run(
|
||||
'failover',
|
||||
() => api.failover.save(policy, revisionRef.current),
|
||||
'failover',
|
||||
)}
|
||||
onPauseFailover={(paused: boolean) => run(
|
||||
'failover',
|
||||
() => api.failover.pause(paused, revisionRef.current),
|
||||
'failover',
|
||||
)}
|
||||
onSwitchFailover={(role: 'primary' | 'reserve') => run(
|
||||
'failover',
|
||||
() => api.failover.switch(role, revisionRef.current),
|
||||
'failover',
|
||||
)}
|
||||
onCheckFailover={() => run(
|
||||
'failover',
|
||||
() => api.failover.check(),
|
||||
'failover',
|
||||
)}
|
||||
onDismissError={() => {
|
||||
setError(null);
|
||||
setDismissedCanonicalError(canonicalErrorId);
|
||||
|
||||
@@ -192,6 +192,26 @@ export const api = {
|
||||
},
|
||||
),
|
||||
},
|
||||
failover: {
|
||||
save: (policy: unknown, expectedRevision: number) => request('/api/failover', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ policy, expectedRevision }),
|
||||
}),
|
||||
pause: (paused: boolean, expectedRevision: number) => request('/api/failover/pause', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ paused, expectedRevision }),
|
||||
}),
|
||||
switch: (role: 'primary' | 'reserve', expectedRevision: number) => request('/api/failover/switch', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ role, expectedRevision }),
|
||||
}),
|
||||
check: () => request('/api/failover/check', {
|
||||
method: 'POST',
|
||||
}),
|
||||
},
|
||||
activityJournal: {
|
||||
page: (cursor: string | null = null) => request(`/api/activity-journal?limit=50${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`),
|
||||
},
|
||||
singbox: {
|
||||
stop: () => request('/api/singbox/stop', { method: 'POST' }),
|
||||
restart: () => request('/api/singbox/restart', { method: 'POST' }),
|
||||
@@ -237,6 +257,7 @@ export function parseHarborState(value: unknown): HarborClientState {
|
||||
selection: snapshot.selection,
|
||||
connection: snapshot.connection,
|
||||
diagnostics: snapshot.diagnostics,
|
||||
failover: snapshot.failover,
|
||||
route: snapshot.route,
|
||||
operation: snapshot.operation,
|
||||
servers: snapshot.servers,
|
||||
|
||||
@@ -47,6 +47,13 @@ import {
|
||||
InstructionsToggle,
|
||||
useInstructionsFeature,
|
||||
} from '../features/instructions/index.js';
|
||||
import { FailoverPanel, FailoverToggle, useFailoverFeature } from '../features/failover/index.js';
|
||||
import {
|
||||
ActivityJournalPanel,
|
||||
ActivityJournalToggle,
|
||||
useActivityJournalFeature,
|
||||
} from '../features/activity-journal/index.js';
|
||||
import type { FailoverPolicy } from '../../shared/failover.js';
|
||||
import {
|
||||
HARBOR_VERSIONS,
|
||||
parseVersion,
|
||||
@@ -65,9 +72,33 @@ const VERSION_PARTS = [
|
||||
] as const;
|
||||
|
||||
const DRAWER_SWITCH_MS = 620;
|
||||
const DRAWER_ORDER = ['subscription', 'instructions', 'devices', 'diagnostics', 'routing'] as const;
|
||||
const DRAWER_ORDER = ['subscription', 'failover', 'instructions', 'devices', 'diagnostics', 'routing', 'journal'] as const;
|
||||
type DrawerKey = typeof DRAWER_ORDER[number];
|
||||
|
||||
const failoverReasonLabel = (reason: string | null) => ({
|
||||
'primary-healthy': 'основной работает',
|
||||
'health-unknown': 'ожидаем проверку',
|
||||
'failure-window': 'подтверждаем сбой',
|
||||
'reserve-not-healthy': 'резерв не подтверждён',
|
||||
'both-unhealthy': 'оба канала недоступны',
|
||||
'primary-not-recovered': 'основной восстанавливается',
|
||||
'recovery-hold': 'проверяем стабильность',
|
||||
'activity-unknown': 'активность неизвестна',
|
||||
'active-traffic': 'ждём завершения работы',
|
||||
'quiet-window': 'проверяем тишину',
|
||||
'primary-failed': 'основной недоступен',
|
||||
'primary-recovered': 'основной восстановился',
|
||||
'pending-activation': 'изменения ожидают запуска',
|
||||
'vpn-stopped': 'VPN выключен',
|
||||
paused: 'автоматика на паузе',
|
||||
disabled: 'резерв выключен',
|
||||
'switch-failed': 'не удалось переключить',
|
||||
'selector-unknown': 'текущий канал неизвестен',
|
||||
'reconcile-failed': 'мониторинг временно недоступен',
|
||||
'revalidation-required': 'условия проверяются заново',
|
||||
'manual-check': 'оба канала проверены',
|
||||
}[reason || ''] || 'наблюдение');
|
||||
|
||||
interface UiError {
|
||||
context?: string;
|
||||
profileId?: string;
|
||||
@@ -93,6 +124,7 @@ interface ComponentActions {
|
||||
setDevicePolicy: (id: string, mode: 'vpn' | 'direct', expectedRevision: number) => Promise<unknown>;
|
||||
pingServers: (profileId: string, ids: string[]) => Promise<unknown>;
|
||||
runConnectivityDiagnostics: (target?: unknown) => Promise<unknown>;
|
||||
loadActivityJournal: (cursor?: string | null) => Promise<unknown>;
|
||||
}
|
||||
|
||||
interface ClientViewState extends StateSnapshot {
|
||||
@@ -119,6 +151,10 @@ interface ClientOverviewPageProps {
|
||||
onSetGatewayAuto: (enabled: boolean) => Promise<unknown>;
|
||||
onSaveRouteRules: (rules: RouteRule[], expectedRevision: number) => Promise<unknown>;
|
||||
onUpdateDiagnosticsSettings: (settings: unknown) => Promise<unknown>;
|
||||
onSaveFailover: (policy: FailoverPolicy) => Promise<unknown>;
|
||||
onPauseFailover: (paused: boolean) => Promise<unknown>;
|
||||
onSwitchFailover: (role: 'primary' | 'reserve') => Promise<unknown>;
|
||||
onCheckFailover: () => Promise<unknown>;
|
||||
onDismissError: () => void;
|
||||
}
|
||||
|
||||
@@ -252,6 +288,7 @@ const operationProgress: Partial<Record<keyof OperationRegistrySnapshot, readonl
|
||||
profileRefresh: ['subscription', 'Обновляем подписку…'],
|
||||
profileDelete: ['subscription', 'Удаляем подписку…'],
|
||||
routeRules: ['routing', 'Применяем локальные правила…'],
|
||||
failover: ['failover', 'Применяем настройки резерва…'],
|
||||
};
|
||||
|
||||
const canonicalOperationKeys: Record<string, OperationKey> = {
|
||||
@@ -266,6 +303,10 @@ const canonicalOperationKeys: Record<string, OperationKey> = {
|
||||
'profile-delete': 'profileDelete',
|
||||
'gateway-auto': 'gatewayAuto',
|
||||
'route-rules': 'routeRules',
|
||||
'failover-save': 'failover',
|
||||
'failover-pause': 'failover',
|
||||
'failover-resume': 'failover',
|
||||
'failover-switch': 'failover',
|
||||
'subscription-import': 'profileAdd',
|
||||
'subscription-refresh': 'profileRefresh',
|
||||
'subscription-forget': 'profileDelete',
|
||||
@@ -425,6 +466,10 @@ export function ClientOverviewPage({
|
||||
onSetGatewayAuto,
|
||||
onSaveRouteRules,
|
||||
onUpdateDiagnosticsSettings,
|
||||
onSaveFailover,
|
||||
onPauseFailover,
|
||||
onSwitchFailover,
|
||||
onCheckFailover,
|
||||
onDismissError,
|
||||
}: ClientOverviewPageProps) {
|
||||
const isGateway = state?.mode === 'gateway';
|
||||
@@ -533,6 +578,8 @@ export function ClientOverviewPage({
|
||||
port: state?.clientRuntime?.proxyPort || (isGateway ? 8080 : 8082),
|
||||
controlHost,
|
||||
});
|
||||
const failoverFeature = useFailoverFeature();
|
||||
const activityJournalFeature = useActivityJournalFeature();
|
||||
const diagnosticsAvailable = hasSubscription;
|
||||
const drawerControls = {
|
||||
subscription: {
|
||||
@@ -541,6 +588,12 @@ export function ClientOverviewPage({
|
||||
show: subscriptionFeature.toggle,
|
||||
close: subscriptionFeature.close,
|
||||
},
|
||||
failover: {
|
||||
isOpen: failoverFeature.isOpen,
|
||||
panelRef: failoverFeature.panelRef,
|
||||
show: failoverFeature.toggle,
|
||||
close: failoverFeature.close,
|
||||
},
|
||||
instructions: {
|
||||
isOpen: instructionsFeature.isOpen,
|
||||
panelRef: instructionsFeature.panelRef,
|
||||
@@ -565,10 +618,16 @@ export function ClientOverviewPage({
|
||||
show: routingFeature.open,
|
||||
close: routingFeature.forceClose,
|
||||
},
|
||||
journal: {
|
||||
isOpen: activityJournalFeature.isOpen,
|
||||
panelRef: activityJournalFeature.panelRef,
|
||||
show: activityJournalFeature.toggle,
|
||||
close: activityJournalFeature.close,
|
||||
},
|
||||
};
|
||||
const drawerOrder = isGateway
|
||||
? DRAWER_ORDER
|
||||
: DRAWER_ORDER.filter((drawer) => drawer !== 'devices');
|
||||
: DRAWER_ORDER.filter((drawer) => !['devices', 'failover', 'journal'].includes(drawer));
|
||||
const activeRailDrawer = drawerSwitchTarget && drawerControls[drawerSwitchTarget].isOpen
|
||||
? drawerSwitchTarget
|
||||
: drawerOrder.find((drawer) => drawerControls[drawer].isOpen) || null;
|
||||
@@ -587,6 +646,8 @@ export function ClientOverviewPage({
|
||||
instructionsFeature.close();
|
||||
devicesFeature.close();
|
||||
diagnosticsFeature.close();
|
||||
failoverFeature.close();
|
||||
activityJournalFeature.close();
|
||||
}
|
||||
}, [hasSubscription, isGateway]);
|
||||
|
||||
@@ -656,6 +717,7 @@ export function ClientOverviewPage({
|
||||
routingFeature.requestClose();
|
||||
return;
|
||||
}
|
||||
if (current === 'failover' && !failoverFeature.beforeCloseRef.current()) return;
|
||||
if (!current) {
|
||||
drawerControls[target].show();
|
||||
return;
|
||||
@@ -733,6 +795,13 @@ export function ClientOverviewPage({
|
||||
: switchingServer && operationProfile && operationServer
|
||||
? `Переключаем на ${subscriptionDomain(operationProfile.subscription.host)} · ${operationServer.label}`
|
||||
: '';
|
||||
const failoverIdentity = isGateway && state.failover.enabled
|
||||
? `${state.failover.currentRole === 'reserve'
|
||||
? 'Резервный канал'
|
||||
: state.failover.currentRole === 'primary'
|
||||
? 'Основной канал'
|
||||
: 'Текущий канал вне резервной пары'} · ${failoverReasonLabel(state.failover.reason)}`
|
||||
: '';
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -752,6 +821,11 @@ export function ClientOverviewPage({
|
||||
open={activeRailDrawer === 'subscription'}
|
||||
onToggle={() => switchDrawer('subscription')}
|
||||
/>
|
||||
{isGateway && <FailoverToggle
|
||||
feature={failoverFeature}
|
||||
open={activeRailDrawer === 'failover'}
|
||||
onToggle={() => switchDrawer('failover')}
|
||||
/>}
|
||||
<InstructionsToggle
|
||||
feature={instructionsFeature}
|
||||
open={activeRailDrawer === 'instructions'}
|
||||
@@ -775,6 +849,11 @@ export function ClientOverviewPage({
|
||||
hasSubscription={hasSubscription}
|
||||
onOpen={() => switchDrawer('routing')}
|
||||
/>
|
||||
{isGateway && <ActivityJournalToggle
|
||||
feature={activityJournalFeature}
|
||||
open={activeRailDrawer === 'journal'}
|
||||
onToggle={() => switchDrawer('journal')}
|
||||
/>}
|
||||
</nav>}
|
||||
<main className={`client-panel${showPower ? '' : ' is-setup'}${!isGateway && hasSubscription ? ' has-subscription' : ''}${isGateway && hasSubscription ? ' is-gateway-home' : ''}`}>
|
||||
<ConnectionPanel
|
||||
@@ -807,7 +886,7 @@ export function ClientOverviewPage({
|
||||
blocked={connectionBlocked}
|
||||
onRestart={onRestart}
|
||||
/>}
|
||||
serverSlot={<AppliedIdentity identity={mainIdentity} operation={switchIdentity} />}
|
||||
serverSlot={<AppliedIdentity identity={mainIdentity} operation={switchIdentity || failoverIdentity} />}
|
||||
statusSlot={<>
|
||||
<InlineError error={error} context="connection" />
|
||||
<InlineProgress operations={visibleOperations} context="connection" />
|
||||
@@ -857,6 +936,22 @@ export function ClientOverviewPage({
|
||||
<InlineProgress operations={visibleOperations} context="routing" />
|
||||
</>}
|
||||
/>}
|
||||
{isGateway && hasSubscription && <FailoverPanel
|
||||
feature={failoverFeature}
|
||||
snapshot={state.failover}
|
||||
profiles={profiles}
|
||||
diagnostics={state.diagnostics}
|
||||
blocked={operationBlocked(visibleOperations, 'failover')}
|
||||
onSave={onSaveFailover}
|
||||
onPause={onPauseFailover}
|
||||
onSwitch={onSwitchFailover}
|
||||
onCheck={onCheckFailover}
|
||||
onUpdateDiagnostics={onUpdateDiagnosticsSettings}
|
||||
/>}
|
||||
{isGateway && hasSubscription && <ActivityJournalPanel
|
||||
feature={activityJournalFeature}
|
||||
loadPage={actions.loadActivityJournal}
|
||||
/>}
|
||||
<RoutingDiscardDialog feature={routingFeature} />
|
||||
<SubscriptionDeleteDialog feature={subscriptionFeature} />
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { assertActivityJournalPage, type ActivityJournalEvent } from '../../../shared/activityJournal.js';
|
||||
import { Drawer } from '../../ui/Drawer.js';
|
||||
import { RailAction } from '../../ui/RailAction.js';
|
||||
|
||||
export function useActivityJournalFeature() {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const panelRef = useRef<HTMLElement>(null);
|
||||
const toggleRef = useRef<HTMLButtonElement>(null);
|
||||
const closeRef = useRef<HTMLButtonElement>(null);
|
||||
useEffect(() => {
|
||||
if (!isOpen) return undefined;
|
||||
const frame = requestAnimationFrame(() => closeRef.current?.focus());
|
||||
const close = (event: PointerEvent | KeyboardEvent) => {
|
||||
if (event.type === 'keydown' && (event as KeyboardEvent).key !== 'Escape') return;
|
||||
if (event.type !== 'keydown' && (
|
||||
panelRef.current?.contains(event.target as Node) || toggleRef.current?.contains(event.target as Node)
|
||||
)) return;
|
||||
setIsOpen(false);
|
||||
};
|
||||
document.addEventListener('pointerdown', close);
|
||||
document.addEventListener('keydown', close);
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
document.removeEventListener('pointerdown', close);
|
||||
document.removeEventListener('keydown', close);
|
||||
requestAnimationFrame(() => {
|
||||
if (panelRef.current?.contains(document.activeElement)) toggleRef.current?.focus();
|
||||
});
|
||||
};
|
||||
}, [isOpen]);
|
||||
return { isOpen, panelRef, toggleRef, closeRef, close: () => setIsOpen(false), toggle: () => setIsOpen((value) => !value) };
|
||||
}
|
||||
|
||||
export type ActivityJournalFeature = ReturnType<typeof useActivityJournalFeature>;
|
||||
|
||||
export function ActivityJournalToggle({ feature, open, onToggle }: {
|
||||
feature: ActivityJournalFeature;
|
||||
open: boolean;
|
||||
onToggle: () => void;
|
||||
}) {
|
||||
return <RailAction
|
||||
buttonRef={feature.toggleRef}
|
||||
className="client-journal-toggle"
|
||||
open={open}
|
||||
controls="client-activity-journal"
|
||||
ariaLabel={open ? 'Закрыть журнал событий' : 'Открыть журнал событий'}
|
||||
label="Журнал"
|
||||
onClick={onToggle}
|
||||
>
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<path d="M5 8V4m0 4h4M5.6 7.1A8 8 0 1 1 4 12M12 7.5V12l3 2" />
|
||||
<circle cx="12" cy="12" r=".8" />
|
||||
</svg>
|
||||
</RailAction>;
|
||||
}
|
||||
|
||||
function eventCopy(event: ActivityJournalEvent) {
|
||||
const value = event.data;
|
||||
const copies: Record<string, [string, string]> = {
|
||||
'connection.started': ['VPN включён', [value.profileLabel, value.serverLabel].filter(Boolean).join(' · ')],
|
||||
'connection.stopped': ['VPN выключен', 'Остановлен пользователем'],
|
||||
'connection.failed': ['VPN не запущен', String(value.errorCode || '')],
|
||||
'subscription.added': ['Подписка добавлена', `${value.profileLabel || ''} · серверов: ${value.serverCount || 0}`],
|
||||
'subscription.refreshed': ['Подписка обновлена', `${value.profileLabel || ''} · серверов: ${value.serverCount || 0} · +${value.added || 0} / −${value.removed || 0}`],
|
||||
'subscription.refresh_failed': ['Подписка не обновлена', `${value.profileLabel || ''} · ${value.errorCode || ''}`],
|
||||
'subscription.deleted': ['Подписка удалена', String(value.profileLabel || '')],
|
||||
'failover.enabled': ['Резервный канал включён', 'Мониторинг начнётся после активации dual-config'],
|
||||
'failover.disabled': ['Резервный канал выключен', 'Автоматика полностью остановлена'],
|
||||
'failover.paused': ['Автопереключение на паузе', 'Проверки продолжаются'],
|
||||
'failover.resumed': ['Автопереключение возобновлено', ''],
|
||||
'failover.waiting_for_idle': ['Переключение отложено', 'Обнаружен активный трафик'],
|
||||
'failover.switched': ['Новые соединения переключены', `${value.fromRole || ''} → ${value.toRole || ''} · ${value.reason || ''}`],
|
||||
'failover.switch_failed': ['Переключение не выполнено', String(value.errorCode || '')],
|
||||
'failover.both_unhealthy': ['Оба канала недоступны', 'Текущий маршрут сохранён'],
|
||||
'failover.recovered': ['Основной канал восстановлен', String(value.reason || '')],
|
||||
'journal.recovered': ['Журнал восстановлен', 'Повреждённый файл сохранён отдельно'],
|
||||
};
|
||||
return copies[event.type] || ['Системное событие', ''];
|
||||
}
|
||||
|
||||
function dayLabel(value: string) {
|
||||
const date = new Date(value);
|
||||
const today = new Date();
|
||||
const startDate = new Date(today.getFullYear(), today.getMonth(), today.getDate());
|
||||
const start = startDate.getTime();
|
||||
const yesterday = new Date(startDate);
|
||||
yesterday.setDate(yesterday.getDate() - 1);
|
||||
const day = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
||||
if (day === start) return 'Сегодня';
|
||||
if (day === yesterday.getTime()) return 'Вчера';
|
||||
return new Intl.DateTimeFormat('ru-RU', { day: 'numeric', month: 'long' }).format(date);
|
||||
}
|
||||
|
||||
export function ActivityJournalPanel({ feature, loadPage }: {
|
||||
feature: ActivityJournalFeature;
|
||||
loadPage: (cursor?: string | null) => Promise<unknown>;
|
||||
}) {
|
||||
const [events, setEvents] = useState<ActivityJournalEvent[]>([]);
|
||||
const [nextCursor, setNextCursor] = useState<string | null>(null);
|
||||
const [status, setStatus] = useState<'idle' | 'loading' | 'ready' | 'refreshing' | 'older' | 'error'>('idle');
|
||||
const [announcement, setAnnouncement] = useState('');
|
||||
|
||||
async function load(cursor: string | null = null, refresh = false) {
|
||||
setStatus(cursor ? 'older' : refresh ? 'refreshing' : 'loading');
|
||||
try {
|
||||
const page = assertActivityJournalPage(await loadPage(cursor));
|
||||
if (page.storage.status === 'error') throw new Error('journal unavailable');
|
||||
if (refresh) {
|
||||
const known = new Set(events.map(({ id }) => id));
|
||||
setAnnouncement(`Журнал обновлён, новых событий: ${page.events.filter(({ id }) => !known.has(id)).length}`);
|
||||
}
|
||||
setEvents((current) => cursor ? [...current, ...page.events] : page.events);
|
||||
setNextCursor(page.nextCursor);
|
||||
setStatus('ready');
|
||||
} catch {
|
||||
setStatus('error');
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (feature.isOpen) void load(null, status !== 'idle');
|
||||
}, [feature.isOpen]);
|
||||
|
||||
const groups = events.reduce<Array<{ label: string; events: ActivityJournalEvent[] }>>((result, event) => {
|
||||
const label = dayLabel(event.occurredAt);
|
||||
const group = result.at(-1);
|
||||
if (group?.label === label) group.events.push(event);
|
||||
else result.push({ label, events: [event] });
|
||||
return result;
|
||||
}, []);
|
||||
|
||||
return <Drawer
|
||||
panelRef={feature.panelRef}
|
||||
closeRef={feature.closeRef}
|
||||
id="client-activity-journal"
|
||||
open={feature.isOpen}
|
||||
label="Журнал Harbor"
|
||||
closeLabel="Закрыть журнал"
|
||||
onClose={feature.close}
|
||||
className="client-journal-drawer"
|
||||
>
|
||||
<header className="client-journal-header">
|
||||
<span>Важные события хранятся 30 дней</span>
|
||||
<div><h2>Журнал</h2><button type="button" aria-label="Обновить журнал" disabled={status === 'refreshing'} onClick={() => void load(null, true)}>
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M20 11a8 8 0 1 0-2.3 6.7M20 5v6h-6" /></svg>
|
||||
</button></div>
|
||||
</header>
|
||||
<span className="client-live-region" role="status" aria-live="polite">{announcement}</span>
|
||||
{status === 'error' && <div className="client-journal-error" role="status">Журнал временно недоступен <button type="button" onClick={() => void load()}>Повторить</button></div>}
|
||||
{status === 'loading' && !events.length ? <div className="client-journal-skeleton" aria-label="Загружаем журнал">{[0, 1, 2, 3].map((value) => <span key={value} />)}</div>
|
||||
: !events.length && status === 'ready' ? <p className="client-journal-empty">За последние 30 дней важных событий пока нет</p>
|
||||
: <div className="client-journal-groups">{groups.map((group) => <section key={group.label}>
|
||||
<h3>{group.label}</h3>
|
||||
<ol>{group.events.map((event) => {
|
||||
const [title, details] = eventCopy(event);
|
||||
return <li key={event.id} className={event.severity === 'error' ? 'is-error' : event.severity === 'warning' ? 'is-warning' : ''}>
|
||||
<div className="client-journal-time"><time dateTime={event.occurredAt}>{new Intl.DateTimeFormat('ru-RU', { hour: '2-digit', minute: '2-digit' }).format(new Date(event.occurredAt))}</time><span>{event.source}</span></div>
|
||||
<div><strong>{title}</strong>{details && <span>{details}</span>}{event.severity !== 'info' && <em>{event.severity === 'error' ? 'Ошибка' : 'Внимание'}</em>}</div>
|
||||
</li>;
|
||||
})}</ol>
|
||||
</section>)}</div>}
|
||||
<footer className="client-journal-footer">
|
||||
{nextCursor ? <button type="button" disabled={status === 'older'} onClick={() => void load(nextCursor)}>{status === 'older' ? 'Загружаем…' : 'Показать ещё'}</button> : <span>{events.length ? 'Это вся история за последние 30 дней' : 'Храним события 30 дней'}</span>}
|
||||
</footer>
|
||||
</Drawer>;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { ActivityJournalPanel, ActivityJournalToggle, useActivityJournalFeature } from './ActivityJournalFeature.js';
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
type DiagnosticSiteResult,
|
||||
} from './connectivityResult.js';
|
||||
import type { DiagnosticsFeature } from './DiagnosticsFeature.js';
|
||||
import { saveCustomDiagnosticService } from './customServiceAction.js';
|
||||
|
||||
const CUSTOM_SERVICES_KEY = 'harbor-diagnostic-services';
|
||||
const HIDDEN_SERVICES_KEY = 'harbor-hidden-diagnostic-services';
|
||||
@@ -334,19 +335,15 @@ export function ConnectivityDiagnosticsPanel({
|
||||
async function addService(event: FormEvent<HTMLFormElement>) {
|
||||
event.preventDefault();
|
||||
try {
|
||||
if (customServices.length >= MAX_CUSTOM_DIAGNOSTIC_SERVICES) return;
|
||||
const parsed = new URL(serviceUrl.trim());
|
||||
if (parsed.protocol !== 'https:') throw new Error('Нужен публичный HTTPS-адрес.');
|
||||
setSettingsSaving(true);
|
||||
const saved = await updateSettings({
|
||||
customServices: [...customServices, {
|
||||
id: `custom-${globalThis.crypto?.randomUUID?.() || Date.now()}`,
|
||||
label: serviceName.trim() || parsed.hostname,
|
||||
url: parsed.href,
|
||||
}],
|
||||
const saved = await saveCustomDiagnosticService({
|
||||
name: serviceName,
|
||||
url: serviceUrl,
|
||||
customServices,
|
||||
hiddenServiceIds,
|
||||
updateSettings,
|
||||
});
|
||||
if (saved === false) throw new Error('Не удалось сохранить сервис.');
|
||||
if (!saved) return;
|
||||
setServiceName('');
|
||||
setServiceUrl('');
|
||||
setFormError('');
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
MAX_CUSTOM_DIAGNOSTIC_SERVICES,
|
||||
type DiagnosticService,
|
||||
type DiagnosticSettings,
|
||||
} from '../../../shared/connectivityDiagnostics.js';
|
||||
|
||||
export async function saveCustomDiagnosticService({
|
||||
name,
|
||||
url,
|
||||
customServices,
|
||||
hiddenServiceIds,
|
||||
updateSettings,
|
||||
}: {
|
||||
name: string;
|
||||
url: string;
|
||||
customServices: DiagnosticService[];
|
||||
hiddenServiceIds: string[];
|
||||
updateSettings: (settings: Pick<DiagnosticSettings, 'customServices' | 'hiddenServiceIds'>) => Promise<unknown>;
|
||||
}) {
|
||||
if (customServices.length >= MAX_CUSTOM_DIAGNOSTIC_SERVICES) return false;
|
||||
const parsed = new URL(url.trim());
|
||||
if (parsed.protocol !== 'https:') throw new Error('Нужен публичный HTTPS-адрес.');
|
||||
const saved = await updateSettings({
|
||||
customServices: [...customServices, {
|
||||
id: `custom-${globalThis.crypto?.randomUUID?.() || Date.now()}`,
|
||||
label: name.trim() || parsed.hostname,
|
||||
url: parsed.href,
|
||||
}],
|
||||
hiddenServiceIds,
|
||||
});
|
||||
if (saved === false) throw new Error('Не удалось сохранить сервис.');
|
||||
return true;
|
||||
}
|
||||
@@ -4,3 +4,4 @@ export {
|
||||
useDiagnosticsFeature,
|
||||
type DiagnosticsFeature,
|
||||
} from './DiagnosticsFeature.js';
|
||||
export { saveCustomDiagnosticService } from './customServiceAction.js';
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { CONNECTIVITY_SITES, MAX_CUSTOM_DIAGNOSTIC_SERVICES, type DiagnosticSettings } from '../../../shared/connectivityDiagnostics.js';
|
||||
import { normalizeFailoverPolicy, type FailoverPolicy, type FailoverSnapshot } from '../../../shared/failover.js';
|
||||
import type { ProfileSnapshot } from '../../../shared/contracts/state.js';
|
||||
import { Drawer } from '../../ui/Drawer.js';
|
||||
import { RailAction } from '../../ui/RailAction.js';
|
||||
import { saveCustomDiagnosticService } from '../diagnostics/index.js';
|
||||
|
||||
export function useFailoverFeature() {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const panelRef = useRef<HTMLElement>(null);
|
||||
const toggleRef = useRef<HTMLButtonElement>(null);
|
||||
const closeRef = useRef<HTMLButtonElement>(null);
|
||||
const beforeCloseRef = useRef<() => boolean>(() => true);
|
||||
const close = () => {
|
||||
if (beforeCloseRef.current()) setIsOpen(false);
|
||||
};
|
||||
useEffect(() => {
|
||||
if (!isOpen) return undefined;
|
||||
const frame = requestAnimationFrame(() => closeRef.current?.focus());
|
||||
const handleClose = (event: PointerEvent | KeyboardEvent) => {
|
||||
if (event.type === 'keydown' && (event as KeyboardEvent).key !== 'Escape') return;
|
||||
if (event.type !== 'keydown' && (
|
||||
panelRef.current?.contains(event.target as Node) || toggleRef.current?.contains(event.target as Node)
|
||||
)) return;
|
||||
close();
|
||||
};
|
||||
document.addEventListener('pointerdown', handleClose);
|
||||
document.addEventListener('keydown', handleClose);
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
document.removeEventListener('pointerdown', handleClose);
|
||||
document.removeEventListener('keydown', handleClose);
|
||||
requestAnimationFrame(() => {
|
||||
if (panelRef.current?.contains(document.activeElement)) toggleRef.current?.focus();
|
||||
});
|
||||
};
|
||||
}, [isOpen]);
|
||||
return {
|
||||
isOpen, panelRef, toggleRef, closeRef, beforeCloseRef, close,
|
||||
toggle: () => isOpen ? close() : setIsOpen(true),
|
||||
};
|
||||
}
|
||||
|
||||
export type FailoverFeature = ReturnType<typeof useFailoverFeature>;
|
||||
|
||||
export function FailoverToggle({ feature, open, onToggle }: {
|
||||
feature: FailoverFeature;
|
||||
open: boolean;
|
||||
onToggle: () => void;
|
||||
}) {
|
||||
return <RailAction
|
||||
buttonRef={feature.toggleRef}
|
||||
className="client-failover-toggle"
|
||||
open={open}
|
||||
controls="client-failover"
|
||||
ariaLabel={open ? 'Закрыть резервный канал' : 'Настроить резервный канал'}
|
||||
label="Резерв"
|
||||
onClick={onToggle}
|
||||
>
|
||||
<svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<path d="M5 7h11m-3-3 3 3-3 3M19 17H8m3-3-3 3 3 3" />
|
||||
<circle cx="4" cy="7" r="1" /><circle cx="20" cy="17" r="1" />
|
||||
</svg>
|
||||
</RailAction>;
|
||||
}
|
||||
|
||||
const seconds = (milliseconds: number) => Math.round(milliseconds / 1000);
|
||||
const milliseconds = (value: string, fallback: number) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? Math.round(parsed * 1000) : fallback;
|
||||
};
|
||||
|
||||
const reasonLabel = (reason: string | null) => ({
|
||||
'primary-healthy': 'Основной канал работает',
|
||||
'health-unknown': 'Ожидаем результаты проверки',
|
||||
'failure-window': 'Подтверждаем сбой основного канала',
|
||||
'reserve-not-healthy': 'Резервный канал ещё не подтверждён',
|
||||
'both-unhealthy': 'Оба канала недоступны',
|
||||
'primary-not-recovered': 'Основной канал восстанавливается',
|
||||
'recovery-hold': 'Проверяем стабильность основного канала',
|
||||
'activity-unknown': 'Не удалось определить активность',
|
||||
'active-traffic': 'Ждём завершения активной работы',
|
||||
'quiet-window': 'Проверяем тишину перед переключением',
|
||||
'primary-failed': 'Основной канал недоступен',
|
||||
'primary-recovered': 'Основной канал восстановился',
|
||||
'pending-activation': 'Изменения ожидают следующего запуска VPN',
|
||||
'vpn-stopped': 'VPN выключен',
|
||||
paused: 'Автоматика на паузе',
|
||||
disabled: 'Резерв выключен',
|
||||
'switch-failed': 'Не удалось переключить канал',
|
||||
'selector-unknown': 'Не удалось подтвердить текущий канал',
|
||||
'reconcile-failed': 'Настройки сохранены, мониторинг временно недоступен',
|
||||
'revalidation-required': 'Условия переключения проверяются заново',
|
||||
'manual-check': 'Оба канала проверены',
|
||||
}[reason || ''] || 'Наблюдаем за каналами');
|
||||
|
||||
function targetLabel(profiles: ProfileSnapshot[], target: { profileId: string; serverId: string }) {
|
||||
const profile = profiles.find(({ id }) => id === target.profileId);
|
||||
const server = profile?.servers.find(({ id }) => id === target.serverId);
|
||||
return profile && server ? `${profile.label} · ${server.label}` : 'Не выбран';
|
||||
}
|
||||
|
||||
export function FailoverPanel({
|
||||
feature,
|
||||
snapshot,
|
||||
profiles,
|
||||
diagnostics,
|
||||
blocked,
|
||||
onSave,
|
||||
onPause,
|
||||
onSwitch,
|
||||
onCheck,
|
||||
onUpdateDiagnostics,
|
||||
}: {
|
||||
feature: FailoverFeature;
|
||||
snapshot: FailoverSnapshot;
|
||||
profiles: ProfileSnapshot[];
|
||||
diagnostics: DiagnosticSettings;
|
||||
blocked: boolean;
|
||||
onSave: (policy: FailoverPolicy) => Promise<unknown>;
|
||||
onPause: (paused: boolean) => Promise<unknown>;
|
||||
onSwitch: (role: 'primary' | 'reserve') => Promise<unknown>;
|
||||
onCheck: () => Promise<unknown>;
|
||||
onUpdateDiagnostics: (settings: unknown) => Promise<unknown>;
|
||||
}) {
|
||||
const [draft, setDraft] = useState(() => snapshot.policy);
|
||||
const [dirty, setDirty] = useState(false);
|
||||
const [confirmDiscard, setConfirmDiscard] = useState(false);
|
||||
const [addingService, setAddingService] = useState(false);
|
||||
const [serviceName, setServiceName] = useState('');
|
||||
const [serviceUrl, setServiceUrl] = useState('');
|
||||
const [serviceError, setServiceError] = useState('');
|
||||
useEffect(() => {
|
||||
if (!dirty) setDraft(snapshot.policy);
|
||||
}, [snapshot.policy, dirty]);
|
||||
useEffect(() => {
|
||||
feature.beforeCloseRef.current = () => {
|
||||
if (!dirty) return true;
|
||||
setConfirmDiscard(true);
|
||||
return false;
|
||||
};
|
||||
const beforeUnload = (event: BeforeUnloadEvent) => {
|
||||
if (!dirty) return;
|
||||
event.preventDefault();
|
||||
};
|
||||
window.addEventListener('beforeunload', beforeUnload);
|
||||
return () => {
|
||||
feature.beforeCloseRef.current = () => true;
|
||||
window.removeEventListener('beforeunload', beforeUnload);
|
||||
};
|
||||
}, [dirty, feature.beforeCloseRef]);
|
||||
const services = useMemo(() => [
|
||||
...CONNECTIVITY_SITES,
|
||||
...diagnostics.customServices,
|
||||
], [diagnostics.customServices]);
|
||||
const update = (value: Partial<FailoverPolicy>) => {
|
||||
setDraft((current) => normalizeFailoverPolicy({ ...current, ...value }));
|
||||
setDirty(true);
|
||||
};
|
||||
const updateTarget = (role: 'primary' | 'reserve', patch: Partial<FailoverPolicy[typeof role]>) => {
|
||||
const next = { ...draft[role], ...patch };
|
||||
const profile = profiles.find(({ id }) => id === next.profileId);
|
||||
if (patch.profileId !== undefined) next.serverId = profile?.desiredServerId || profile?.servers[0]?.id || '';
|
||||
update({ [role]: next } as Partial<FailoverPolicy>);
|
||||
};
|
||||
const role = snapshot.currentRole === 'primary' || snapshot.currentRole === 'reserve'
|
||||
? snapshot.currentRole
|
||||
: null;
|
||||
const switchRole = role === 'primary' ? 'reserve' : role === 'reserve' ? 'primary' : null;
|
||||
const activity = snapshot.trafficActivity;
|
||||
const targetExists = (target: FailoverPolicy['primary']) => profiles
|
||||
.find(({ id }) => id === target.profileId)?.servers.some(({ id }) => id === target.serverId);
|
||||
const targetsValid = targetExists(draft.primary) && targetExists(draft.reserve)
|
||||
&& (draft.primary.profileId !== draft.reserve.profileId || draft.primary.serverId !== draft.reserve.serverId);
|
||||
const draftValid = Boolean(targetsValid && draft.checks.length);
|
||||
const quietElapsed = activity?.quietSince
|
||||
? Math.max(0, Date.now() - Date.parse(activity.quietSince))
|
||||
: 0;
|
||||
const addService = async () => {
|
||||
try {
|
||||
const saved = await saveCustomDiagnosticService({
|
||||
name: serviceName,
|
||||
url: serviceUrl,
|
||||
customServices: diagnostics.customServices,
|
||||
hiddenServiceIds: diagnostics.hiddenServiceIds,
|
||||
updateSettings: onUpdateDiagnostics,
|
||||
});
|
||||
if (!saved) return;
|
||||
setServiceName('');
|
||||
setServiceUrl('');
|
||||
setServiceError('');
|
||||
setAddingService(false);
|
||||
} catch (error) {
|
||||
setServiceError(error instanceof Error ? error.message : 'Проверьте адрес.');
|
||||
}
|
||||
};
|
||||
|
||||
return <Drawer
|
||||
panelRef={feature.panelRef}
|
||||
closeRef={feature.closeRef}
|
||||
id="client-failover"
|
||||
open={feature.isOpen}
|
||||
label="Резервный канал"
|
||||
closeLabel="Закрыть резервный канал"
|
||||
onClose={feature.close}
|
||||
className="client-failover-drawer"
|
||||
>
|
||||
<header className="client-failover-header">
|
||||
<span>Gateway</span>
|
||||
<h2>Резервный канал</h2>
|
||||
<p>Переключает только новые соединения. Уже открытые соединения Harbor не закрывает.</p>
|
||||
</header>
|
||||
|
||||
<form className="client-failover-form" onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
void onSave(draft).then((result) => { if (result !== false) setDirty(false); });
|
||||
}}>
|
||||
<label className="client-failover-master">
|
||||
<span><strong>Использовать резерв</strong><small>{draft.enabled ? 'Мониторинг включён' : 'Полностью пассивен'}</small></span>
|
||||
<input type="checkbox" checked={draft.enabled} onChange={(event) => update({ enabled: event.target.checked })} />
|
||||
</label>
|
||||
|
||||
{(['primary', 'reserve'] as const).map((channel) => {
|
||||
const profile = profiles.find(({ id }) => id === draft[channel].profileId);
|
||||
const health = snapshot[channel].health;
|
||||
const missing = Boolean(snapshot[channel].target.profileId) && !targetExists(snapshot[channel].target);
|
||||
const healthLabel = missing
|
||||
? 'Цель недоступна'
|
||||
: channel === 'primary' && snapshot.reason === 'failure-window'
|
||||
? `Нестабилен · ${seconds(Math.max(0, Date.now() - Date.parse(snapshot.primary.stateSince || new Date().toISOString())))} из ${seconds(draft.failureWindowMs)} с`
|
||||
: channel === 'primary' && snapshot.currentRole === 'reserve' && health === 'healthy' && snapshot.reason === 'recovery-hold'
|
||||
? 'Восстанавливается'
|
||||
: health === 'healthy' ? 'Работает' : health === 'unhealthy' ? 'Недоступен' : health === 'not-monitoring' ? 'Не проверяется' : 'Нет данных';
|
||||
return <section className="client-failover-channel" key={channel}>
|
||||
<div className="client-failover-channel-title">
|
||||
<span>{channel === 'primary' ? 'Основной' : 'Резервный'}</span>
|
||||
<strong className={health === 'healthy' ? 'is-healthy' : health === 'unhealthy' || missing ? 'is-unhealthy' : ''}>{healthLabel}</strong>
|
||||
</div>
|
||||
<label><span>Подписка</span><select value={draft[channel].profileId} onChange={(event) => updateTarget(channel, { profileId: event.target.value })}>
|
||||
<option value="">Не выбрана</option>
|
||||
{profiles.map((item) => <option value={item.id} key={item.id}>{item.label}</option>)}
|
||||
</select></label>
|
||||
<label><span>Сервер</span><select value={draft[channel].serverId} onChange={(event) => updateTarget(channel, { serverId: event.target.value })} disabled={!profile}>
|
||||
<option value="">Не выбран</option>
|
||||
{profile?.servers.map((server) => <option value={server.id} key={server.id}>{server.label}</option>)}
|
||||
</select></label>
|
||||
</section>;
|
||||
})}
|
||||
|
||||
<fieldset className="client-failover-services">
|
||||
<legend>Что проверять</legend>
|
||||
{services.map((service) => {
|
||||
const check = draft.checks.find(({ serviceId }) => serviceId === service.id);
|
||||
return <div className="client-failover-service" key={service.id}>
|
||||
<label>
|
||||
<input type="checkbox" checked={Boolean(check)} onChange={(event) => update({
|
||||
checks: event.target.checked
|
||||
? [...draft.checks, { serviceId: service.id, timeoutMs: 6_000 }]
|
||||
: draft.checks.filter(({ serviceId }) => serviceId !== service.id),
|
||||
})} />
|
||||
<span>{service.label}</span>
|
||||
</label>
|
||||
{check && <label className="client-failover-service-timeout">
|
||||
<span>таймаут, сек</span>
|
||||
<input
|
||||
type="number"
|
||||
min="2"
|
||||
max="30"
|
||||
aria-label={`Таймаут проверки: ${service.label}`}
|
||||
value={seconds(check.timeoutMs)}
|
||||
onChange={(event) => update({ checks: draft.checks.map((item) => item.serviceId === service.id
|
||||
? { ...item, timeoutMs: milliseconds(event.target.value, item.timeoutMs) }
|
||||
: item) })}
|
||||
/>
|
||||
</label>}
|
||||
</div>;
|
||||
})}
|
||||
{!addingService && <button type="button" className="client-failover-add-service" disabled={diagnostics.customServices.length >= MAX_CUSTOM_DIAGNOSTIC_SERVICES} onClick={() => setAddingService(true)}>+ Добавить HTTPS-сервис</button>}
|
||||
{addingService && <div className="client-failover-service-editor">
|
||||
<input aria-label="Название HTTPS-сервиса" placeholder="Название" value={serviceName} onChange={(event) => setServiceName(event.target.value)} />
|
||||
<input aria-label="HTTPS-адрес сервиса" placeholder="https://example.com/health" value={serviceUrl} onChange={(event) => setServiceUrl(event.target.value)} />
|
||||
{serviceError && <span role="alert">{serviceError}</span>}
|
||||
<button type="button" onClick={() => void addService()}>Добавить</button>
|
||||
<button type="button" onClick={() => { setAddingService(false); setServiceError(''); }}>Отмена</button>
|
||||
</div>}
|
||||
</fieldset>
|
||||
|
||||
<section className="client-failover-timing" aria-label="Пороги переключения">
|
||||
<label><span>Проверять каждые, сек</span><input type="number" min="15" max="900" value={seconds(draft.intervalMs)} onChange={(event) => update({ intervalMs: milliseconds(event.target.value, draft.intervalMs) })} /></label>
|
||||
<label><span>Сбой должен длиться, сек</span><input type="number" min={seconds(draft.intervalMs * 2)} max="1800" value={seconds(draft.failureWindowMs)} onChange={(event) => update({ failureWindowMs: milliseconds(event.target.value, draft.failureWindowMs) })} /></label>
|
||||
<label><span>Восстановление, сек</span><input type="number" min="60" max="86400" value={seconds(draft.recoveryWindowMs)} onChange={(event) => update({ recoveryWindowMs: milliseconds(event.target.value, draft.recoveryWindowMs) })} /></label>
|
||||
<label><span>Не переключать во время работы</span><input type="checkbox" checked={draft.trafficGuard.enabled} onChange={(event) => update({ trafficGuard: { ...draft.trafficGuard, enabled: event.target.checked } })} /></label>
|
||||
<label><span>Тишина перед переключением, сек</span><input type="number" min="5" max="600" value={seconds(draft.trafficGuard.quietWindowMs)} onChange={(event) => update({ trafficGuard: { ...draft.trafficGuard, quietWindowMs: milliseconds(event.target.value, draft.trafficGuard.quietWindowMs) } })} /></label>
|
||||
<label><span>Активный трафик, КБ/с</span><input type="number" min="1" max="102400" value={Math.round(draft.trafficGuard.thresholdBytesPerSecond / 1024)} onChange={(event) => update({ trafficGuard: { ...draft.trafficGuard, thresholdBytesPerSecond: Number(event.target.value) * 1024 } })} /></label>
|
||||
</section>
|
||||
|
||||
<details className="client-failover-advanced"><summary>Защита от повторных сбоев</summary>
|
||||
<label><span>Минимум на резерве, сек</span><input type="number" min="60" max="86400" value={seconds(draft.minimumReserveMs)} onChange={(event) => update({ minimumReserveMs: milliseconds(event.target.value, draft.minimumReserveMs) })} /></label>
|
||||
<label><span>Падений до карантина</span><input type="number" min="2" max="10" value={draft.flapProtection.count} onChange={(event) => update({ flapProtection: { ...draft.flapProtection, count: Number(event.target.value) } })} /></label>
|
||||
<label><span>Окно повторных сбоев, ч</span><input type="number" min="1" max="72" value={Math.round(draft.flapProtection.windowMs / 3_600_000)} onChange={(event) => update({ flapProtection: { ...draft.flapProtection, windowMs: Number(event.target.value) * 3_600_000 } })} /></label>
|
||||
<label><span>Карантин основного, мин</span><input type="number" min="10" max="10080" value={Math.round(draft.flapProtection.quarantineMs / 60_000)} onChange={(event) => update({ flapProtection: { ...draft.flapProtection, quarantineMs: Number(event.target.value) * 60_000 } })} /></label>
|
||||
</details>
|
||||
|
||||
<div className="client-failover-runtime">
|
||||
<strong role="status" aria-live="polite">{blocked ? 'Harbor выполняет действие…' : snapshot.activation === 'pending' ? 'Включится при следующем запуске VPN' : reasonLabel(snapshot.reason)}</strong>
|
||||
<span>Новые соединения: {role ? targetLabel(profiles, snapshot[role].target) : 'текущий канал вне настроенной пары'}</span>
|
||||
{snapshot.nextDecisionAt && <span>Следующее решение не раньше чем через {seconds(Math.max(0, Date.parse(snapshot.nextDecisionAt) - Date.now()))} с</span>}
|
||||
{snapshot.currentRole === 'other' && <span>Чтобы запустить автоматику, выключите VPN и включите снова на основном канале.</span>}
|
||||
{activity && <span>{activity.state === 'active' ? `Активный трафик · ${Math.round(activity.totalBytesPerSecond / 1024)} КБ/с · соединений: ${activity.transmittingConnections}` : activity.state === 'quiet' ? `Проверяем тишину · ${Math.min(seconds(quietElapsed), seconds(draft.trafficGuard.quietWindowMs))} из ${seconds(draft.trafficGuard.quietWindowMs)} с` : 'Не удалось определить активность · автоматическое переключение остановлено'}</span>}
|
||||
{activity?.blockers.slice(0, 2).map((blocker) => <small key={`${blocker.device}:${blocker.service}`}>{blocker.device} · {blocker.service} · {Math.round((blocker.uploadBytesPerSecond + blocker.downloadBytesPerSecond) / 1024)} КБ/с</small>)}
|
||||
{activity && activity.blockers.length > 2 && <small>Ещё {activity.blockers.length - 2}</small>}
|
||||
</div>
|
||||
|
||||
{!draftValid && <p className="client-failover-validation">Выберите два разных сервера. Они могут быть из одной подписки.</p>}
|
||||
{confirmDiscard && <div className="client-failover-discard" role="alert">
|
||||
<span>Отменить несохранённые изменения?</span>
|
||||
<button type="button" onClick={() => { setDraft(snapshot.policy); setDirty(false); setConfirmDiscard(false); feature.beforeCloseRef.current = () => true; feature.close(); }}>Отменить изменения</button>
|
||||
<button type="button" onClick={() => setConfirmDiscard(false)}>Продолжить настройку</button>
|
||||
</div>}
|
||||
|
||||
<div className="client-failover-actions">
|
||||
<button type="button" disabled={blocked || !snapshot.enabled || snapshot.activation !== 'active'} onClick={() => void onCheck()}>Проверить оба канала</button>
|
||||
<button type="submit" disabled={blocked || !dirty || !draftValid}>Сохранить</button>
|
||||
{snapshot.enabled && <button type="button" disabled={blocked} onClick={() => void onPause(!snapshot.paused)}>{snapshot.paused ? 'Возобновить' : 'Пауза'}</button>}
|
||||
{switchRole && snapshot.enabled && snapshot.activation === 'active' && <button type="button" disabled={blocked} aria-label={`Переключить новые соединения на ${switchRole === 'reserve' ? 'резервный' : 'основной'} сейчас`} onClick={() => void onSwitch(switchRole)}>Переключить новые сейчас</button>}
|
||||
{(snapshot.currentRole === 'primary' || snapshot.currentRole === 'reserve') && <small>Текущие соединения Harbor не закроет</small>}
|
||||
</div>
|
||||
</form>
|
||||
</Drawer>;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { FailoverPanel, FailoverToggle, useFailoverFeature } from './FailoverFeature.js';
|
||||
@@ -4,6 +4,10 @@ export type SyncErrorKind = 'incompatible-api' | 'control-unreachable' | 'fatal'
|
||||
|
||||
export interface HarborReducerState {
|
||||
snapshot: HarborClientState | null;
|
||||
failoverTransport: {
|
||||
activeEpoch: string;
|
||||
retiredEpochs: string[];
|
||||
};
|
||||
transport: {
|
||||
bootStatus: 'loading' | 'ready' | SyncErrorKind;
|
||||
lastSuccessfulSyncAt: string | null;
|
||||
@@ -20,6 +24,7 @@ export type HarborAction =
|
||||
|
||||
export const initialHarborState: HarborReducerState = {
|
||||
snapshot: null,
|
||||
failoverTransport: { activeEpoch: '', retiredEpochs: [] },
|
||||
transport: {
|
||||
bootStatus: 'loading',
|
||||
lastSuccessfulSyncAt: null,
|
||||
@@ -31,6 +36,14 @@ export const initialHarborState: HarborReducerState = {
|
||||
|
||||
export const STALE_FAILURE_THRESHOLD = 3;
|
||||
|
||||
function failoverEpoch(snapshot: HarborClientState | null) {
|
||||
return snapshot?.failover?.observationEpoch || '';
|
||||
}
|
||||
|
||||
function failoverSequence(snapshot: HarborClientState | null) {
|
||||
return snapshot?.failover?.observationSequence || 0;
|
||||
}
|
||||
|
||||
export function classifySyncError(error: unknown): SyncErrorKind {
|
||||
const candidate = error && typeof error === 'object' ? error as Record<string, unknown> : {};
|
||||
const status = Number(candidate.status) || 0;
|
||||
@@ -71,9 +84,29 @@ export function harborReducer(current: HarborReducerState, action: HarborAction)
|
||||
|
||||
const snapshot = action.snapshot;
|
||||
const newer = !current.snapshot || snapshot.revision > current.snapshot.revision;
|
||||
const equal = Boolean(current.snapshot) && snapshot.revision === current.snapshot?.revision;
|
||||
const incomingEpoch = failoverEpoch(snapshot);
|
||||
const activeEpoch = current.failoverTransport.activeEpoch || failoverEpoch(current.snapshot);
|
||||
const unseenEpoch = Boolean(incomingEpoch)
|
||||
&& incomingEpoch !== activeEpoch
|
||||
&& !current.failoverTransport.retiredEpochs.includes(incomingEpoch);
|
||||
const newerObservation = equal && Boolean(snapshot.failover) && (
|
||||
unseenEpoch
|
||||
|| (incomingEpoch === activeEpoch && failoverSequence(snapshot) > failoverSequence(current.snapshot))
|
||||
);
|
||||
const nextSnapshot = newer
|
||||
? snapshot
|
||||
: newerObservation && current.snapshot
|
||||
? { ...current.snapshot, failover: snapshot.failover }
|
||||
: current.snapshot;
|
||||
const nextActiveEpoch = newer || unseenEpoch ? incomingEpoch : activeEpoch;
|
||||
const retiredEpochs = unseenEpoch && activeEpoch
|
||||
? [...current.failoverTransport.retiredEpochs, activeEpoch]
|
||||
: current.failoverTransport.retiredEpochs;
|
||||
|
||||
return {
|
||||
snapshot: newer ? snapshot : current.snapshot,
|
||||
snapshot: nextSnapshot,
|
||||
failoverTransport: { activeEpoch: nextActiveEpoch, retiredEpochs },
|
||||
transport: {
|
||||
bootStatus: 'ready',
|
||||
lastSuccessfulSyncAt: action.receivedAt,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export type OperationKey = 'connection' | 'serverApply' | 'profileAdd' | 'profileRename'
|
||||
| 'profileSelect' | 'profileActivate' | 'profileRefresh' | 'profileDelete'
|
||||
| 'gatewayAuto' | 'routeRules' | 'diagnosticsSettings';
|
||||
| 'gatewayAuto' | 'routeRules' | 'diagnosticsSettings' | 'failover';
|
||||
|
||||
export interface OperationState {
|
||||
status: 'running';
|
||||
@@ -22,6 +22,7 @@ const OPERATION_KEYS: readonly OperationKey[] = [
|
||||
'gatewayAuto',
|
||||
'routeRules',
|
||||
'diagnosticsSettings',
|
||||
'failover',
|
||||
];
|
||||
|
||||
// The backend has one canonical revision and one mutation queue, so the UI mirrors that lock.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
.client-journal-header { display: grid; gap: 7px; margin: 0 8px 30px; }
|
||||
.client-journal-header > span, .client-journal-groups h3 { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-journal-header > div { display: flex; align-items: center; gap: 12px; }
|
||||
.client-journal-header h2 { margin: 0; font: var(--type-drawer-title); letter-spacing: var(--type-drawer-title-tracking); text-transform: var(--type-drawer-title-transform); }
|
||||
.client-journal-header button { width: 34px; height: 34px; display: grid; place-items: center; border: 0; background: transparent; color: var(--client-muted); cursor: pointer; }
|
||||
.client-journal-header svg { width: 17px; fill: none; stroke: currentColor; stroke-width: 1.7; }
|
||||
.client-journal-header button:hover, .client-journal-header button:focus-visible { color: var(--client-accent); outline: 0; filter: drop-shadow(0 0 7px var(--client-accent)); }
|
||||
.client-journal-header button:disabled svg { animation: client-spin 900ms linear infinite; }
|
||||
.client-journal-groups { display: grid; gap: 26px; margin: 0 8px; }
|
||||
.client-journal-groups section { display: grid; gap: 7px; }
|
||||
.client-journal-groups h3 { margin: 0; }
|
||||
.client-journal-groups ol { display: grid; margin: 0; padding: 0; }
|
||||
.client-journal-groups li { min-height: 58px; display: grid; grid-template-columns: 52px minmax(0, 1fr); gap: 14px; align-items: start; padding: 10px 0; border-top: 1px solid color-mix(in oklch, var(--client-border) 48%, transparent); }
|
||||
.client-journal-time, .client-journal-groups li > div:last-child { min-width: 0; display: grid; gap: 3px; }
|
||||
.client-journal-time time { color: var(--client-text); font: var(--type-data); font-variant-numeric: var(--numeric-tabular); letter-spacing: var(--type-data-tracking); text-transform: var(--type-data-transform); }
|
||||
.client-journal-time span, .client-journal-groups li span { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); overflow-wrap: anywhere; }
|
||||
.client-journal-time span { text-transform: var(--type-label-transform); }
|
||||
.client-journal-groups li strong { font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-journal-groups li em { width: fit-content; color: oklch(0.68 0.15 28); font: var(--type-label); font-style: normal; letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-journal-error, .client-journal-empty { min-height: 80px; margin: 0 8px; color: var(--client-muted); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-journal-error button, .client-journal-footer button { border: 0; background: transparent; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-journal-skeleton { display: grid; gap: 14px; margin: 0 8px; }
|
||||
.client-journal-skeleton span { height: 48px; background: color-mix(in oklch, var(--client-border) 24%, transparent); opacity: .55; }
|
||||
.client-journal-footer { min-height: 72px; display: grid; place-items: center; margin: 12px 8px 0; color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); text-align: center; }
|
||||
.client-journal-footer button:focus-visible, .client-journal-error button:focus-visible { outline: 2px solid var(--client-accent); outline-offset: 2px; }
|
||||
@media (max-width: 560px) { .client-journal-groups li { grid-template-columns: 1fr; gap: 5px; } .client-journal-time { display: flex; gap: 8px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .client-journal-header button:disabled svg { animation: none; } }
|
||||
@@ -0,0 +1,45 @@
|
||||
.client-failover-header { display: grid; gap: 8px; margin: 0 8px 30px; }
|
||||
.client-failover-header > span, .client-failover-channel-title > span, .client-failover-services legend { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-header h2 { margin: 0; font: var(--type-drawer-title); letter-spacing: var(--type-drawer-title-tracking); text-transform: var(--type-drawer-title-transform); }
|
||||
.client-failover-header p { margin: 0; color: var(--client-muted); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-form { display: grid; gap: 24px; margin: 0 8px; }
|
||||
.client-failover-master { min-height: 54px; display: flex; align-items: center; justify-content: space-between; gap: 20px; border-bottom: 1px solid color-mix(in oklch, var(--client-border) 48%, transparent); }
|
||||
.client-failover-master > span { display: grid; gap: 3px; }
|
||||
.client-failover-master strong { font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-master small { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-master input { width: 42px; height: 22px; }
|
||||
.client-failover-channel { display: grid; gap: 10px; }
|
||||
.client-failover-channel-title { min-height: 28px; display: flex; align-items: center; justify-content: space-between; gap: 12px; }
|
||||
.client-failover-channel-title strong { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); transition: color 600ms ease, filter 600ms ease; }
|
||||
.client-failover-channel-title strong.is-healthy { color: var(--client-accent); }
|
||||
.client-failover-channel-title strong.is-unhealthy { color: oklch(0.68 0.15 28); }
|
||||
.client-failover-channel label, .client-failover-timing label, .client-failover-advanced label { display: grid; grid-template-columns: minmax(0, 1fr) minmax(150px, 48%); align-items: center; gap: 16px; min-height: 38px; color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-channel select, .client-failover-timing input, .client-failover-advanced input { min-width: 0; min-height: 34px; border: 0; border-bottom: 1px solid var(--client-border); background: transparent; color: var(--client-text); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-services { display: grid; gap: 9px; margin: 0; padding: 0; border: 0; }
|
||||
.client-failover-services legend { margin-bottom: 10px; }
|
||||
.client-failover-service { display: grid; grid-template-columns: minmax(0, 1fr) auto; align-items: center; gap: 12px; min-height: 34px; }
|
||||
.client-failover-service > label { display: flex; align-items: center; gap: 10px; font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-service > .client-failover-service-timeout { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-service-timeout input { width: 52px; min-height: 30px; border: 0; border-bottom: 1px solid var(--client-border); background: transparent; color: var(--client-text); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-add-service { justify-self: start; padding: 7px 0; border: 0; background: transparent; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-failover-add-service:disabled { color: var(--client-muted); cursor: default; opacity: .45; }
|
||||
.client-failover-service-editor { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.6fr); gap: 8px 12px; padding: 8px 0; }
|
||||
.client-failover-service-editor input { min-width: 0; min-height: 34px; border: 0; border-bottom: 1px solid var(--client-border); background: transparent; color: var(--client-text); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-service-editor span { grid-column: 1 / -1; color: oklch(0.68 0.15 28); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-service-editor button { justify-self: start; padding: 4px 0; border: 0; background: transparent; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-failover-timing { display: grid; gap: 5px; }
|
||||
.client-failover-advanced summary { min-height: 38px; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-failover-runtime { min-height: 132px; display: grid; align-content: center; gap: 5px; padding: 14px 0; border-top: 1px solid color-mix(in oklch, var(--client-border) 48%, transparent); border-bottom: 1px solid color-mix(in oklch, var(--client-border) 48%, transparent); font-variant-numeric: var(--numeric-tabular); }
|
||||
.client-failover-runtime strong { font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-runtime span, .client-failover-runtime small { color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-validation { margin: -12px 0 0; color: oklch(0.68 0.15 28); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-discard { min-height: 74px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 10px 0; border-top: 1px solid var(--client-border); border-bottom: 1px solid var(--client-border); }
|
||||
.client-failover-discard span { flex-basis: 100%; color: var(--client-text); font: var(--type-body); letter-spacing: var(--type-body-tracking); text-transform: var(--type-body-transform); }
|
||||
.client-failover-discard button { padding: 4px 0; border: 0; background: transparent; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-failover-actions { min-height: 42px; display: flex; flex-wrap: wrap; gap: 18px; }
|
||||
.client-failover-actions button { padding: 7px 0; border: 0; background: transparent; color: var(--client-accent); font: var(--type-control); letter-spacing: var(--type-control-tracking); text-transform: var(--type-control-transform); cursor: pointer; }
|
||||
.client-failover-actions small { flex-basis: 100%; margin-top: -14px; color: var(--client-muted); font: var(--type-label); letter-spacing: var(--type-label-tracking); text-transform: var(--type-label-transform); }
|
||||
.client-failover-actions button:disabled { color: var(--client-muted); cursor: default; opacity: .45; }
|
||||
.client-failover-actions button:focus-visible, .client-failover-form input:focus-visible, .client-failover-form select:focus-visible, .client-failover-advanced summary:focus-visible, .client-failover-add-service:focus-visible, .client-failover-service-editor button:focus-visible, .client-failover-discard button:focus-visible { outline: 2px solid var(--client-accent); outline-offset: 2px; }
|
||||
@media (max-width: 560px) { .client-failover-channel label, .client-failover-timing label, .client-failover-advanced label { grid-template-columns: 1fr; gap: 4px; } .client-failover-service, .client-failover-service-editor { grid-template-columns: 1fr; gap: 2px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .client-failover-channel-title strong { transition: none; } }
|
||||
@@ -8,5 +8,7 @@
|
||||
@import './features/servers.css';
|
||||
@import './primitives.css';
|
||||
@import './features/diagnostics.css';
|
||||
@import './features/failover.css';
|
||||
@import './features/activity-journal.css';
|
||||
@import './layout.css';
|
||||
@import './themes.css';
|
||||
|
||||
Reference in New Issue
Block a user