2313 lines
81 KiB
Rust
2313 lines
81 KiB
Rust
use proxywarden_lib::component_cutover::*;
|
|
use proxywarden_lib::component_detection::{
|
|
matches_legacy_proxifyre_2_2_1_manifest, LegacyPackageFileIdentity,
|
|
LEGACY_PROXIFYRE_2_2_1_MANIFEST,
|
|
};
|
|
use proxywarden_lib::component_inventory::{
|
|
classify_component_candidates, legacy_proxifyre_topshelf_path_matches, prove_legacy_cutover,
|
|
BinaryIdentityEvidence, CandidateRole, ComponentCandidateProbe, InventoryIssue,
|
|
LegacyCutoverEvidence, LegacyCutoverProof, MarkerEvidence, ServiceEvidence,
|
|
};
|
|
use proxywarden_lib::models::ComponentId;
|
|
use proxywarden_lib::process::{
|
|
FullServiceSnapshot, ServiceBaseConfigSnapshot, ServiceConfig2Kind, ServiceConfig2Snapshot,
|
|
ServiceFailureActionsSnapshot, ServiceSecuritySnapshot, ServiceStableState,
|
|
SERVICE_CONFIG2_KINDS,
|
|
};
|
|
use proxywarden_lib::safe_fs::{
|
|
SecurityDescriptorSnapshot, StableObjectIdentity, StableObjectKind,
|
|
};
|
|
use proxywarden_lib::storage::JsonStorage;
|
|
use serde::Serialize;
|
|
use std::cell::{Cell, RefCell};
|
|
use std::collections::BTreeSet;
|
|
use std::fs;
|
|
use std::io;
|
|
use std::path::{Path, PathBuf};
|
|
use uuid::Uuid;
|
|
|
|
#[derive(Default)]
|
|
struct MemoryStore {
|
|
journal: RefCell<Option<CutoverJournal>>,
|
|
observation: RefCell<Option<ComponentCutoverObservation>>,
|
|
fail_observation: Cell<bool>,
|
|
fail_retirement_once: Cell<bool>,
|
|
retirement_calls: Cell<u32>,
|
|
}
|
|
|
|
impl CutoverJournalStore for MemoryStore {
|
|
fn load(&self) -> Result<Option<CutoverJournal>, CutoverError> {
|
|
Ok(self.journal.borrow().clone())
|
|
}
|
|
|
|
fn create(&self, journal: &CutoverJournal) -> Result<(), CutoverError> {
|
|
let mut slot = self.journal.borrow_mut();
|
|
if slot.is_some() {
|
|
return Err(CutoverError::AlreadyExists);
|
|
}
|
|
*slot = Some(journal.clone());
|
|
Ok(())
|
|
}
|
|
|
|
fn replace(&self, journal: &CutoverJournal) -> Result<(), CutoverError> {
|
|
if self.journal.borrow().is_none() {
|
|
return Err(CutoverError::Missing);
|
|
}
|
|
*self.journal.borrow_mut() = Some(journal.clone());
|
|
Ok(())
|
|
}
|
|
|
|
fn write_observation(
|
|
&self,
|
|
observation: &ComponentCutoverObservation,
|
|
) -> Result<(), CutoverError> {
|
|
if self.fail_observation.get() {
|
|
return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected").into());
|
|
}
|
|
*self.observation.borrow_mut() = Some(observation.clone());
|
|
Ok(())
|
|
}
|
|
|
|
fn retire_completed(&self, journal: &CutoverJournal) -> Result<(), CutoverError> {
|
|
if journal.phase != CutoverPhase::ComponentComplete
|
|
|| journal.execution_mode != CutoverExecutionMode::Finished
|
|
{
|
|
return Err(CutoverError::StateConflict);
|
|
}
|
|
self.retirement_calls
|
|
.set(self.retirement_calls.get().saturating_add(1));
|
|
if self.fail_retirement_once.replace(false) {
|
|
return Err(CutoverError::Interrupted);
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
|
struct MockMachine {
|
|
legacy_root_live: bool,
|
|
quarantine_live: bool,
|
|
tombstone_live: bool,
|
|
quarantine_entries: BTreeSet<String>,
|
|
hardened_legacy: BTreeSet<String>,
|
|
legacy_service_exists: bool,
|
|
legacy_service_running: bool,
|
|
legacy_service_policies: BTreeSet<String>,
|
|
legacy_service_security_exact: bool,
|
|
current_components_parent: bool,
|
|
current_candidate_root: bool,
|
|
current_candidate_entries: BTreeSet<String>,
|
|
current_root: bool,
|
|
current_config: bool,
|
|
current_tombstone: bool,
|
|
current_service_exists: bool,
|
|
current_service_running: bool,
|
|
current_service_policies: BTreeSet<String>,
|
|
current_service_security: bool,
|
|
windows_packet_filter_present: bool,
|
|
vc_runtime_present: bool,
|
|
foreign_change: bool,
|
|
}
|
|
|
|
impl MockMachine {
|
|
fn from_before(before: &LegacyBeforeState) -> Self {
|
|
Self {
|
|
legacy_root_live: true,
|
|
quarantine_live: false,
|
|
tombstone_live: false,
|
|
quarantine_entries: BTreeSet::new(),
|
|
hardened_legacy: BTreeSet::new(),
|
|
legacy_service_exists: true,
|
|
legacy_service_running: before.service.original_state == ServiceStableState::Running,
|
|
legacy_service_policies: SERVICE_CONFIG2_KINDS.iter().map(policy_key).collect(),
|
|
legacy_service_security_exact: true,
|
|
current_components_parent: matches!(
|
|
&before.current_components_parent,
|
|
CurrentComponentsParentBeforeState::Present(_)
|
|
),
|
|
current_candidate_root: false,
|
|
current_candidate_entries: BTreeSet::new(),
|
|
current_root: false,
|
|
current_config: false,
|
|
current_tombstone: false,
|
|
current_service_exists: false,
|
|
current_service_running: false,
|
|
current_service_policies: BTreeSet::new(),
|
|
current_service_security: false,
|
|
windows_packet_filter_present: before.prerequisites.windows_packet_filter_present,
|
|
vc_runtime_present: before.prerequisites.vc_runtime_present,
|
|
foreign_change: false,
|
|
}
|
|
}
|
|
|
|
fn fingerprint(&self) -> StateFingerprint {
|
|
StateFingerprint::digest(
|
|
"mock-cutover-machine-v1",
|
|
&serde_json::to_vec(self).unwrap(),
|
|
)
|
|
}
|
|
|
|
fn current_fingerprint(&self) -> StateFingerprint {
|
|
StateFingerprint::digest(
|
|
"mock-cutover-current-v1",
|
|
&serde_json::to_vec(&(
|
|
self.current_components_parent,
|
|
self.current_root,
|
|
self.current_config,
|
|
&self.current_candidate_entries,
|
|
self.current_service_exists,
|
|
self.current_service_running,
|
|
&self.current_service_policies,
|
|
self.current_service_security,
|
|
self.windows_packet_filter_present,
|
|
self.vc_runtime_present,
|
|
))
|
|
.unwrap(),
|
|
)
|
|
}
|
|
|
|
fn current_inventory_fingerprint(&self) -> StateFingerprint {
|
|
StateFingerprint::digest(
|
|
"mock-component-inventory-v1",
|
|
&serde_json::to_vec(&(
|
|
self.current_root,
|
|
self.current_service_exists,
|
|
self.current_service_running,
|
|
))
|
|
.unwrap(),
|
|
)
|
|
}
|
|
|
|
fn apply(
|
|
&mut self,
|
|
operation: &CutoverOperation,
|
|
journal: &CutoverJournal,
|
|
) -> Result<(), CutoverHostError> {
|
|
match operation {
|
|
CutoverOperation::HardenLegacyRootSecurity => {
|
|
self.hardened_legacy.insert("<root>".to_string());
|
|
}
|
|
CutoverOperation::HardenLegacyEntrySecurity(path) => {
|
|
self.hardened_legacy.insert(path_key(path));
|
|
}
|
|
CutoverOperation::InstallWindowsPacketFilterPrerequisite => {
|
|
self.windows_packet_filter_present = true;
|
|
}
|
|
CutoverOperation::InstallVcRuntimePrerequisite => self.vc_runtime_present = true,
|
|
CutoverOperation::CreateCurrentComponentsParent => {
|
|
self.current_components_parent = true;
|
|
}
|
|
CutoverOperation::CreateCurrentCandidateRoot => self.current_candidate_root = true,
|
|
CutoverOperation::WriteCurrentCandidatePackageEntry(path) => {
|
|
self.current_candidate_entries.insert(path_key(path));
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateConfig => {
|
|
self.current_candidate_entries
|
|
.insert("app-config.json".to_string());
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateMarker => {
|
|
self.current_candidate_entries
|
|
.insert("proxywarden-component.json".to_string());
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateReceipt => {
|
|
self.current_candidate_entries
|
|
.insert("proxywarden-install-receipt.json".to_string());
|
|
}
|
|
CutoverOperation::PromoteCurrentCandidate => {
|
|
self.current_candidate_root = false;
|
|
self.current_root = true;
|
|
self.current_config = self.current_candidate_entries.contains("app-config.json");
|
|
}
|
|
CutoverOperation::StopLegacyService => self.legacy_service_running = false,
|
|
CutoverOperation::DeleteLegacyService => {
|
|
self.legacy_service_exists = false;
|
|
self.legacy_service_running = false;
|
|
self.legacy_service_policies.clear();
|
|
self.legacy_service_security_exact = false;
|
|
}
|
|
CutoverOperation::CreateCurrentService => {
|
|
self.current_service_exists = true;
|
|
self.current_service_policies =
|
|
SERVICE_CONFIG2_KINDS.iter().map(policy_key).collect();
|
|
}
|
|
CutoverOperation::SetCurrentServicePolicy(kind) => {
|
|
self.current_service_policies.insert(policy_key(kind));
|
|
}
|
|
CutoverOperation::SetCurrentServiceSecurity => self.current_service_security = true,
|
|
CutoverOperation::StartCurrentService => self.current_service_running = true,
|
|
CutoverOperation::QuarantineLegacyRoot => {
|
|
self.legacy_root_live = false;
|
|
self.quarantine_live = true;
|
|
self.quarantine_entries = journal
|
|
.before_state
|
|
.filesystem
|
|
.entries
|
|
.iter()
|
|
.map(|entry| path_key(&entry.relative_path))
|
|
.collect();
|
|
}
|
|
CutoverOperation::RestoreLegacyRoot => {
|
|
self.quarantine_live = false;
|
|
self.legacy_root_live = true;
|
|
self.quarantine_entries.clear();
|
|
}
|
|
CutoverOperation::StopCurrentService => self.current_service_running = false,
|
|
CutoverOperation::DeleteCurrentService => {
|
|
self.current_service_exists = false;
|
|
self.current_service_running = false;
|
|
self.current_service_policies.clear();
|
|
self.current_service_security = false;
|
|
}
|
|
CutoverOperation::TombstoneCurrentRoot => {
|
|
self.current_root = false;
|
|
self.current_tombstone = true;
|
|
}
|
|
CutoverOperation::TombstoneCurrentCandidate => {
|
|
self.current_candidate_root = false;
|
|
self.current_tombstone = true;
|
|
}
|
|
CutoverOperation::DeleteCurrentEntry(path) => {
|
|
self.current_candidate_entries.remove(&path_key(path));
|
|
}
|
|
CutoverOperation::DeleteCurrentTombstoneRoot => {
|
|
if !self.current_candidate_entries.is_empty() {
|
|
return Err(CutoverHostError::MutationFailed);
|
|
}
|
|
self.current_tombstone = false;
|
|
self.current_config = false;
|
|
}
|
|
CutoverOperation::DeleteTransactionCurrentComponentsParent => {
|
|
if self.current_root
|
|
|| self.current_candidate_root
|
|
|| self.current_tombstone
|
|
|| !self.current_candidate_entries.is_empty()
|
|
{
|
|
return Err(CutoverHostError::MutationFailed);
|
|
}
|
|
self.current_components_parent = false;
|
|
}
|
|
CutoverOperation::UninstallTransactionWindowsPacketFilter => {
|
|
self.windows_packet_filter_present = false;
|
|
}
|
|
CutoverOperation::CreateLegacyService => {
|
|
self.legacy_service_exists = true;
|
|
self.legacy_service_policies = SERVICE_CONFIG2_KINDS
|
|
.iter()
|
|
.copied()
|
|
.filter(|kind| *kind != ServiceConfig2Kind::Description)
|
|
.map(|kind| policy_key(&kind))
|
|
.collect();
|
|
self.legacy_service_security_exact = true;
|
|
}
|
|
CutoverOperation::RestoreLegacyServicePolicy(kind) => {
|
|
self.legacy_service_policies.insert(policy_key(kind));
|
|
}
|
|
CutoverOperation::RestoreLegacyServiceSecurity => {
|
|
self.legacy_service_security_exact = true;
|
|
}
|
|
CutoverOperation::RestoreLegacyEntrySecurity(path) => {
|
|
self.hardened_legacy.remove(&path_key(path));
|
|
}
|
|
CutoverOperation::RestoreLegacyRootSecurity => {
|
|
self.hardened_legacy.remove("<root>");
|
|
}
|
|
CutoverOperation::StartLegacyService => self.legacy_service_running = true,
|
|
CutoverOperation::TombstoneQuarantine => {
|
|
self.quarantine_live = false;
|
|
self.tombstone_live = true;
|
|
}
|
|
CutoverOperation::DeleteQuarantineEntry(path) => {
|
|
self.quarantine_entries.remove(&path_key(path));
|
|
}
|
|
CutoverOperation::DeleteQuarantineRoot => {
|
|
if !self.quarantine_entries.is_empty() {
|
|
return Err(CutoverHostError::MutationFailed);
|
|
}
|
|
self.tombstone_live = false;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
struct MockHost {
|
|
before: LegacyBeforeState,
|
|
machine: MockMachine,
|
|
capture_calls: usize,
|
|
mutation_calls: usize,
|
|
fail_once: Option<CutoverOperation>,
|
|
fail_verify_current: bool,
|
|
fail_verify_current_observation: bool,
|
|
fail_verify_rollback: bool,
|
|
fail_verify_cleanup_once: Option<CutoverHostError>,
|
|
wpf_installer_reported_unchanged_once: bool,
|
|
fail_capture_precondition: bool,
|
|
fail_reacquire_once: bool,
|
|
externally_apply_before_observe: Option<CutoverOperation>,
|
|
mutation_order: Vec<CutoverOperation>,
|
|
}
|
|
|
|
impl MockHost {
|
|
fn new(before: LegacyBeforeState) -> Self {
|
|
let machine = MockMachine::from_before(&before);
|
|
Self {
|
|
before,
|
|
machine,
|
|
capture_calls: 0,
|
|
mutation_calls: 0,
|
|
fail_once: None,
|
|
fail_verify_current: false,
|
|
fail_verify_current_observation: false,
|
|
fail_verify_rollback: false,
|
|
fail_verify_cleanup_once: None,
|
|
wpf_installer_reported_unchanged_once: false,
|
|
fail_capture_precondition: false,
|
|
fail_reacquire_once: false,
|
|
externally_apply_before_observe: None,
|
|
mutation_order: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl CutoverHost for MockHost {
|
|
fn capture_and_lease(
|
|
&mut self,
|
|
_plan: &ProxifyreCutoverPlan,
|
|
) -> Result<LegacyBeforeState, CutoverHostError> {
|
|
self.capture_calls += 1;
|
|
if self.fail_capture_precondition {
|
|
return Err(CutoverHostError::PreconditionFailed);
|
|
}
|
|
Ok(self.before.clone())
|
|
}
|
|
|
|
fn reacquire_leases(&mut self, _journal: &CutoverJournal) -> Result<(), CutoverHostError> {
|
|
if self.fail_reacquire_once {
|
|
self.fail_reacquire_once = false;
|
|
return Err(CutoverHostError::ObservationFailed);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn observe(
|
|
&mut self,
|
|
operation: &CutoverOperation,
|
|
journal: &CutoverJournal,
|
|
) -> Result<StateFingerprint, CutoverHostError> {
|
|
if self.externally_apply_before_observe.as_ref() == Some(operation) {
|
|
self.externally_apply_before_observe = None;
|
|
self.machine.apply(operation, journal)?;
|
|
}
|
|
Ok(self.machine.fingerprint())
|
|
}
|
|
|
|
fn observed_object_identity(
|
|
&mut self,
|
|
operation: &CutoverOperation,
|
|
journal: &CutoverJournal,
|
|
) -> Result<Option<StableObjectIdentity>, CutoverHostError> {
|
|
if operation == &CutoverOperation::CreateCurrentComponentsParent {
|
|
return Ok(Some(StableObjectIdentity {
|
|
volume_serial_number: 8,
|
|
file_id: 199,
|
|
kind: StableObjectKind::Directory,
|
|
size: 0,
|
|
}));
|
|
}
|
|
if operation == &CutoverOperation::CreateCurrentCandidateRoot {
|
|
return Ok(Some(StableObjectIdentity {
|
|
volume_serial_number: 8,
|
|
file_id: 200,
|
|
kind: StableObjectKind::Directory,
|
|
size: 0,
|
|
}));
|
|
}
|
|
let (role, package_path, file_id) = match operation {
|
|
CutoverOperation::WriteCurrentCandidatePackageEntry(path) => {
|
|
(CurrentCandidateFileRole::Package, Some(path.as_path()), 201)
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateConfig => {
|
|
(CurrentCandidateFileRole::Config, None, 202)
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateMarker => {
|
|
(CurrentCandidateFileRole::Marker, None, 203)
|
|
}
|
|
CutoverOperation::WriteCurrentCandidateReceipt => {
|
|
(CurrentCandidateFileRole::Receipt, None, 204)
|
|
}
|
|
_ => return Ok(None),
|
|
};
|
|
let file = journal
|
|
.before_state
|
|
.current_candidate
|
|
.files
|
|
.iter()
|
|
.find(|file| {
|
|
file.role == role
|
|
&& package_path
|
|
.is_none_or(|path| path_key(&file.relative_path) == path_key(path))
|
|
})
|
|
.ok_or(CutoverHostError::VerificationFailed)?;
|
|
Ok(Some(StableObjectIdentity {
|
|
volume_serial_number: 8,
|
|
file_id,
|
|
kind: StableObjectKind::RegularFile,
|
|
size: file.size,
|
|
}))
|
|
}
|
|
|
|
fn expected_effect(
|
|
&mut self,
|
|
operation: &CutoverOperation,
|
|
journal: &CutoverJournal,
|
|
) -> Result<StateFingerprint, CutoverHostError> {
|
|
let mut expected = self.machine.clone();
|
|
expected.apply(operation, journal)?;
|
|
Ok(expected.fingerprint())
|
|
}
|
|
|
|
fn mutate(
|
|
&mut self,
|
|
operation: &CutoverOperation,
|
|
journal: &CutoverJournal,
|
|
) -> Result<Option<MutationAuthorityEvidence>, CutoverHostError> {
|
|
self.mutation_calls += 1;
|
|
self.mutation_order.push(operation.clone());
|
|
if operation == &CutoverOperation::InstallWindowsPacketFilterPrerequisite
|
|
&& self.wpf_installer_reported_unchanged_once
|
|
{
|
|
self.wpf_installer_reported_unchanged_once = false;
|
|
self.machine.apply(operation, journal)?;
|
|
return Err(CutoverHostError::MutationFailed);
|
|
}
|
|
if self.fail_once.as_ref() == Some(operation) {
|
|
self.fail_once = None;
|
|
return Err(CutoverHostError::MutationFailed);
|
|
}
|
|
self.machine.apply(operation, journal)?;
|
|
Ok(
|
|
(operation == &CutoverOperation::InstallWindowsPacketFilterPrerequisite)
|
|
.then_some(MutationAuthorityEvidence::WindowsPacketFilterInstalledByTransaction),
|
|
)
|
|
}
|
|
|
|
fn verify_current(
|
|
&mut self,
|
|
journal: &CutoverJournal,
|
|
) -> Result<StateFingerprint, CutoverHostError> {
|
|
if self.fail_verify_current {
|
|
return Err(CutoverHostError::VerificationFailed);
|
|
}
|
|
if self.fail_verify_current_observation {
|
|
return Err(CutoverHostError::ObservationFailed);
|
|
}
|
|
let should_run = journal.plan.original_service_state == LegacyServiceState::Running;
|
|
if !self.machine.current_root
|
|
|| !self.machine.current_config
|
|
|| !self.machine.current_service_exists
|
|
|| self.machine.current_service_running != should_run
|
|
|| self.machine.current_service_policies.len() != SERVICE_CONFIG2_KINDS.len()
|
|
|| !self.machine.current_service_security
|
|
{
|
|
return Err(CutoverHostError::VerificationFailed);
|
|
}
|
|
Ok(self.machine.current_fingerprint())
|
|
}
|
|
|
|
fn verify_rollback(&mut self, _journal: &CutoverJournal) -> Result<(), CutoverHostError> {
|
|
if self.fail_verify_rollback || self.machine != MockMachine::from_before(&self.before) {
|
|
Err(CutoverHostError::VerificationFailed)
|
|
} else {
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn verify_current_inventory(
|
|
&mut self,
|
|
_journal: &CutoverJournal,
|
|
) -> Result<StateFingerprint, CutoverHostError> {
|
|
Ok(self.machine.current_inventory_fingerprint())
|
|
}
|
|
|
|
fn verify_cleanup(&mut self, _journal: &CutoverJournal) -> Result<(), CutoverHostError> {
|
|
if let Some(error) = self.fail_verify_cleanup_once.take() {
|
|
return Err(error);
|
|
}
|
|
if self.machine.current_service_exists
|
|
&& (self.machine.quarantine_live || self.machine.tombstone_live)
|
|
{
|
|
Ok(())
|
|
} else {
|
|
Err(CutoverHostError::VerificationFailed)
|
|
}
|
|
}
|
|
}
|
|
|
|
struct CrashAfterMutation {
|
|
target: usize,
|
|
seen: usize,
|
|
}
|
|
|
|
impl CutoverFaultInjector for CrashAfterMutation {
|
|
fn after_host_mutation(
|
|
&mut self,
|
|
_direction: MutationDirection,
|
|
_operation: &CutoverOperation,
|
|
) -> Result<(), CutoverError> {
|
|
self.seen += 1;
|
|
if self.seen == self.target {
|
|
Err(CutoverError::Interrupted)
|
|
} else {
|
|
Ok(())
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn crash_after_every_forward_mutation_recovers_without_ambiguous_state() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let clean_plan = sample_plan(LegacyServiceState::Running);
|
|
let clean_store = MemoryStore::default();
|
|
let mut clean_host = MockHost::new(before.clone());
|
|
let clean_outcome =
|
|
begin_proxifyre_cutover(&clean_store, &mut clean_host, clean_plan, 10).unwrap();
|
|
assert_eq!(clean_outcome, CutoverRunOutcome::AwaitingNextStart);
|
|
let forward_mutations = clean_host.mutation_calls;
|
|
assert!(forward_mutations > 20);
|
|
|
|
for target in 1..=forward_mutations {
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before.clone());
|
|
let initial = host.machine.clone();
|
|
let mut fault = CrashAfterMutation { target, seen: 0 };
|
|
let error = begin_proxifyre_cutover_with_fault(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Running),
|
|
20,
|
|
&mut fault,
|
|
)
|
|
.expect_err("fault must interrupt after the selected mutation");
|
|
assert!(matches!(error, CutoverError::Interrupted));
|
|
|
|
let outcome = recover_proxifyre_cutover(&store, &mut host, 21).unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
assert!(journal
|
|
.mutations
|
|
.iter()
|
|
.all(|record| record.effect.is_some()));
|
|
if target == forward_mutations {
|
|
assert_eq!(outcome, CutoverRunOutcome::AwaitingNextStart);
|
|
assert_eq!(journal.phase, CutoverPhase::LegacyQuarantined);
|
|
} else {
|
|
assert_eq!(outcome, CutoverRunOutcome::RolledBack);
|
|
assert_eq!(journal.phase, CutoverPhase::RolledBack);
|
|
assert_eq!(
|
|
host.machine, initial,
|
|
"failed recovery at mutation {target}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn legacy_root_is_sealed_before_other_mutations_and_entries_wait_for_service_stop() {
|
|
let before = sample_before(LegacyServiceState::Running, false, false);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Running),
|
|
25,
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
host.mutation_order.first(),
|
|
Some(&CutoverOperation::HardenLegacyRootSecurity),
|
|
"no prerequisite or candidate write may precede the root trust boundary"
|
|
);
|
|
let promote = host
|
|
.mutation_order
|
|
.iter()
|
|
.position(|operation| operation == &CutoverOperation::PromoteCurrentCandidate)
|
|
.unwrap();
|
|
let stop = host
|
|
.mutation_order
|
|
.iter()
|
|
.position(|operation| operation == &CutoverOperation::StopLegacyService)
|
|
.unwrap();
|
|
let first_entry_harden = host
|
|
.mutation_order
|
|
.iter()
|
|
.position(|operation| matches!(operation, CutoverOperation::HardenLegacyEntrySecurity(_)))
|
|
.unwrap();
|
|
assert!(
|
|
promote < stop,
|
|
"candidate staging must finish before downtime"
|
|
);
|
|
assert!(
|
|
stop < first_entry_harden,
|
|
"legacy entry mutation leases and ACL writes require the durable stop effect"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn observed_packet_filter_presence_without_changed_authority_never_becomes_owned() {
|
|
let before = sample_before(LegacyServiceState::Stopped, false, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.wpf_installer_reported_unchanged_once = true;
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
26,
|
|
),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
let install = failed
|
|
.mutations
|
|
.iter()
|
|
.find(|record| record.operation == CutoverOperation::InstallWindowsPacketFilterPrerequisite)
|
|
.expect("WPF install intent must remain sealed");
|
|
assert!(install.effect.is_none());
|
|
assert!(install.authority_evidence.is_none());
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(
|
|
failed.recovery_code.as_deref(),
|
|
Some("mutation_authority_ambiguous")
|
|
);
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::UninstallTransactionWindowsPacketFilter));
|
|
|
|
let mutations_before_restart = host.mutation_calls;
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 27),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutations_before_restart);
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::UninstallTransactionWindowsPacketFilter));
|
|
}
|
|
|
|
#[test]
|
|
fn forged_or_missing_packet_filter_authority_is_rejected_on_load() {
|
|
let before = sample_before(LegacyServiceState::Stopped, false, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
27,
|
|
)
|
|
.unwrap();
|
|
let exact = store.load().unwrap().unwrap();
|
|
let mutation_calls = host.mutation_calls;
|
|
|
|
let mut missing = exact.clone();
|
|
missing
|
|
.mutations
|
|
.iter_mut()
|
|
.find(|record| record.operation == CutoverOperation::InstallWindowsPacketFilterPrerequisite)
|
|
.unwrap()
|
|
.authority_evidence = None;
|
|
store.replace(&missing).unwrap();
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 28),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_calls);
|
|
|
|
let mut forged = exact;
|
|
forged.mutations[0].authority_evidence =
|
|
Some(MutationAuthorityEvidence::WindowsPacketFilterInstalledByTransaction);
|
|
store.replace(&forged).unwrap();
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 29),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_calls);
|
|
}
|
|
|
|
#[test]
|
|
fn running_legacy_root_only_hardening_rolls_back_after_restart_before_service_stop() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before.clone());
|
|
let clean_store = MemoryStore::default();
|
|
let mut clean = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&clean_store,
|
|
&mut clean,
|
|
sample_plan(LegacyServiceState::Running),
|
|
28,
|
|
)
|
|
.unwrap();
|
|
let crash_target = clean
|
|
.mutation_order
|
|
.iter()
|
|
.position(|operation| operation == &CutoverOperation::PromoteCurrentCandidate)
|
|
.map(|index| index + 1)
|
|
.unwrap();
|
|
let mut fault = CrashAfterMutation {
|
|
target: crash_target,
|
|
seen: 0,
|
|
};
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover_with_fault(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Running),
|
|
29,
|
|
&mut fault,
|
|
),
|
|
Err(CutoverError::Interrupted)
|
|
));
|
|
assert!(host.machine.legacy_service_running);
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::StopLegacyService));
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 30).unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
assert!(host.machine.legacy_service_running);
|
|
assert!(!host.machine.hardened_legacy.contains("<root>"));
|
|
assert!(host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::RestoreLegacyRootSecurity));
|
|
}
|
|
|
|
#[test]
|
|
fn ambiguous_live_state_is_sealed_as_recovery_required_without_another_mutation() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let plan = sample_plan(LegacyServiceState::Running);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
let mut fault = CrashAfterMutation { target: 1, seen: 0 };
|
|
begin_proxifyre_cutover_with_fault(&store, &mut host, plan, 30, &mut fault)
|
|
.expect_err("injected crash");
|
|
let calls_before_recovery = host.mutation_calls;
|
|
host.machine.foreign_change = true;
|
|
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 31),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(host.mutation_calls, calls_before_recovery);
|
|
let journal = store.load().unwrap().unwrap();
|
|
assert_eq!(journal.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(
|
|
journal.recovery_code.as_deref(),
|
|
Some("ambiguous_live_state")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn external_pre_effect_is_recovery_required_with_zero_host_mutations() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.externally_apply_before_observe = Some(CutoverOperation::HardenLegacyRootSecurity);
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
32,
|
|
),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(host.mutation_calls, 0);
|
|
let journal = store.load().unwrap().unwrap();
|
|
assert_eq!(journal.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(
|
|
journal.recovery_code.as_deref(),
|
|
Some("operation_already_at_expected_effect")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn ambiguous_current_verification_never_starts_rollback() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_verify_current_observation = true;
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Running),
|
|
33,
|
|
),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert!(!host.mutation_order.iter().any(|operation| matches!(
|
|
operation,
|
|
CutoverOperation::CreateLegacyService
|
|
| CutoverOperation::RestoreLegacyServicePolicy(_)
|
|
| CutoverOperation::RestoreLegacyServiceSecurity
|
|
| CutoverOperation::TombstoneCurrentRoot
|
|
)));
|
|
let journal = store.load().unwrap().unwrap();
|
|
assert_eq!(journal.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(
|
|
journal.recovery_code.as_deref(),
|
|
Some("current_verification_ambiguous")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn quarantine_failure_keeps_verified_current_and_never_moves_legacy_root() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let plan = sample_plan(LegacyServiceState::Stopped);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_once = Some(CutoverOperation::QuarantineLegacyRoot);
|
|
let outcome = begin_proxifyre_cutover(&store, &mut host, plan, 40).unwrap();
|
|
|
|
assert_eq!(outcome, CutoverRunOutcome::QuarantinePending);
|
|
assert!(host.machine.current_service_exists);
|
|
assert!(host.machine.legacy_root_live);
|
|
assert!(!host.machine.quarantine_live);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::ComponentVerified
|
|
);
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 41).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
assert!(host.machine.quarantine_live);
|
|
}
|
|
|
|
#[test]
|
|
fn cleanup_requires_new_start_and_explicit_route_smoke() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let store = MemoryStore::default();
|
|
let plan = sample_plan(LegacyServiceState::Running);
|
|
let original_session = plan.created_startup_session_id.clone();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(&store, &mut host, plan, 50).unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let original_evidence = sample_user_evidence(&journal, &host, original_session, 51, false);
|
|
|
|
assert!(matches!(
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &original_evidence, 51),
|
|
Err(CutoverError::StateConflict)
|
|
));
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
52,
|
|
false,
|
|
);
|
|
let mut mismatched_evidence = evidence.clone();
|
|
mismatched_evidence.current_inventory_fingerprint = hash('d');
|
|
let mutations_before_mismatch = host.mutation_calls;
|
|
assert!(matches!(
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &mismatched_evidence, 52),
|
|
Err(CutoverError::StateConflict)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutations_before_mismatch);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::LegacyQuarantined
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 52).unwrap();
|
|
assert!(matches!(
|
|
confirm_proxifyre_cleanup(&store, &evidence, 53),
|
|
Err(CutoverError::StateConflict)
|
|
));
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(54),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 54).unwrap();
|
|
assert_eq!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 55).unwrap(),
|
|
CutoverRunOutcome::Complete
|
|
);
|
|
assert!(!host.machine.quarantine_live);
|
|
assert!(!host.machine.tombstone_live);
|
|
assert!(host.machine.current_service_running);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::ComponentComplete
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn completed_retirement_failure_retries_from_durable_terminal_journal() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
55,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
56,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 56).unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(57),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 57).unwrap();
|
|
|
|
store.fail_retirement_once.set(true);
|
|
assert!(matches!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 58),
|
|
Err(CutoverError::Interrupted)
|
|
));
|
|
let terminal = store.load().unwrap().unwrap();
|
|
assert_eq!(terminal.phase, CutoverPhase::ComponentComplete);
|
|
assert_eq!(terminal.execution_mode, CutoverExecutionMode::Finished);
|
|
assert_eq!(store.retirement_calls.get(), 1);
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 59).unwrap(),
|
|
CutoverRunOutcome::Complete
|
|
);
|
|
assert_eq!(store.retirement_calls.get(), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn awaiting_user_recovery_preserves_next_start_milestone_and_evidence() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
55,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
56,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 56).unwrap();
|
|
let sealed_next_start = store.load().unwrap().unwrap().next_start.unwrap();
|
|
let mutations_before = host.mutation_calls;
|
|
|
|
host.fail_reacquire_once = true;
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 57),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(failed.execution_mode, CutoverExecutionMode::AwaitingUser);
|
|
assert_eq!(failed.next_start.as_ref(), Some(&sealed_next_start));
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 58).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
let recovered = store.load().unwrap().unwrap();
|
|
assert_eq!(recovered.phase, CutoverPhase::NextStartVerified);
|
|
assert_eq!(recovered.next_start.as_ref(), Some(&sealed_next_start));
|
|
assert_eq!(host.mutation_calls, mutations_before);
|
|
}
|
|
|
|
#[test]
|
|
fn awaiting_user_recovery_preserves_cleanup_confirmation_and_rechecks_preflight() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
59,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
60,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 60).unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(61),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 61).unwrap();
|
|
let sealed = store.load().unwrap().unwrap();
|
|
let mutations_before = host.mutation_calls;
|
|
|
|
host.fail_reacquire_once = true;
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 62),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 63).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
let recovered = store.load().unwrap().unwrap();
|
|
assert_eq!(recovered.phase, CutoverPhase::CleanupConfirmed);
|
|
assert_eq!(recovered.next_start, sealed.next_start);
|
|
assert_eq!(recovered.route_smoke, sealed.route_smoke);
|
|
assert_eq!(host.mutation_calls, mutations_before);
|
|
}
|
|
|
|
#[test]
|
|
fn cleanup_preflight_failures_are_durable_and_retry_without_mutation() {
|
|
for (index, verify_failure) in [
|
|
None,
|
|
Some(CutoverHostError::ObservationFailed),
|
|
Some(CutoverHostError::VerificationFailed),
|
|
]
|
|
.into_iter()
|
|
.enumerate()
|
|
{
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
64 + index as u64 * 10,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
65 + index as u64 * 10,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 65 + index as u64 * 10)
|
|
.unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(66 + index as u64 * 10),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 66 + index as u64 * 10).unwrap();
|
|
if let Some(failure) = verify_failure {
|
|
host.fail_verify_cleanup_once = Some(failure);
|
|
} else {
|
|
host.fail_reacquire_once = true;
|
|
}
|
|
let mutations_before = host.mutation_calls;
|
|
|
|
assert!(matches!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 67 + index as u64 * 10),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(failed.execution_mode, CutoverExecutionMode::AwaitingUser);
|
|
assert!(failed.next_start.is_some());
|
|
assert!(failed.route_smoke.is_some());
|
|
assert_eq!(host.mutation_calls, mutations_before);
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 68 + index as u64 * 10).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::CleanupConfirmed
|
|
);
|
|
assert_eq!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 69 + index as u64 * 10).unwrap(),
|
|
CutoverRunOutcome::Complete
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn next_start_observation_failure_preserves_awaiting_user_without_mutation() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
56,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
57,
|
|
false,
|
|
);
|
|
|
|
let machine_before = host.machine.clone();
|
|
let mutation_calls_before = host.mutation_calls;
|
|
host.fail_verify_current_observation = true;
|
|
assert!(matches!(
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 57),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(failed.execution_mode, CutoverExecutionMode::AwaitingUser);
|
|
assert!(
|
|
read_sealed_cutover_status(&store)
|
|
.unwrap()
|
|
.unwrap()
|
|
.can_recover
|
|
);
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 58),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_calls_before);
|
|
assert_eq!(host.machine, machine_before);
|
|
|
|
host.fail_verify_current_observation = false;
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 59).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
assert_eq!(host.mutation_calls, mutation_calls_before);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::LegacyQuarantined
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 60).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn next_start_definite_verification_failure_selects_retryable_rollback() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let original_machine = MockMachine::from_before(&before);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
61,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
62,
|
|
false,
|
|
);
|
|
|
|
host.fail_verify_current = true;
|
|
assert!(matches!(
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 62),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().execution_mode,
|
|
CutoverExecutionMode::RollingBack
|
|
);
|
|
|
|
host.fail_verify_current = false;
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 63).unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
assert_eq!(host.machine, original_machine);
|
|
}
|
|
|
|
#[test]
|
|
fn unsupported_identity_or_downgrade_has_zero_host_calls_and_zero_journal() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let exact_plan = sample_plan(LegacyServiceState::Stopped);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before.clone());
|
|
let mut unsupported = before.clone();
|
|
unsupported.additional_matching_service = true;
|
|
assert!(sample_cutover_proof_result(&exact_plan, &unsupported).is_err());
|
|
assert_eq!(host.capture_calls, 0);
|
|
assert!(store.load().unwrap().is_none());
|
|
|
|
let mut downgrade = exact_plan;
|
|
downgrade.bundled_version = "2.1.9".to_string();
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(&store, &mut host, downgrade, 61,),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.capture_calls, 0);
|
|
assert!(store.load().unwrap().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn additional_matching_service_in_leased_snapshot_aborts_before_journal_or_mutation() {
|
|
let mut before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
before.additional_matching_service = true;
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
65,
|
|
),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.capture_calls, 1);
|
|
assert_eq!(host.mutation_calls, 0);
|
|
assert!(store.load().unwrap().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn unsafe_scm_profile_in_leased_snapshot_aborts_before_journal_or_mutation() {
|
|
let mut before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let delayed_auto_start = before
|
|
.service
|
|
.config2
|
|
.iter_mut()
|
|
.find_map(|value| match value {
|
|
ServiceConfig2Snapshot::DelayedAutoStart(enabled) => Some(enabled),
|
|
_ => None,
|
|
})
|
|
.expect("complete sample config2");
|
|
*delayed_auto_start = true;
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
66,
|
|
),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.capture_calls, 1);
|
|
assert_eq!(host.mutation_calls, 0);
|
|
assert!(store.load().unwrap().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn scm_snapshot_fingerprint_binds_base_config2_security_and_original_state() {
|
|
let before = sample_service(LegacyServiceState::Running);
|
|
let expected = legacy_service_snapshot_fingerprint(&before);
|
|
let mutations: [fn(&mut FullServiceSnapshot); 4] = [
|
|
|snapshot| snapshot.base.tag_id = 9,
|
|
|snapshot| {
|
|
let value = snapshot
|
|
.config2
|
|
.iter_mut()
|
|
.find_map(|value| match value {
|
|
ServiceConfig2Snapshot::PreshutdownTimeout(timeout) => Some(timeout),
|
|
_ => None,
|
|
})
|
|
.expect("complete sample config2");
|
|
*value += 1;
|
|
},
|
|
|snapshot| snapshot.security.self_relative_descriptor.push(5),
|
|
|snapshot| snapshot.original_state = ServiceStableState::Stopped,
|
|
];
|
|
|
|
for mutate in mutations {
|
|
let mut changed = before.clone();
|
|
mutate(&mut changed);
|
|
assert_ne!(legacy_service_snapshot_fingerprint(&changed), expected);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn forged_persisted_legacy_proof_fingerprint_aborts_recovery_without_mutation() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
let mut fault = CrashAfterMutation { target: 1, seen: 0 };
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover_with_fault(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
67,
|
|
&mut fault,
|
|
),
|
|
Err(CutoverError::Interrupted)
|
|
));
|
|
let mutation_count = host.mutation_calls;
|
|
let mut forged = store.load().unwrap().unwrap();
|
|
forged.before_state.identity_fingerprint = hash('0');
|
|
store.replace(&forged).unwrap();
|
|
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 68),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_count);
|
|
}
|
|
|
|
#[test]
|
|
fn forged_operation_order_or_terminal_phase_is_rejected_without_host_mutation() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
68,
|
|
)
|
|
.unwrap();
|
|
let mutation_count = host.mutation_calls;
|
|
let exact = store.load().unwrap().unwrap();
|
|
|
|
let mut reordered = exact.clone();
|
|
let first = reordered.mutations[0].operation.clone();
|
|
reordered.mutations[0].operation = reordered.mutations[1].operation.clone();
|
|
reordered.mutations[1].operation = first;
|
|
store.replace(&reordered).unwrap();
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 69),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_count);
|
|
|
|
let mut forged_forward_noop = exact.clone();
|
|
let first = &mut forged_forward_noop.mutations[0];
|
|
first.before_state = first.expected_effect.clone();
|
|
first.effect = Some(MutationEffect {
|
|
disposition: EffectDisposition::AlreadySatisfied,
|
|
observed: first.expected_effect.clone(),
|
|
object_identity: None,
|
|
observed_at_epoch_seconds: first.intent_written_at_epoch_seconds,
|
|
});
|
|
store.replace(&forged_forward_noop).unwrap();
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 69),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_count);
|
|
|
|
store.replace(&exact).unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&exact,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
69,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 69).unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(70),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 70).unwrap();
|
|
let mut forged_complete = store.load().unwrap().unwrap();
|
|
forged_complete.phase = CutoverPhase::ComponentComplete;
|
|
forged_complete.execution_mode = CutoverExecutionMode::Finished;
|
|
store.replace(&forged_complete).unwrap();
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 71),
|
|
Err(CutoverError::InvalidPlan)
|
|
));
|
|
assert_eq!(host.mutation_calls, mutation_count);
|
|
}
|
|
|
|
#[test]
|
|
fn transaction_fingerprint_binds_all_immutable_before_state_but_not_progress() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
68,
|
|
)
|
|
.unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let fingerprint = cutover_transaction_fingerprint(&journal).unwrap();
|
|
assert_eq!(
|
|
store
|
|
.observation
|
|
.borrow()
|
|
.as_ref()
|
|
.unwrap()
|
|
.transaction_fingerprint,
|
|
fingerprint
|
|
);
|
|
|
|
let mut progress_only = journal.clone();
|
|
progress_only.phase = CutoverPhase::RecoveryRequired;
|
|
progress_only.execution_mode = CutoverExecutionMode::RollingBack;
|
|
progress_only.mutations.clear();
|
|
progress_only.recovery_code = Some("injected_progress_only".to_string());
|
|
progress_only.updated_at_epoch_seconds += 100;
|
|
assert_eq!(
|
|
cutover_transaction_fingerprint(&progress_only).unwrap(),
|
|
fingerprint
|
|
);
|
|
|
|
let mut changed_filesystem = journal.clone();
|
|
changed_filesystem
|
|
.before_state
|
|
.filesystem
|
|
.root
|
|
.identity
|
|
.file_id += 1;
|
|
assert_ne!(
|
|
cutover_transaction_fingerprint(&changed_filesystem).unwrap(),
|
|
fingerprint
|
|
);
|
|
let mut changed_security = journal.clone();
|
|
changed_security.before_state.filesystem.root.security = serde_json::from_value(
|
|
serde_json::json!({"selfRelative": [9, 8, 7, 6], "sacl": "present"}),
|
|
)
|
|
.unwrap();
|
|
assert_ne!(
|
|
cutover_transaction_fingerprint(&changed_security).unwrap(),
|
|
fingerprint
|
|
);
|
|
let mut changed_service = journal.clone();
|
|
changed_service
|
|
.before_state
|
|
.service
|
|
.security
|
|
.self_relative_descriptor
|
|
.push(9);
|
|
assert_ne!(
|
|
cutover_transaction_fingerprint(&changed_service).unwrap(),
|
|
fingerprint
|
|
);
|
|
let mut changed_prerequisite = journal.clone();
|
|
changed_prerequisite
|
|
.before_state
|
|
.prerequisites
|
|
.windows_packet_filter_present = false;
|
|
assert_ne!(
|
|
cutover_transaction_fingerprint(&changed_prerequisite).unwrap(),
|
|
fingerprint
|
|
);
|
|
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
69,
|
|
false,
|
|
);
|
|
let next_start = mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 69).unwrap();
|
|
assert_eq!(next_start.transaction_fingerprint, fingerprint);
|
|
assert!(next_start.next_start_verified);
|
|
assert!(!next_start.route_smoke_confirmed);
|
|
let next_evidence = next_start.evidence_fingerprint.unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(70),
|
|
..evidence
|
|
};
|
|
let cleanup = confirm_proxifyre_cleanup(&store, &confirmed, 70).unwrap();
|
|
assert_eq!(cleanup.transaction_fingerprint, fingerprint);
|
|
assert!(cleanup.route_smoke_confirmed);
|
|
assert_ne!(cleanup.evidence_fingerprint.unwrap(), next_evidence);
|
|
}
|
|
|
|
#[test]
|
|
fn observation_cache_failure_cannot_block_sealed_transaction() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let plan = sample_plan(LegacyServiceState::Stopped);
|
|
let store = MemoryStore::default();
|
|
store.fail_observation.set(true);
|
|
let mut host = MockHost::new(before);
|
|
assert_eq!(
|
|
begin_proxifyre_cutover(&store, &mut host, plan, 70,).unwrap(),
|
|
CutoverRunOutcome::AwaitingNextStart
|
|
);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::LegacyQuarantined
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn stopped_route_smoke_lifecycle_hint_is_narrow_and_never_general_authority() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
71,
|
|
)
|
|
.unwrap();
|
|
let observation = read_sealed_cutover_status(&store).unwrap().unwrap();
|
|
|
|
assert!(observation_suggests_lifecycle_blocked(Some(&observation)));
|
|
assert!(observation_allows_stopped_route_smoke_lifecycle(Some(
|
|
&observation
|
|
)));
|
|
|
|
let mut denied = observation.clone();
|
|
denied.original_service_state = LegacyServiceState::Running;
|
|
assert!(!observation_allows_stopped_route_smoke_lifecycle(Some(
|
|
&denied
|
|
)));
|
|
denied = observation.clone();
|
|
denied.disabled_code = Some("manual_intervention_required".to_string());
|
|
assert!(!observation_allows_stopped_route_smoke_lifecycle(Some(
|
|
&denied
|
|
)));
|
|
denied = observation;
|
|
denied.phase = CutoverPhase::NextStartVerified;
|
|
denied.state = CutoverDisplayState::AwaitingRouteSmoke;
|
|
assert!(!observation_allows_stopped_route_smoke_lifecycle(Some(
|
|
&denied
|
|
)));
|
|
assert!(!observation_allows_stopped_route_smoke_lifecycle(None));
|
|
}
|
|
|
|
#[test]
|
|
fn corrupt_redacted_observation_read_is_strictly_read_only() {
|
|
let root = std::env::temp_dir().join(format!(
|
|
"proxywarden-cutover-observation-{}",
|
|
Uuid::new_v4().hyphenated()
|
|
));
|
|
let state = root.join("state");
|
|
let observation = state.join("component-cutover-observation.json");
|
|
fs::create_dir_all(&state).unwrap();
|
|
fs::write(&observation, b"{ definitely not valid json").unwrap();
|
|
fs::write(observation.with_extension("json.bak"), b"backup sentinel").unwrap();
|
|
let before: BTreeSet<_> = fs::read_dir(&state)
|
|
.unwrap()
|
|
.map(|entry| entry.unwrap().file_name())
|
|
.collect();
|
|
|
|
let error = read_redacted_observation(&JsonStorage::new(&root))
|
|
.expect_err("invalid observation must be reported without recovery writes");
|
|
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
|
assert_eq!(
|
|
fs::read(&observation).unwrap(),
|
|
b"{ definitely not valid json"
|
|
);
|
|
assert_eq!(
|
|
fs::read(observation.with_extension("json.bak")).unwrap(),
|
|
b"backup sentinel"
|
|
);
|
|
let after: BTreeSet<_> = fs::read_dir(&state)
|
|
.unwrap()
|
|
.map(|entry| entry.unwrap().file_name())
|
|
.collect();
|
|
assert_eq!(after, before);
|
|
|
|
let structurally_invalid = serde_json::json!({
|
|
"schemaVersion": CUTOVER_OBSERVATION_SCHEMA_VERSION + 1,
|
|
"component": "proxifyre",
|
|
"cutoverId": Uuid::new_v4().hyphenated().to_string(),
|
|
"state": "awaiting_next_start",
|
|
"phase": "legacy_quarantined",
|
|
"originalServiceState": "stopped",
|
|
"legacyVersion": "2.2.1",
|
|
"bundledVersion": "2.4.0",
|
|
"operationFingerprint": hash('a'),
|
|
"transactionFingerprint": hash('b'),
|
|
"evidenceFingerprint": null,
|
|
"nextStartVerified": false,
|
|
"routeSmokeConfirmed": false,
|
|
"legacyPathLabel": "legacy ProxiFyre installation",
|
|
"currentPathLabel": "ProxyWarden managed components",
|
|
"canRecover": false,
|
|
"canCleanup": false,
|
|
"disabledCode": null,
|
|
"updatedAtEpochSeconds": 1,
|
|
});
|
|
let invalid_bytes = serde_json::to_vec(&structurally_invalid).unwrap();
|
|
fs::write(&observation, &invalid_bytes).unwrap();
|
|
assert_eq!(
|
|
read_redacted_observation(&JsonStorage::new(&root))
|
|
.expect_err("schema-valid but inconsistent observation must be rejected")
|
|
.kind(),
|
|
io::ErrorKind::InvalidData
|
|
);
|
|
assert_eq!(fs::read(&observation).unwrap(), invalid_bytes);
|
|
let final_files: BTreeSet<_> = fs::read_dir(&state)
|
|
.unwrap()
|
|
.map(|entry| entry.unwrap().file_name())
|
|
.collect();
|
|
assert_eq!(final_files, before);
|
|
fs::remove_dir_all(root).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn redacted_observation_validator_rejects_impossible_lifecycle_flags() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
72,
|
|
)
|
|
.unwrap();
|
|
let observation = read_sealed_cutover_status(&store).unwrap().unwrap();
|
|
validate_component_cutover_observation(&observation).unwrap();
|
|
|
|
let mut invalid_state = observation.clone();
|
|
invalid_state.state = CutoverDisplayState::Complete;
|
|
assert!(validate_component_cutover_observation(&invalid_state).is_err());
|
|
|
|
let mut invalid_cleanup = observation.clone();
|
|
invalid_cleanup.can_cleanup = true;
|
|
assert!(validate_component_cutover_observation(&invalid_cleanup).is_err());
|
|
|
|
let mut invalid_evidence = observation;
|
|
invalid_evidence.evidence_fingerprint = Some(hash('c'));
|
|
assert!(validate_component_cutover_observation(&invalid_evidence).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn untrusted_user_evidence_is_structured_hashed_and_stored_separately() {
|
|
let root = std::env::temp_dir().join(format!(
|
|
"proxywarden-cutover-user-evidence-{}",
|
|
Uuid::new_v4().hyphenated()
|
|
));
|
|
let storage = JsonStorage::new(&root);
|
|
let evidence = ComponentCutoverUserEvidence {
|
|
schema_version: CUTOVER_USER_EVIDENCE_SCHEMA_VERSION,
|
|
cutover_id: Uuid::new_v4().hyphenated().to_string(),
|
|
startup_session_id: Uuid::new_v4().hyphenated().to_string(),
|
|
current_inventory_fingerprint: hash('a'),
|
|
route_smoke_confirmed: false,
|
|
observed_at_epoch_seconds: 100,
|
|
confirmed_at_epoch_seconds: None,
|
|
};
|
|
validate_component_cutover_user_evidence(&evidence).unwrap();
|
|
let fingerprint = component_cutover_user_evidence_fingerprint(&evidence).unwrap();
|
|
storage
|
|
.write_component_cutover_user_evidence(&evidence)
|
|
.unwrap();
|
|
assert_eq!(
|
|
storage.read_component_cutover_user_evidence().unwrap(),
|
|
Some(evidence.clone())
|
|
);
|
|
|
|
let mut confirmed = evidence.clone();
|
|
confirmed.route_smoke_confirmed = true;
|
|
confirmed.confirmed_at_epoch_seconds = Some(101);
|
|
validate_component_cutover_user_evidence(&confirmed).unwrap();
|
|
assert_ne!(
|
|
component_cutover_user_evidence_fingerprint(&confirmed).unwrap(),
|
|
fingerprint
|
|
);
|
|
|
|
let mut malformed = evidence;
|
|
malformed.route_smoke_confirmed = true;
|
|
assert!(validate_component_cutover_user_evidence(&malformed).is_err());
|
|
assert!(storage
|
|
.write_component_cutover_user_evidence(&malformed)
|
|
.is_err());
|
|
fs::remove_dir_all(root).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn current_verification_failure_rolls_back_in_same_run_before_artifact_cleanup() {
|
|
let before = sample_before(LegacyServiceState::Running, false, true);
|
|
let plan = sample_plan(LegacyServiceState::Running);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
let initial = host.machine.clone();
|
|
host.fail_verify_current = true;
|
|
|
|
assert_eq!(
|
|
begin_proxifyre_cutover(&store, &mut host, plan, 80,).unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
assert_eq!(host.machine, initial);
|
|
let start_legacy = operation_index(&host.mutation_order, &CutoverOperation::StartLegacyService);
|
|
let delete_current = operation_index(
|
|
&host.mutation_order,
|
|
&CutoverOperation::TombstoneCurrentRoot,
|
|
);
|
|
let remove_wpf = operation_index(
|
|
&host.mutation_order,
|
|
&CutoverOperation::UninstallTransactionWindowsPacketFilter,
|
|
);
|
|
assert!(start_legacy < delete_current);
|
|
assert!(start_legacy < remove_wpf);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::RolledBack
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn transaction_created_service_defaults_are_journaled_without_redundant_mutation() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_verify_current = true;
|
|
|
|
assert_eq!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
82,
|
|
)
|
|
.unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
let journal = store.load().unwrap().unwrap();
|
|
let already_satisfied: Vec<_> = journal
|
|
.mutations
|
|
.iter()
|
|
.filter(|record| {
|
|
record
|
|
.effect
|
|
.as_ref()
|
|
.is_some_and(|effect| effect.disposition == EffectDisposition::AlreadySatisfied)
|
|
})
|
|
.map(|record| record.operation.clone())
|
|
.collect();
|
|
assert!(
|
|
already_satisfied.contains(&CutoverOperation::RestoreLegacyServicePolicy(
|
|
ServiceConfig2Kind::FailureActions,
|
|
))
|
|
);
|
|
assert!(already_satisfied.contains(&CutoverOperation::RestoreLegacyServiceSecurity));
|
|
assert!(host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::RestoreLegacyServicePolicy(
|
|
ServiceConfig2Kind::Description
|
|
)));
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::RestoreLegacyServicePolicy(
|
|
ServiceConfig2Kind::FailureActions
|
|
)));
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::RestoreLegacyServiceSecurity));
|
|
assert!(!host
|
|
.mutation_order
|
|
.iter()
|
|
.any(|operation| matches!(operation, CutoverOperation::SetCurrentServicePolicy(_))));
|
|
}
|
|
|
|
#[test]
|
|
fn transaction_created_components_parent_is_deleted_last_and_preexisting_parent_is_preserved() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_verify_current = true;
|
|
assert_eq!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
84,
|
|
)
|
|
.unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
let tombstone = operation_index(
|
|
&host.mutation_order,
|
|
&CutoverOperation::TombstoneCurrentRoot,
|
|
);
|
|
let delete_root = operation_index(
|
|
&host.mutation_order,
|
|
&CutoverOperation::DeleteCurrentTombstoneRoot,
|
|
);
|
|
let delete_parent = operation_index(
|
|
&host.mutation_order,
|
|
&CutoverOperation::DeleteTransactionCurrentComponentsParent,
|
|
);
|
|
assert!(tombstone < delete_root && delete_root < delete_parent);
|
|
assert!(!host.machine.current_components_parent);
|
|
|
|
let mut present_before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
present_before.current_components_parent =
|
|
CurrentComponentsParentBeforeState::Present(SealedPathSnapshot {
|
|
identity: StableObjectIdentity {
|
|
volume_serial_number: 8,
|
|
file_id: 99,
|
|
kind: StableObjectKind::Directory,
|
|
size: 0,
|
|
},
|
|
security: security_snapshot(),
|
|
});
|
|
let present_store = MemoryStore::default();
|
|
let mut present_host = MockHost::new(present_before);
|
|
present_host.fail_verify_current = true;
|
|
begin_proxifyre_cutover(
|
|
&present_store,
|
|
&mut present_host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
85,
|
|
)
|
|
.unwrap();
|
|
assert!(present_host.machine.current_components_parent);
|
|
assert!(!present_host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::CreateCurrentComponentsParent));
|
|
assert!(!present_host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::DeleteTransactionCurrentComponentsParent));
|
|
}
|
|
|
|
#[test]
|
|
fn foreign_or_weak_components_parent_fails_capture_before_journal_or_mutation() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_capture_precondition = true;
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Stopped),
|
|
86,
|
|
),
|
|
Err(CutoverError::Host(CutoverHostError::PreconditionFailed))
|
|
));
|
|
assert_eq!(host.capture_calls, 1);
|
|
assert_eq!(host.mutation_calls, 0);
|
|
assert!(store.load().unwrap().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn rollback_failure_is_durable_and_retryable_without_claiming_rollback() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let plan = sample_plan(LegacyServiceState::Running);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_verify_current = true;
|
|
host.fail_once = Some(CutoverOperation::RestoreLegacyServicePolicy(
|
|
ServiceConfig2Kind::Description,
|
|
));
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(&store, &mut host, plan, 90,),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(failed.execution_mode, CutoverExecutionMode::RollingBack);
|
|
assert!(host.machine.legacy_root_live);
|
|
assert!(host.machine.current_root);
|
|
assert!(!host
|
|
.mutation_order
|
|
.contains(&CutoverOperation::TombstoneCurrentRoot));
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 91).unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
assert_eq!(
|
|
store.load().unwrap().unwrap().phase,
|
|
CutoverPhase::RolledBack
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn final_rollback_verification_failure_never_claims_rolled_back() {
|
|
let before = sample_before(LegacyServiceState::Running, true, true);
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
host.fail_verify_current = true;
|
|
host.fail_verify_rollback = true;
|
|
|
|
assert!(matches!(
|
|
begin_proxifyre_cutover(
|
|
&store,
|
|
&mut host,
|
|
sample_plan(LegacyServiceState::Running),
|
|
95,
|
|
),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let failed = store.load().unwrap().unwrap();
|
|
assert_eq!(failed.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(
|
|
failed.recovery_code.as_deref(),
|
|
Some("rollback_verification_failed")
|
|
);
|
|
assert_eq!(host.machine, MockMachine::from_before(&host.before));
|
|
|
|
host.fail_verify_rollback = false;
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 96).unwrap(),
|
|
CutoverRunOutcome::RolledBack
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn partial_cleanup_stays_pending_and_retry_completes_idempotently() {
|
|
let before = sample_before(LegacyServiceState::Stopped, true, true);
|
|
let plan = sample_plan(LegacyServiceState::Stopped);
|
|
let first_entry = before.filesystem.entries[0].relative_path.clone();
|
|
let store = MemoryStore::default();
|
|
let mut host = MockHost::new(before);
|
|
begin_proxifyre_cutover(&store, &mut host, plan, 100).unwrap();
|
|
let journal = store.load().unwrap().unwrap();
|
|
let evidence = sample_user_evidence(
|
|
&journal,
|
|
&host,
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
101,
|
|
false,
|
|
);
|
|
mark_proxifyre_next_start_verified(&store, &mut host, &evidence, 101).unwrap();
|
|
let confirmed = ComponentCutoverUserEvidence {
|
|
route_smoke_confirmed: true,
|
|
confirmed_at_epoch_seconds: Some(102),
|
|
..evidence
|
|
};
|
|
confirm_proxifyre_cleanup(&store, &confirmed, 102).unwrap();
|
|
host.fail_once = Some(CutoverOperation::DeleteQuarantineEntry(first_entry));
|
|
|
|
assert_eq!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 103).unwrap(),
|
|
CutoverRunOutcome::CleanupPending
|
|
);
|
|
let pending = store.load().unwrap().unwrap();
|
|
assert_eq!(pending.phase, CutoverPhase::CleanupConfirmed);
|
|
assert_eq!(pending.execution_mode, CutoverExecutionMode::CleaningUp);
|
|
assert!(host.machine.tombstone_live);
|
|
|
|
host.fail_reacquire_once = true;
|
|
let mutations_before_reacquire = host.mutation_calls;
|
|
assert!(matches!(
|
|
recover_proxifyre_cutover(&store, &mut host, 104),
|
|
Err(CutoverError::RecoveryRequired)
|
|
));
|
|
let recovery = store.load().unwrap().unwrap();
|
|
assert_eq!(recovery.phase, CutoverPhase::RecoveryRequired);
|
|
assert_eq!(recovery.execution_mode, CutoverExecutionMode::CleaningUp);
|
|
assert_eq!(host.mutation_calls, mutations_before_reacquire);
|
|
|
|
assert_eq!(
|
|
recover_proxifyre_cutover(&store, &mut host, 105).unwrap(),
|
|
CutoverRunOutcome::Complete
|
|
);
|
|
assert_eq!(
|
|
cleanup_proxifyre_quarantine(&store, &mut host, 106)
|
|
.unwrap_err()
|
|
.to_string(),
|
|
CutoverError::StateConflict.to_string()
|
|
);
|
|
assert!(!host.machine.tombstone_live);
|
|
}
|
|
|
|
fn sample_plan(state: LegacyServiceState) -> ProxifyreCutoverPlan {
|
|
ProxifyreCutoverPlan::new(
|
|
Path::new(r"C:\Program Files\ProxyWarden"),
|
|
PathBuf::from(r"C:\Tools\ProxiFyre"),
|
|
state,
|
|
"2.4.0".to_string(),
|
|
hash('a'),
|
|
hash('b'),
|
|
hash('c'),
|
|
Uuid::new_v4().hyphenated().to_string(),
|
|
)
|
|
}
|
|
|
|
fn sample_user_evidence(
|
|
journal: &CutoverJournal,
|
|
host: &MockHost,
|
|
startup_session_id: String,
|
|
observed_at_epoch_seconds: u64,
|
|
route_smoke_confirmed: bool,
|
|
) -> ComponentCutoverUserEvidence {
|
|
ComponentCutoverUserEvidence {
|
|
schema_version: CUTOVER_USER_EVIDENCE_SCHEMA_VERSION,
|
|
cutover_id: journal.plan.cutover_id.clone(),
|
|
startup_session_id,
|
|
current_inventory_fingerprint: host
|
|
.machine
|
|
.current_inventory_fingerprint()
|
|
.as_str()
|
|
.to_string(),
|
|
route_smoke_confirmed,
|
|
observed_at_epoch_seconds,
|
|
confirmed_at_epoch_seconds: route_smoke_confirmed.then_some(observed_at_epoch_seconds),
|
|
}
|
|
}
|
|
|
|
fn sample_cutover_proof_result(
|
|
plan: &ProxifyreCutoverPlan,
|
|
before: &LegacyBeforeState,
|
|
) -> Result<LegacyCutoverProof, InventoryIssue> {
|
|
let executable = plan.legacy_root.join("ProxiFyre.exe");
|
|
let status = match before.service.original_state {
|
|
ServiceStableState::Running => "running",
|
|
ServiceStableState::Stopped => "stopped",
|
|
};
|
|
let inventory = classify_component_candidates(
|
|
ComponentId::Proxyfier,
|
|
vec![ComponentCandidateProbe {
|
|
component_id: ComponentId::Proxyfier,
|
|
role: CandidateRole::Legacy,
|
|
root: plan.legacy_root.clone(),
|
|
root_exists: true,
|
|
has_reparse_point: false,
|
|
executable_path: Some(executable.clone()),
|
|
missing_files: Vec::new(),
|
|
marker: MarkerEvidence::NotRequired,
|
|
marker_required: false,
|
|
binary_identity: BinaryIdentityEvidence::KnownPackage,
|
|
binary_version: Some(plan.legacy_version.clone()),
|
|
service: Some(ServiceEvidence {
|
|
name: before.service.service_name.clone(),
|
|
status: status.to_string(),
|
|
path_name: Some(before.service.base.binary_path_name.clone()),
|
|
executable_path: Some(executable.clone()),
|
|
path_matches_candidate: legacy_proxifyre_topshelf_path_matches(
|
|
&before.service.base.binary_path_name,
|
|
&executable,
|
|
),
|
|
binary_version: Some(plan.legacy_version.clone()),
|
|
}),
|
|
service_required: true,
|
|
legacy_identity_complete: true,
|
|
}],
|
|
);
|
|
let immutable_files: Vec<_> = LEGACY_PROXIFYRE_2_2_1_MANIFEST
|
|
.iter()
|
|
.filter_map(|expected| {
|
|
before
|
|
.filesystem
|
|
.entries
|
|
.iter()
|
|
.find(|entry| entry.relative_path == Path::new(expected.relative_path))
|
|
.map(|entry| LegacyPackageFileIdentity {
|
|
relative_path: entry.relative_path.clone(),
|
|
size: entry.identity.size,
|
|
sha256: entry.sha256.clone().unwrap_or_default(),
|
|
})
|
|
})
|
|
.collect();
|
|
let evidence = LegacyCutoverEvidence {
|
|
proxifyre_manifest_matches: matches_legacy_proxifyre_2_2_1_manifest(&immutable_files),
|
|
proxifyre_scm_profile: legacy_proxifyre_scm_profile_from_snapshot(&before.service),
|
|
proxifyre_scm_snapshot_fingerprint: legacy_service_snapshot_fingerprint(&before.service),
|
|
additional_matching_service: before.additional_matching_service,
|
|
};
|
|
prove_legacy_cutover(&inventory, &evidence)
|
|
}
|
|
|
|
fn sample_before(
|
|
state: LegacyServiceState,
|
|
windows_packet_filter_present: bool,
|
|
vc_runtime_present: bool,
|
|
) -> LegacyBeforeState {
|
|
let mut entries: Vec<_> = LEGACY_PROXIFYRE_2_2_1_MANIFEST
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(index, expected)| LegacyManifestEntry {
|
|
relative_path: PathBuf::from(expected.relative_path),
|
|
identity: file_identity(index as u64 + 10, expected.size),
|
|
sha256: Some(expected.sha256.to_string()),
|
|
security: security_snapshot(),
|
|
})
|
|
.collect();
|
|
let config_entry = LegacyManifestEntry {
|
|
relative_path: PathBuf::from("app-config.json"),
|
|
identity: file_identity(100, 128),
|
|
sha256: Some(hash('b')),
|
|
security: security_snapshot(),
|
|
};
|
|
entries.push(config_entry.clone());
|
|
LegacyBeforeState {
|
|
identity: ProxifyreLegacyIdentity::V2_2_1ToolsPrimaryService,
|
|
identity_fingerprint: hash('e'),
|
|
additional_matching_service: false,
|
|
service: sample_service(state),
|
|
filesystem: LegacyFilesystemSnapshot {
|
|
root: SealedPathSnapshot {
|
|
identity: StableObjectIdentity {
|
|
volume_serial_number: 7,
|
|
file_id: 1,
|
|
kind: StableObjectKind::Directory,
|
|
size: 0,
|
|
},
|
|
security: security_snapshot(),
|
|
},
|
|
config: SealedPathSnapshot {
|
|
identity: config_entry.identity.clone(),
|
|
security: config_entry.security.clone(),
|
|
},
|
|
config_relative_path: PathBuf::from("app-config.json"),
|
|
entries,
|
|
},
|
|
current_components_parent: CurrentComponentsParentBeforeState::Absent,
|
|
current_candidate: sample_current_candidate(),
|
|
prerequisites: PrerequisiteBeforeState {
|
|
windows_packet_filter_present,
|
|
vc_runtime_present,
|
|
},
|
|
package_fingerprint: hash('a'),
|
|
config_fingerprint: hash('b'),
|
|
}
|
|
}
|
|
|
|
fn sample_current_candidate() -> CurrentCandidateSnapshot {
|
|
let mut files: Vec<_> = CURRENT_PROXIFYRE_PACKAGE_FILES
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(index, relative_path)| CurrentCandidateFile {
|
|
relative_path: PathBuf::from(relative_path),
|
|
role: CurrentCandidateFileRole::Package,
|
|
size: 2_048 + index as u64,
|
|
sha256: hash('c'),
|
|
})
|
|
.collect();
|
|
files.extend([
|
|
CurrentCandidateFile {
|
|
relative_path: PathBuf::from("app-config.json"),
|
|
role: CurrentCandidateFileRole::Config,
|
|
size: 128,
|
|
sha256: hash('b'),
|
|
},
|
|
CurrentCandidateFile {
|
|
relative_path: PathBuf::from("proxywarden-component.json"),
|
|
role: CurrentCandidateFileRole::Marker,
|
|
size: 256,
|
|
sha256: hash('d'),
|
|
},
|
|
CurrentCandidateFile {
|
|
relative_path: PathBuf::from("proxywarden-install-receipt.json"),
|
|
role: CurrentCandidateFileRole::Receipt,
|
|
size: 256,
|
|
sha256: hash('f'),
|
|
},
|
|
]);
|
|
CurrentCandidateSnapshot {
|
|
manifest_fingerprint: current_candidate_manifest_fingerprint(&files)
|
|
.expect("sample current candidate must be valid"),
|
|
files,
|
|
}
|
|
}
|
|
|
|
fn sample_service(state: LegacyServiceState) -> FullServiceSnapshot {
|
|
FullServiceSnapshot {
|
|
service_name: "ProxiFyreService".to_string(),
|
|
base: ServiceBaseConfigSnapshot {
|
|
service_type: 0x10,
|
|
start_type: 2,
|
|
error_control: 1,
|
|
binary_path_name: r#"C:\Tools\ProxiFyre\ProxiFyre.exe -displayname "ProxiFyre Service" -servicename "ProxiFyreService""#.to_string(),
|
|
load_order_group: None,
|
|
tag_id: 0,
|
|
dependencies: Vec::new(),
|
|
service_start_name: "LocalSystem".to_string(),
|
|
display_name: "ProxiFyre Service".to_string(),
|
|
},
|
|
config2: vec![
|
|
ServiceConfig2Snapshot::Description(Some(
|
|
"ProxiFyre - SOCKS5 ProxiFyre Service".to_string(),
|
|
)),
|
|
ServiceConfig2Snapshot::FailureActions(ServiceFailureActionsSnapshot {
|
|
reset_period_seconds: 0,
|
|
reboot_message: None,
|
|
command: None,
|
|
actions: Vec::new(),
|
|
}),
|
|
ServiceConfig2Snapshot::FailureActionsFlag(false),
|
|
ServiceConfig2Snapshot::DelayedAutoStart(false),
|
|
ServiceConfig2Snapshot::SidType(0),
|
|
ServiceConfig2Snapshot::RequiredPrivileges(Vec::new()),
|
|
ServiceConfig2Snapshot::PreshutdownTimeout(180_000),
|
|
ServiceConfig2Snapshot::Triggers(Vec::new()),
|
|
ServiceConfig2Snapshot::PreferredNode {
|
|
node: 0,
|
|
delete: false,
|
|
},
|
|
ServiceConfig2Snapshot::LaunchProtected(0),
|
|
],
|
|
security: ServiceSecuritySnapshot {
|
|
self_relative_descriptor: vec![1, 2, 3, 4],
|
|
untrusted_mutation_rights: false,
|
|
},
|
|
original_state: match state {
|
|
LegacyServiceState::Running => ServiceStableState::Running,
|
|
LegacyServiceState::Stopped => ServiceStableState::Stopped,
|
|
},
|
|
}
|
|
}
|
|
|
|
fn security_snapshot() -> SecurityDescriptorSnapshot {
|
|
serde_json::from_value(serde_json::json!({
|
|
"selfRelative": [1, 2, 3, 4],
|
|
"sacl": "absent"
|
|
}))
|
|
.unwrap()
|
|
}
|
|
|
|
fn file_identity(file_id: u64, size: u64) -> StableObjectIdentity {
|
|
StableObjectIdentity {
|
|
volume_serial_number: 7,
|
|
file_id,
|
|
kind: StableObjectKind::RegularFile,
|
|
size,
|
|
}
|
|
}
|
|
|
|
fn hash(value: char) -> String {
|
|
std::iter::repeat_n(value, 64).collect()
|
|
}
|
|
|
|
fn policy_key(kind: &ServiceConfig2Kind) -> String {
|
|
format!("{kind:?}")
|
|
}
|
|
|
|
fn path_key(path: &Path) -> String {
|
|
path.as_os_str().to_string_lossy().replace('\\', "/")
|
|
}
|
|
|
|
fn operation_index(order: &[CutoverOperation], expected: &CutoverOperation) -> usize {
|
|
order
|
|
.iter()
|
|
.position(|operation| operation == expected)
|
|
.expect("operation must be present")
|
|
}
|