using GatewayKeepalive.Core; namespace GatewayKeepalive.Tests; internal static class RecoveryRegressionTests { public static async Task RestorePauseWriteFailurePreventsNetworkWritesAsync() { using var h = new Harness(); await h.AdoptProxyAsync(); var before = h.Backend.Current; var sessionBytes = File.ReadAllBytes(h.PathFor("session.json")); File.Delete(h.PathFor("settings.json")); Directory.CreateDirectory(h.PathFor("settings.json")); h.Backend.Writes.Clear(); await Check.ThrowsAsync(() => h.Controller.RestoreAsync(), "Pause persistence failure must abort restore"); Check.Equal(0, h.Backend.Writes.Count, "No network operation may precede durable Pause"); Check.True(NetworkComparison.SameConfiguration(before, h.Backend.Current), "Failed Pause write must preserve the entire live configuration"); Check.SequenceEqual(sessionBytes, File.ReadAllBytes(h.PathFor("session.json")), "Failed Pause must not update session metadata"); Check.Equal(null, h.Store.Read(SwitchService.JournalFile), "Failed Pause must not start a restore transaction"); } public static async Task RestorePersistsPauseBeforeWritesAndRestartAsync() { using var h = new Harness(); await h.AdoptProxyAsync(); h.Backend.Writes.Clear(); h.Backend.RouteWriteBehavior = desired => { Check.Equal(ControlPolicy.Paused, h.Store.Read("settings.json")?.Policy, "Pause must be durable before the first route write"); h.Backend.ApplyRoutes(desired); return Task.CompletedTask; }; h.Backend.DnsWriteBehavior = desired => { Check.Equal(ControlPolicy.Paused, h.Store.Read("settings.json")?.Policy, "Pause must be durable before DNS writes"); h.Backend.ApplyDns(desired); return Task.CompletedTask; }; await h.Controller.RestoreAsync(); Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "Restore must reach the exact original baseline"); Check.Equal(null, h.Store.Read(SwitchService.JournalFile), "Committed restore should clear its journal"); var writes = h.Backend.Writes.Count; var restarted = h.Restart(); await restarted.RefreshAsync(); Check.Equal(ControlPolicy.Paused, restarted.Settings.Policy, "Restart must load Pause rather than the former Auto policy"); Check.Equal(ControlPolicy.Paused, restarted.Status.Policy, "Healthy recovery candidates must remain paused after restart"); Check.Equal(writes, h.Backend.Writes.Count, "Restart must not automatically undo a successful restore"); } public static Task RestoreSessionFailureRetainsBaselineJournalAsync() => RestoreSessionFailureAsync(false); public static Task RestoreSessionFailureRetainsRollbackJournalAsync() => RestoreSessionFailureAsync(true); private static async Task RestoreSessionFailureAsync(bool pendingJournal) { using var h = new Harness(); await h.AdoptProxyAsync(); var proxy = h.Backend.Current; if (pendingJournal) { // A failed direct -> proxy transaction must roll back to the original direct state. h.Store.Write(SwitchService.JournalFile, new SwitchJournal(h.Baseline, proxy, "ApplyingDns", DateTimeOffset.UtcNow)); } var oldSessionBytes = File.ReadAllBytes(h.PathFor("session.json")); var blockedMetadata = false; h.Backend.CaptureBehavior = (_, _) => { if (!blockedMetadata && NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current)) { blockedMetadata = true; File.Delete(h.PathFor("session.json")); Directory.CreateDirectory(h.PathFor("session.json")); } return Task.FromResult(h.Backend.Current); }; await Check.ThrowsAsync(() => h.Controller.RestoreAsync(), "A session commit failure should be observable"); Check.True(blockedMetadata, "The injected failure must occur after network restoration"); Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "The network must already be at the restored baseline"); Check.Equal(ControlPolicy.Paused, h.Store.Read("settings.json")?.Policy, "Session failure must leave Pause durable"); var journal = h.Store.Read(SwitchService.JournalFile); Check.True(journal is not null, "Uncommitted session metadata must retain its recovery journal"); Check.Equal(pendingJournal ? "RolledBack" : "Verified", journal!.Stage, "The journal must record the completed network phase until session commit"); // Simulate a restart seeing the last durable session rather than the uncommitted one. Directory.Delete(h.PathFor("session.json")); File.WriteAllBytes(h.PathFor("session.json"), oldSessionBytes); h.Backend.CaptureBehavior = null; var writes = h.Backend.Writes.Count; var restarted = h.Restart(); await restarted.RefreshAsync(); Check.Equal(ControlPolicy.Paused, restarted.Settings.Policy, "Restart after a metadata failure must load Pause"); Check.Equal(NetworkMode.RecoveryRequired, restarted.Status.Mode, "Uncommitted restoration must require explicit recovery"); Check.Equal(writes, h.Backend.Writes.Count, "Restart must protect the restored state from Auto"); Check.True(h.Store.Read(SwitchService.JournalFile) is not null, "Refresh must retain the uncommitted journal"); } public static async Task StopPendingJournalContinuesToOriginalBaselineAsync() { using var h = new Harness(); await h.AdoptProxyAsync(restoreOnExit: true); var proxy = h.Backend.Current; // Recovery first returns to Before=Proxy. Stop must then restore Baseline=Direct. h.Store.Write(SwitchService.JournalFile, new SwitchJournal(proxy, h.Baseline, "Verified", DateTimeOffset.UtcNow)); h.Backend.Current = h.Baseline; h.Backend.Writes.Clear(); var restarted = h.Restart(); await restarted.StopAsync(); Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "Stop must continue beyond journal.Before=Proxy to the original direct baseline"); Check.Equal(ControlPolicy.Paused, h.Store.Read("settings.json")?.Policy, "Stop must persist Pause"); Check.Equal(ControlPolicy.Paused, restarted.Status.Policy, "Stop must publish paused state"); var session = h.Store.Read("session.json"); Check.True(session is not null, "Stop must persist the final session"); Check.True(NetworkComparison.SameConfiguration(h.Baseline, session!.Expected), "Final session must describe the direct baseline"); Check.True(NetworkComparison.SameConfiguration(h.Baseline, session.Baseline), "Original baseline must survive journal recovery"); Check.Equal(null, h.Store.Read(SwitchService.JournalFile), "Both recovery phases must commit before clearing the final journal"); Check.SequenceEqual( [new BackendWrite("route", "192.168.6.2"), new BackendWrite("dns", "192.168.6.2"), new BackendWrite("route", "192.168.6.1"), new BackendWrite("dns", "auto")], h.Backend.Writes, "Stop should first recover the pending transaction and then restore its baseline"); } public static Task ForeignRouteAfterPreflightIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: false, changeDns: false); public static Task ForeignDnsAfterPreflightIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: false, changeDns: true); public static Task ForeignRouteBeforeDnsIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: true, changeDns: false); public static Task ForeignDnsBeforeDnsIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: true, changeDns: true); private static async Task ForeignChangeDuringSwitchAsync(bool afterRoutes, bool changeDns) { var before = Fixtures.Baseline(); using var h = new SwitchHarness(before); var foreignRoute = before.Routes[0] with { Gateway = "192.168.6.254", Metric = 0 }; var foreignDns = new DnsConfiguration(false, ["192.168.6.253"], []); NetworkSnapshot? foreignState = null; void ChangeExternally() { h.Backend.Current = changeDns ? h.Backend.Current with { Dns = foreignDns } : h.Backend.Current with { Routes = [.. h.Backend.Current.Routes, foreignRoute] }; foreignState = h.Backend.Current; } if (afterRoutes) { h.Backend.RouteWriteBehavior = desired => { h.Backend.ApplyRoutes(desired); ChangeExternally(); return Task.CompletedTask; }; } else { h.Backend.CaptureBehavior = (_, _) => { if (h.Backend.CaptureCount == 2) ChangeExternally(); return Task.FromResult(h.Backend.Current); }; } var failure = await Check.ThrowsAsync( () => h.Service.SwitchAsync(Fixtures.Settings(), before, true, null, CancellationToken.None), "A foreign change after preflight must stop guarded writes and require recovery"); Check.True(foreignState is not null, "The external change must actually be injected"); Check.True(NetworkComparison.SameConfiguration(foreignState!, h.Backend.Current), "Forward and rollback paths must preserve the complete foreign configuration"); Check.SequenceEqual(afterRoutes ? [new BackendWrite("route", "192.168.6.2")] : Array.Empty(), h.Backend.Writes, "No DNS or compensation write may overwrite the foreign configuration"); var journal = h.Store.Read(SwitchService.JournalFile); Check.True(journal is not null, "Foreign changes must leave a journal for explicit recovery"); Check.Equal(afterRoutes ? "ApplyingDns" : "ApplyingRoutes", journal!.Stage, "The journal must identify the blocked phase"); Check.True(NetworkComparison.SameConfiguration(before, journal.Before), "The journal must preserve the genuine pre-transaction snapshot"); Check.True(failure.InnerException is AggregateException, "Both guarded apply and ownership refusal must be retained"); Check.Equal(0, h.Probe.VerifyCount, "Rejected guarded writes must not reach egress verification"); } private sealed class Harness : IDisposable { public TempDirectory Temp { get; } = new(); public NetworkSnapshot Baseline { get; } = Fixtures.Baseline(); public FakeBackend Backend { get; } public FakeProbe Probe { get; } = new(); public JsonStore Store { get; } public GatewayController Controller { get; } public List Logs { get; } = []; public Harness() { Backend = new FakeBackend(Baseline); Store = new JsonStore(Temp.DirectoryPath); Controller = Restart(); } public string PathFor(string name) => Path.Combine(Temp.DirectoryPath, name); public GatewayController Restart() => new(Backend, Probe, Store, Logs.Add); public async Task AdoptProxyAsync(bool restoreOnExit = false) { await Controller.SaveSettingsAsync(Fixtures.Settings() with { Policy = ControlPolicy.Paused, RestoreOnExit = restoreOnExit, SuccessThreshold = 1, RecoveryCooldownSeconds = 0, }); await Controller.SetPolicyAsync(ControlPolicy.ForceProxy); await Controller.SetPolicyAsync(ControlPolicy.Auto); Check.Equal(ControlPolicy.Auto, Store.Read("settings.json")?.Policy, "Regression setup must begin with durable Auto"); } public void Dispose() => Temp.Dispose(); } }