Gateway-keepaliveForDotnet/tests/GatewayKeepalive.Tests/RecoveryRegressionTests.cs

using GatewayKeepalive.Core;

namespace GatewayKeepalive.Tests;

internal static class RecoveryRegressionTests
{
    public static async Task RestorePauseWriteFailurePreventsNetworkWritesAsync()
    {
        using var h = new Harness();
        await h.AdoptProxyAsync();
        var before = h.Backend.Current;
        var sessionBytes = File.ReadAllBytes(h.PathFor("session.json"));
        File.Delete(h.PathFor("settings.json"));
        Directory.CreateDirectory(h.PathFor("settings.json"));
        h.Backend.Writes.Clear();

        await Check.ThrowsAsync<IOException>(() => h.Controller.RestoreAsync(), "Pause persistence failure must abort restore");

        Check.Equal(0, h.Backend.Writes.Count, "No network operation may precede durable Pause");
        Check.True(NetworkComparison.SameConfiguration(before, h.Backend.Current), "Failed Pause write must preserve the entire live configuration");
        Check.SequenceEqual(sessionBytes, File.ReadAllBytes(h.PathFor("session.json")), "Failed Pause must not update session metadata");
        Check.Equal<SwitchJournal?>(null, h.Store.Read<SwitchJournal>(SwitchService.JournalFile), "Failed Pause must not start a restore transaction");
    }

    public static async Task RestorePersistsPauseBeforeWritesAndRestartAsync()
    {
        using var h = new Harness();
        await h.AdoptProxyAsync();
        h.Backend.Writes.Clear();
        h.Backend.RouteWriteBehavior = desired =>
        {
            Check.Equal(ControlPolicy.Paused, h.Store.Read<AppSettings>("settings.json")?.Policy, "Pause must be durable before the first route write");
            h.Backend.ApplyRoutes(desired);
            return Task.CompletedTask;
        };
        h.Backend.DnsWriteBehavior = desired =>
        {
            Check.Equal(ControlPolicy.Paused, h.Store.Read<AppSettings>("settings.json")?.Policy, "Pause must be durable before DNS writes");
            h.Backend.ApplyDns(desired);
            return Task.CompletedTask;
        };

        await h.Controller.RestoreAsync();
        Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "Restore must reach the exact original baseline");
        Check.Equal<SwitchJournal?>(null, h.Store.Read<SwitchJournal>(SwitchService.JournalFile), "Committed restore should clear its journal");
        var writes = h.Backend.Writes.Count;
        var restarted = h.Restart();
        await restarted.RefreshAsync();
        Check.Equal(ControlPolicy.Paused, restarted.Settings.Policy, "Restart must load Pause rather than the former Auto policy");
        Check.Equal(ControlPolicy.Paused, restarted.Status.Policy, "Healthy recovery candidates must remain paused after restart");
        Check.Equal(writes, h.Backend.Writes.Count, "Restart must not automatically undo a successful restore");
    }

    public static Task RestoreSessionFailureRetainsBaselineJournalAsync() => RestoreSessionFailureAsync(false);
    public static Task RestoreSessionFailureRetainsRollbackJournalAsync() => RestoreSessionFailureAsync(true);

    private static async Task RestoreSessionFailureAsync(bool pendingJournal)
    {
        using var h = new Harness();
        await h.AdoptProxyAsync();
        var proxy = h.Backend.Current;
        if (pendingJournal)
        {
            // A failed direct -> proxy transaction must roll back to the original direct state.
            h.Store.Write(SwitchService.JournalFile, new SwitchJournal(h.Baseline, proxy, "ApplyingDns", DateTimeOffset.UtcNow));
        }
        var oldSessionBytes = File.ReadAllBytes(h.PathFor("session.json"));
        var blockedMetadata = false;
        h.Backend.CaptureBehavior = (_, _) =>
        {
            if (!blockedMetadata && NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current))
            {
                blockedMetadata = true;
                File.Delete(h.PathFor("session.json"));
                Directory.CreateDirectory(h.PathFor("session.json"));
            }
            return Task.FromResult(h.Backend.Current);
        };

        await Check.ThrowsAsync<IOException>(() => h.Controller.RestoreAsync(), "A session commit failure should be observable");

        Check.True(blockedMetadata, "The injected failure must occur after network restoration");
        Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "The network must already be at the restored baseline");
        Check.Equal(ControlPolicy.Paused, h.Store.Read<AppSettings>("settings.json")?.Policy, "Session failure must leave Pause durable");
        var journal = h.Store.Read<SwitchJournal>(SwitchService.JournalFile);
        Check.True(journal is not null, "Uncommitted session metadata must retain its recovery journal");
        Check.Equal(pendingJournal ? "RolledBack" : "Verified", journal!.Stage, "The journal must record the completed network phase until session commit");

        // Simulate a restart seeing the last durable session rather than the uncommitted one.
        Directory.Delete(h.PathFor("session.json"));
        File.WriteAllBytes(h.PathFor("session.json"), oldSessionBytes);
        h.Backend.CaptureBehavior = null;
        var writes = h.Backend.Writes.Count;
        var restarted = h.Restart();
        await restarted.RefreshAsync();
        Check.Equal(ControlPolicy.Paused, restarted.Settings.Policy, "Restart after a metadata failure must load Pause");
        Check.Equal(NetworkMode.RecoveryRequired, restarted.Status.Mode, "Uncommitted restoration must require explicit recovery");
        Check.Equal(writes, h.Backend.Writes.Count, "Restart must protect the restored state from Auto");
        Check.True(h.Store.Read<SwitchJournal>(SwitchService.JournalFile) is not null, "Refresh must retain the uncommitted journal");
    }

    public static async Task StopPendingJournalContinuesToOriginalBaselineAsync()
    {
        using var h = new Harness();
        await h.AdoptProxyAsync(restoreOnExit: true);
        var proxy = h.Backend.Current;
        // Recovery first returns to Before=Proxy. Stop must then restore Baseline=Direct.
        h.Store.Write(SwitchService.JournalFile, new SwitchJournal(proxy, h.Baseline, "Verified", DateTimeOffset.UtcNow));
        h.Backend.Current = h.Baseline;
        h.Backend.Writes.Clear();
        var restarted = h.Restart();

        await restarted.StopAsync();

        Check.True(NetworkComparison.SameConfiguration(h.Baseline, h.Backend.Current), "Stop must continue beyond journal.Before=Proxy to the original direct baseline");
        Check.Equal(ControlPolicy.Paused, h.Store.Read<AppSettings>("settings.json")?.Policy, "Stop must persist Pause");
        Check.Equal(ControlPolicy.Paused, restarted.Status.Policy, "Stop must publish paused state");
        var session = h.Store.Read<SessionState>("session.json");
        Check.True(session is not null, "Stop must persist the final session");
        Check.True(NetworkComparison.SameConfiguration(h.Baseline, session!.Expected), "Final session must describe the direct baseline");
        Check.True(NetworkComparison.SameConfiguration(h.Baseline, session.Baseline), "Original baseline must survive journal recovery");
        Check.Equal<SwitchJournal?>(null, h.Store.Read<SwitchJournal>(SwitchService.JournalFile), "Both recovery phases must commit before clearing the final journal");
        Check.SequenceEqual(
            [new BackendWrite("route", "192.168.6.2"), new BackendWrite("dns", "192.168.6.2"), new BackendWrite("route", "192.168.6.1"), new BackendWrite("dns", "auto")],
            h.Backend.Writes, "Stop should first recover the pending transaction and then restore its baseline");
    }

    public static Task ForeignRouteAfterPreflightIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: false, changeDns: false);
    public static Task ForeignDnsAfterPreflightIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: false, changeDns: true);
    public static Task ForeignRouteBeforeDnsIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: true, changeDns: false);
    public static Task ForeignDnsBeforeDnsIsProtectedAsync() => ForeignChangeDuringSwitchAsync(afterRoutes: true, changeDns: true);

    private static async Task ForeignChangeDuringSwitchAsync(bool afterRoutes, bool changeDns)
    {
        var before = Fixtures.Baseline();
        using var h = new SwitchHarness(before);
        var foreignRoute = before.Routes[0] with { Gateway = "192.168.6.254", Metric = 0 };
        var foreignDns = new DnsConfiguration(false, ["192.168.6.253"], []);
        NetworkSnapshot? foreignState = null;
        void ChangeExternally()
        {
            h.Backend.Current = changeDns
                ? h.Backend.Current with { Dns = foreignDns }
                : h.Backend.Current with { Routes = [.. h.Backend.Current.Routes, foreignRoute] };
            foreignState = h.Backend.Current;
        }

        if (afterRoutes)
        {
            h.Backend.RouteWriteBehavior = desired =>
            {
                h.Backend.ApplyRoutes(desired);
                ChangeExternally();
                return Task.CompletedTask;
            };
        }
        else
        {
            h.Backend.CaptureBehavior = (_, _) =>
            {
                if (h.Backend.CaptureCount == 2) ChangeExternally();
                return Task.FromResult(h.Backend.Current);
            };
        }

        var failure = await Check.ThrowsAsync<RecoveryRequiredException>(
            () => h.Service.SwitchAsync(Fixtures.Settings(), before, true, null, CancellationToken.None),
            "A foreign change after preflight must stop guarded writes and require recovery");

        Check.True(foreignState is not null, "The external change must actually be injected");
        Check.True(NetworkComparison.SameConfiguration(foreignState!, h.Backend.Current), "Forward and rollback paths must preserve the complete foreign configuration");
        Check.SequenceEqual(afterRoutes ? [new BackendWrite("route", "192.168.6.2")] : Array.Empty<BackendWrite>(), h.Backend.Writes,
            "No DNS or compensation write may overwrite the foreign configuration");
        var journal = h.Store.Read<SwitchJournal>(SwitchService.JournalFile);
        Check.True(journal is not null, "Foreign changes must leave a journal for explicit recovery");
        Check.Equal(afterRoutes ? "ApplyingDns" : "ApplyingRoutes", journal!.Stage, "The journal must identify the blocked phase");
        Check.True(NetworkComparison.SameConfiguration(before, journal.Before), "The journal must preserve the genuine pre-transaction snapshot");
        Check.True(failure.InnerException is AggregateException, "Both guarded apply and ownership refusal must be retained");
        Check.Equal(0, h.Probe.VerifyCount, "Rejected guarded writes must not reach egress verification");
    }

    private sealed class Harness : IDisposable
    {
        public TempDirectory Temp { get; } = new();
        public NetworkSnapshot Baseline { get; } = Fixtures.Baseline();
        public FakeBackend Backend { get; }
        public FakeProbe Probe { get; } = new();
        public JsonStore Store { get; }
        public GatewayController Controller { get; }
        public List<string> Logs { get; } = [];

        public Harness()
        {
            Backend = new FakeBackend(Baseline);
            Store = new JsonStore(Temp.DirectoryPath);
            Controller = Restart();
        }

        public string PathFor(string name) => Path.Combine(Temp.DirectoryPath, name);
        public GatewayController Restart() => new(Backend, Probe, Store, Logs.Add);

        public async Task AdoptProxyAsync(bool restoreOnExit = false)
        {
            await Controller.SaveSettingsAsync(Fixtures.Settings() with
            {
                Policy = ControlPolicy.Paused, RestoreOnExit = restoreOnExit,
                SuccessThreshold = 1, RecoveryCooldownSeconds = 0,
            });
            await Controller.SetPolicyAsync(ControlPolicy.ForceProxy);
            await Controller.SetPolicyAsync(ControlPolicy.Auto);
            Check.Equal(ControlPolicy.Auto, Store.Read<AppSettings>("settings.json")?.Policy, "Regression setup must begin with durable Auto");
        }

        public void Dispose() => Temp.Dispose();
    }
}