using AutoForge.Application.Features.Vehicles.Calibration; using AutoForge.Application.Features.Vehicles.Telemetry.Recording; using AutoForge.Simulation; namespace AutoForge.IntegrationTests; public sealed partial class CalibrationRehearsalTests { [Theory] [InlineData(false, "evaluate")] [InlineData(false, "save")] [InlineData(true, "evaluate")] [InlineData(true, "save")] public async Task ComparisonOwnershipTail_RealCancellationCallbackReservesAdmissionUntilRelease( bool failReadback, string contender) { using var fixture = new Fixture(dynamics: true); await using var rehearsal = fixture.Session; var (operation, head) = await PrepareComparisonAsync(fixture); using var store = new CompletionTailStore(failReadback); await using var comparison = CreateComparison(fixture, store); await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); var result = comparison.Snapshot.Result; var releaseEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion); store.ReadState = () => comparison.Snapshot; var save = comparison.SaveCapturesAsync(); Task? cancellation = null; Task? competing = null; try { await store.ReadbackEntered.Task.WaitAsync(Deadline); cancellation = Task.Run(() => comparison.CancelAsync()); await store.CallbackEntered.Task.WaitAsync(Deadline); Assert.True(store.CallbackState!.IsBusy); store.ReleaseReadback.TrySetResult(); Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline)); var tail = comparison.Snapshot; var reads = fixture.Port.ReadCount; var writes = fixture.Port.WriteCount; var configurations = fixture.Configurations.List().Count; CalibrationComparisonException? rejection = null; try { competing = contender == "evaluate" ? comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier) : comparison.SaveCapturesAsync(); } catch (CalibrationComparisonException exception) { rejection = exception; } Assert.True(rejection?.Failure == CalibrationComparisonFailure.Busy, $"tail busy={tail.IsBusy}; state={tail.State}; old drain completed={save.IsCompleted}; " + $"callback exited={store.CallbackExited}; {contender} admitted={competing is not null}"); Assert.True(tail.IsBusy); Assert.Equal(CalibrationComparisonState.Saving, tail.State); Assert.False(save.IsCompleted); Assert.False(store.CallbackExited); Assert.Equal(reads, fixture.Port.ReadCount); Assert.Equal(writes, fixture.Port.WriteCount); Assert.Equal(configurations, fixture.Configurations.List().Count); Assert.Equal(2, store.Inner.SaveCalls); Assert.Same(result, comparison.Snapshot.Result); } finally { store.ReleaseReadback.TrySetResult(); store.ReleaseCallback.TrySetResult(); await ObserveTailTaskAsync(save); if (cancellation is not null) await ObserveTailTaskAsync(cancellation); if (competing is not null) await ObserveTailTaskAsync(competing); } Assert.True(store.CallbackExited); Assert.Equal(1, store.CallbackCalls); Assert.False(comparison.Snapshot.IsBusy); Assert.Equal(failReadback ? CalibrationComparisonState.Failed : CalibrationComparisonState.Completed, comparison.Snapshot.State); Assert.True(comparison.Snapshot.Saves[0].Persisted && comparison.Snapshot.Saves[0].ReadbackVerified); Assert.True(comparison.Snapshot.Saves[1].Persisted); Assert.Equal(!failReadback, comparison.Snapshot.Saves[1].ReadbackVerified); if (failReadback) Assert.Contains("controlled final readback failure", comparison.Snapshot.Error); var retry = comparison.SaveCapturesAsync(); Assert.NotSame(save, retry); await retry.WaitAsync(Deadline); Assert.Same(retry, comparison.CancelAsync()); Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified)); Assert.Equal(2, store.Inner.SaveCalls); } [Theory] [InlineData(false)] [InlineData(true)] public async Task ComparisonOwnershipTail_CloseOrDisposeIncludesOldDrainAfterCompetingSave(bool dispose) { using var fixture = new Fixture(dynamics: true); await using var rehearsal = fixture.Session; var (operation, head) = await PrepareComparisonAsync(fixture); using var store = new CompletionTailStore(false); await using var comparison = CreateComparison(fixture, store); await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); var result = comparison.Snapshot.Result; var releaseEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var closed = new TaskCompletionSource<(bool DrainSettled, bool CallbackExited)>( TaskCreationOptions.RunContinuationsAsynchronously); comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion); store.ReadState = () => comparison.Snapshot; var save = comparison.SaveCapturesAsync(); comparison.Changed += () => { if (comparison.Snapshot.State == CalibrationComparisonState.Closed) closed.TrySetResult((save.IsCompleted, store.CallbackExited)); }; Task? cancellation = null; Task? competing = null; Task? close = null; try { await store.ReadbackEntered.Task.WaitAsync(Deadline); cancellation = Task.Run(() => comparison.CancelAsync()); await store.CallbackEntered.Task.WaitAsync(Deadline); store.ReleaseReadback.TrySetResult(); Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline)); try { competing = comparison.SaveCapturesAsync(); } catch (CalibrationComparisonException exception) { Assert.Equal(CalibrationComparisonFailure.Busy, exception.Failure); } if (competing is not null) { await competing.WaitAsync(Deadline); close = dispose ? comparison.DisposeAsync().AsTask() : comparison.CloseAsync(); Assert.False(close.IsCompleted, $"replacement admitted=True; old drain completed={save.IsCompleted}; " + $"callback exited={store.CallbackExited}; Close/Dispose completed={close.IsCompleted}; state={comparison.Snapshot.State}"); } else { var closeRequested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); close = Task.Run(async () => { closeRequested.TrySetResult(); if (dispose) await comparison.DisposeAsync(); else await comparison.CloseAsync(); }); await closeRequested.Task.WaitAsync(Deadline); Assert.False(save.IsCompleted); Assert.False(closed.Task.IsCompleted); } } finally { store.ReleaseReadback.TrySetResult(); store.ReleaseCallback.TrySetResult(); await ObserveTailTaskAsync(save); if (cancellation is not null) await ObserveTailTaskAsync(cancellation); if (competing is not null) await ObserveTailTaskAsync(competing); if (close is not null) await close.WaitAsync(Deadline); } var terminal = await closed.Task.WaitAsync(Deadline); Assert.True(terminal.DrainSettled); Assert.True(terminal.CallbackExited); Assert.Equal(1, store.CallbackCalls); Assert.Equal(CalibrationComparisonState.Closed, comparison.Snapshot.State); Assert.False(comparison.Snapshot.IsBusy); Assert.Same(result, comparison.Snapshot.Result); Assert.False(result!.IsCurrent); Assert.Same(comparison.CloseAsync(), comparison.CloseAsync()); Assert.Throws(() => { _ = comparison.SaveCapturesAsync(); }); await using var fresh = CreateComparison(fixture, new ComparisonMemoryStore()); await fresh.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); Assert.Equal(CalibrationComparisonState.Completed, fresh.Snapshot.State); Assert.NotSame(result, fresh.Snapshot.Result); Assert.Equal(1, fixture.Port.WriteCount); } [Fact] public async Task ComparisonOwnershipTail_CancelledEvaluationKeepsExactDrainUntilCleanupAndThenRetries() { var factory = new ComparisonSignalFaultFactory("held"); using var fixture = new Fixture(dynamics: true, dynamicsFactory: factory); await using var rehearsal = fixture.Session; var (operation, head) = await PrepareComparisonAsync(fixture); await using var comparison = CreateComparison(fixture, new ComparisonMemoryStore()); var releaseEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseCleanup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseCalls = 0; void HoldRelease(Task completion) { Interlocked.Increment(ref releaseCalls); releaseEntered.TrySetResult(completion); releaseCleanup.Task.GetAwaiter().GetResult(); } comparison.ReleasingOwner += HoldRelease; var evaluation = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); Task? cancellation = null; try { await factory.Entered.Task.WaitAsync(Deadline); cancellation = comparison.CancelAsync(); Assert.Same(evaluation, cancellation); factory.Release.TrySetResult(); Assert.Same(evaluation, await releaseEntered.Task.WaitAsync(Deadline)); var tail = comparison.Snapshot; Assert.True(tail.IsBusy, $"cancelled evaluation tail busy={tail.IsBusy}; state={tail.State}; old drain completed={evaluation.IsCompleted}"); Assert.Equal(CalibrationComparisonState.Evaluating, tail.State); Assert.False(evaluation.IsCompleted); Assert.Same(evaluation, comparison.CancelAsync()); Assert.Throws(() => { _ = comparison.SaveCapturesAsync(); }); Assert.Null(tail.Result); } finally { factory.Release.TrySetResult(); releaseCleanup.TrySetResult(); await ObserveTailTaskAsync(evaluation); if (cancellation is not null) await ObserveTailTaskAsync(cancellation); comparison.ReleasingOwner -= HoldRelease; } await Assert.ThrowsAnyAsync(() => evaluation); Assert.Equal(1, releaseCalls); Assert.Equal(1, factory.Disposals); Assert.Equal(CalibrationComparisonState.Cancelled, comparison.Snapshot.State); Assert.False(comparison.Snapshot.IsBusy); Assert.False(comparison.Snapshot.PartialRun!.IsComplete); var next = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); Assert.NotSame(evaluation, next); await next.WaitAsync(Deadline); Assert.Equal(CalibrationComparisonState.Completed, comparison.Snapshot.State); Assert.True(comparison.Snapshot.Result!.IsCurrent); } [Theory] [InlineData(false)] [InlineData(true)] public async Task ComparisonOwnerCleanup_FailureSettlesWaitersAndPreservesPrimaryAndSaveEvidence(bool primaryFailure) { using var fixture = new Fixture(dynamics: true); await using var rehearsal = fixture.Session; var (operation, head) = await PrepareComparisonAsync(fixture); var store = new ComparisonMemoryStore { FailSaveCall = primaryFailure ? 2 : null }; await using var comparison = CreateComparison(fixture, store); await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier, DynamicsDiagnosticTransportFactory.ProviderIdentifier); var result = comparison.Snapshot.Result; var failure = new IOException("controlled completion cleanup failure"); void FailRelease(Task _) => throw failure; comparison.ReleasingOwner += FailRelease; var save = comparison.SaveCapturesAsync(); try { var error = await Assert.ThrowsAnyAsync(() => save.WaitAsync(Deadline)); Assert.IsNotType(error); if (primaryFailure) { var aggregate = Assert.IsType(error); Assert.Contains(aggregate.InnerExceptions, item => item is CalibrationComparisonException); Assert.Contains(failure, aggregate.InnerExceptions); } else Assert.Same(failure, error); Assert.Equal(CalibrationComparisonState.Failed, comparison.Snapshot.State); Assert.Contains(failure.Message, comparison.Snapshot.Error); Assert.False(comparison.Snapshot.IsBusy); Assert.Same(result, comparison.Snapshot.Result); Assert.True(comparison.Snapshot.Saves[0].ReadbackVerified); Assert.Equal(!primaryFailure, comparison.Snapshot.Saves[1].Persisted); } finally { comparison.ReleasingOwner -= FailRelease; } store.FailSaveCall = null; await comparison.SaveCapturesAsync().WaitAsync(Deadline); Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified)); } private static async Task ObserveTailTaskAsync(Task task) { try { await task.WaitAsync(Deadline); } catch (Exception exception) when (exception is CalibrationComparisonException or OperationCanceledException) { } } private sealed class CompletionTailStore(bool failReadback) : ITelemetryArchiveStore, IDisposable { public ComparisonMemoryStore Inner { get; } = new(); public Func? ReadState { get; set; } public CalibrationComparisonSnapshot? CallbackState { get; private set; } public TaskCompletionSource ReadbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource ReleaseReadback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource CallbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource ReleaseCallback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public bool CallbackExited { get; private set; } public int CallbackCalls { get; private set; } private CancellationTokenRegistration _registration; private int _reads; public Task SaveAsync(TelemetryCaptureSnapshot capture, CancellationToken cancellationToken = default) { if (Inner.SaveCalls == 0) _registration = cancellationToken.Register(() => { CallbackCalls++; CallbackState = ReadState!(); CallbackEntered.TrySetResult(); ReleaseCallback.Task.GetAwaiter().GetResult(); CallbackExited = true; }); return Inner.SaveAsync(capture, cancellationToken); } public async Task LoadAsync(string captureId, CancellationToken cancellationToken = default) { var readback = await Inner.LoadAsync(captureId); if (++_reads != 2) return readback; ReadbackEntered.TrySetResult(); await ReleaseReadback.Task; if (failReadback) throw new TelemetryArchiveException(TelemetryArchiveFailure.ReadFailure, "controlled final readback failure"); return readback; } public Task> ListAsync(CancellationToken cancellationToken = default) => Inner.ListAsync(cancellationToken); public void Dispose() => _registration.Dispose(); } }