AutoForge/src/tests/AutoForge.IntegrationTests/CalibrationComparisonOwners...

321 lines
17 KiB
C#

using AutoForge.Application.Features.Vehicles.Calibration;
using AutoForge.Application.Features.Vehicles.Telemetry.Recording;
using AutoForge.Simulation;
namespace AutoForge.IntegrationTests;
public sealed partial class CalibrationRehearsalTests
{
[Theory]
[InlineData(false, "evaluate")]
[InlineData(false, "save")]
[InlineData(true, "evaluate")]
[InlineData(true, "save")]
public async Task ComparisonOwnershipTail_RealCancellationCallbackReservesAdmissionUntilRelease(
bool failReadback, string contender)
{
using var fixture = new Fixture(dynamics: true);
await using var rehearsal = fixture.Session;
var (operation, head) = await PrepareComparisonAsync(fixture);
using var store = new CompletionTailStore(failReadback);
await using var comparison = CreateComparison(fixture, store);
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
var result = comparison.Snapshot.Result;
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion);
store.ReadState = () => comparison.Snapshot;
var save = comparison.SaveCapturesAsync();
Task? cancellation = null;
Task? competing = null;
try
{
await store.ReadbackEntered.Task.WaitAsync(Deadline);
cancellation = Task.Run(() => comparison.CancelAsync());
await store.CallbackEntered.Task.WaitAsync(Deadline);
Assert.True(store.CallbackState!.IsBusy);
store.ReleaseReadback.TrySetResult();
Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline));
var tail = comparison.Snapshot;
var reads = fixture.Port.ReadCount;
var writes = fixture.Port.WriteCount;
var configurations = fixture.Configurations.List().Count;
CalibrationComparisonException? rejection = null;
try
{
competing = contender == "evaluate"
? comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier)
: comparison.SaveCapturesAsync();
}
catch (CalibrationComparisonException exception) { rejection = exception; }
Assert.True(rejection?.Failure == CalibrationComparisonFailure.Busy,
$"tail busy={tail.IsBusy}; state={tail.State}; old drain completed={save.IsCompleted}; " +
$"callback exited={store.CallbackExited}; {contender} admitted={competing is not null}");
Assert.True(tail.IsBusy);
Assert.Equal(CalibrationComparisonState.Saving, tail.State);
Assert.False(save.IsCompleted);
Assert.False(store.CallbackExited);
Assert.Equal(reads, fixture.Port.ReadCount);
Assert.Equal(writes, fixture.Port.WriteCount);
Assert.Equal(configurations, fixture.Configurations.List().Count);
Assert.Equal(2, store.Inner.SaveCalls);
Assert.Same(result, comparison.Snapshot.Result);
}
finally
{
store.ReleaseReadback.TrySetResult();
store.ReleaseCallback.TrySetResult();
await ObserveTailTaskAsync(save);
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
if (competing is not null) await ObserveTailTaskAsync(competing);
}
Assert.True(store.CallbackExited);
Assert.Equal(1, store.CallbackCalls);
Assert.False(comparison.Snapshot.IsBusy);
Assert.Equal(failReadback ? CalibrationComparisonState.Failed : CalibrationComparisonState.Completed,
comparison.Snapshot.State);
Assert.True(comparison.Snapshot.Saves[0].Persisted && comparison.Snapshot.Saves[0].ReadbackVerified);
Assert.True(comparison.Snapshot.Saves[1].Persisted);
Assert.Equal(!failReadback, comparison.Snapshot.Saves[1].ReadbackVerified);
if (failReadback) Assert.Contains("controlled final readback failure", comparison.Snapshot.Error);
var retry = comparison.SaveCapturesAsync();
Assert.NotSame(save, retry);
await retry.WaitAsync(Deadline);
Assert.Same(retry, comparison.CancelAsync());
Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified));
Assert.Equal(2, store.Inner.SaveCalls);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task ComparisonOwnershipTail_CloseOrDisposeIncludesOldDrainAfterCompetingSave(bool dispose)
{
using var fixture = new Fixture(dynamics: true);
await using var rehearsal = fixture.Session;
var (operation, head) = await PrepareComparisonAsync(fixture);
using var store = new CompletionTailStore(false);
await using var comparison = CreateComparison(fixture, store);
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
var result = comparison.Snapshot.Result;
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
var closed = new TaskCompletionSource<(bool DrainSettled, bool CallbackExited)>(
TaskCreationOptions.RunContinuationsAsynchronously);
comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion);
store.ReadState = () => comparison.Snapshot;
var save = comparison.SaveCapturesAsync();
comparison.Changed += () =>
{
if (comparison.Snapshot.State == CalibrationComparisonState.Closed)
closed.TrySetResult((save.IsCompleted, store.CallbackExited));
};
Task? cancellation = null;
Task? competing = null;
Task? close = null;
try
{
await store.ReadbackEntered.Task.WaitAsync(Deadline);
cancellation = Task.Run(() => comparison.CancelAsync());
await store.CallbackEntered.Task.WaitAsync(Deadline);
store.ReleaseReadback.TrySetResult();
Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline));
try { competing = comparison.SaveCapturesAsync(); }
catch (CalibrationComparisonException exception) { Assert.Equal(CalibrationComparisonFailure.Busy, exception.Failure); }
if (competing is not null)
{
await competing.WaitAsync(Deadline);
close = dispose ? comparison.DisposeAsync().AsTask() : comparison.CloseAsync();
Assert.False(close.IsCompleted,
$"replacement admitted=True; old drain completed={save.IsCompleted}; " +
$"callback exited={store.CallbackExited}; Close/Dispose completed={close.IsCompleted}; state={comparison.Snapshot.State}");
}
else
{
var closeRequested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
close = Task.Run(async () =>
{
closeRequested.TrySetResult();
if (dispose) await comparison.DisposeAsync(); else await comparison.CloseAsync();
});
await closeRequested.Task.WaitAsync(Deadline);
Assert.False(save.IsCompleted);
Assert.False(closed.Task.IsCompleted);
}
}
finally
{
store.ReleaseReadback.TrySetResult();
store.ReleaseCallback.TrySetResult();
await ObserveTailTaskAsync(save);
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
if (competing is not null) await ObserveTailTaskAsync(competing);
if (close is not null) await close.WaitAsync(Deadline);
}
var terminal = await closed.Task.WaitAsync(Deadline);
Assert.True(terminal.DrainSettled);
Assert.True(terminal.CallbackExited);
Assert.Equal(1, store.CallbackCalls);
Assert.Equal(CalibrationComparisonState.Closed, comparison.Snapshot.State);
Assert.False(comparison.Snapshot.IsBusy);
Assert.Same(result, comparison.Snapshot.Result);
Assert.False(result!.IsCurrent);
Assert.Same(comparison.CloseAsync(), comparison.CloseAsync());
Assert.Throws<CalibrationComparisonException>(() => { _ = comparison.SaveCapturesAsync(); });
await using var fresh = CreateComparison(fixture, new ComparisonMemoryStore());
await fresh.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
Assert.Equal(CalibrationComparisonState.Completed, fresh.Snapshot.State);
Assert.NotSame(result, fresh.Snapshot.Result);
Assert.Equal(1, fixture.Port.WriteCount);
}
[Fact]
public async Task ComparisonOwnershipTail_CancelledEvaluationKeepsExactDrainUntilCleanupAndThenRetries()
{
var factory = new ComparisonSignalFaultFactory("held");
using var fixture = new Fixture(dynamics: true, dynamicsFactory: factory);
await using var rehearsal = fixture.Session;
var (operation, head) = await PrepareComparisonAsync(fixture);
await using var comparison = CreateComparison(fixture, new ComparisonMemoryStore());
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
var releaseCleanup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var releaseCalls = 0;
void HoldRelease(Task completion)
{
Interlocked.Increment(ref releaseCalls);
releaseEntered.TrySetResult(completion);
releaseCleanup.Task.GetAwaiter().GetResult();
}
comparison.ReleasingOwner += HoldRelease;
var evaluation = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
Task? cancellation = null;
try
{
await factory.Entered.Task.WaitAsync(Deadline);
cancellation = comparison.CancelAsync();
Assert.Same(evaluation, cancellation);
factory.Release.TrySetResult();
Assert.Same(evaluation, await releaseEntered.Task.WaitAsync(Deadline));
var tail = comparison.Snapshot;
Assert.True(tail.IsBusy,
$"cancelled evaluation tail busy={tail.IsBusy}; state={tail.State}; old drain completed={evaluation.IsCompleted}");
Assert.Equal(CalibrationComparisonState.Evaluating, tail.State);
Assert.False(evaluation.IsCompleted);
Assert.Same(evaluation, comparison.CancelAsync());
Assert.Throws<CalibrationComparisonException>(() => { _ = comparison.SaveCapturesAsync(); });
Assert.Null(tail.Result);
}
finally
{
factory.Release.TrySetResult();
releaseCleanup.TrySetResult();
await ObserveTailTaskAsync(evaluation);
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
comparison.ReleasingOwner -= HoldRelease;
}
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => evaluation);
Assert.Equal(1, releaseCalls);
Assert.Equal(1, factory.Disposals);
Assert.Equal(CalibrationComparisonState.Cancelled, comparison.Snapshot.State);
Assert.False(comparison.Snapshot.IsBusy);
Assert.False(comparison.Snapshot.PartialRun!.IsComplete);
var next = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
Assert.NotSame(evaluation, next);
await next.WaitAsync(Deadline);
Assert.Equal(CalibrationComparisonState.Completed, comparison.Snapshot.State);
Assert.True(comparison.Snapshot.Result!.IsCurrent);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task ComparisonOwnerCleanup_FailureSettlesWaitersAndPreservesPrimaryAndSaveEvidence(bool primaryFailure)
{
using var fixture = new Fixture(dynamics: true);
await using var rehearsal = fixture.Session;
var (operation, head) = await PrepareComparisonAsync(fixture);
var store = new ComparisonMemoryStore { FailSaveCall = primaryFailure ? 2 : null };
await using var comparison = CreateComparison(fixture, store);
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
var result = comparison.Snapshot.Result;
var failure = new IOException("controlled completion cleanup failure");
void FailRelease(Task _) => throw failure;
comparison.ReleasingOwner += FailRelease;
var save = comparison.SaveCapturesAsync();
try
{
var error = await Assert.ThrowsAnyAsync<Exception>(() => save.WaitAsync(Deadline));
Assert.IsNotType<TimeoutException>(error);
if (primaryFailure)
{
var aggregate = Assert.IsType<AggregateException>(error);
Assert.Contains(aggregate.InnerExceptions, item => item is CalibrationComparisonException);
Assert.Contains(failure, aggregate.InnerExceptions);
}
else Assert.Same(failure, error);
Assert.Equal(CalibrationComparisonState.Failed, comparison.Snapshot.State);
Assert.Contains(failure.Message, comparison.Snapshot.Error);
Assert.False(comparison.Snapshot.IsBusy);
Assert.Same(result, comparison.Snapshot.Result);
Assert.True(comparison.Snapshot.Saves[0].ReadbackVerified);
Assert.Equal(!primaryFailure, comparison.Snapshot.Saves[1].Persisted);
}
finally { comparison.ReleasingOwner -= FailRelease; }
store.FailSaveCall = null;
await comparison.SaveCapturesAsync().WaitAsync(Deadline);
Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified));
}
private static async Task ObserveTailTaskAsync(Task task)
{
try { await task.WaitAsync(Deadline); }
catch (Exception exception) when (exception is CalibrationComparisonException or OperationCanceledException) { }
}
private sealed class CompletionTailStore(bool failReadback) : ITelemetryArchiveStore, IDisposable
{
public ComparisonMemoryStore Inner { get; } = new();
public Func<CalibrationComparisonSnapshot>? ReadState { get; set; }
public CalibrationComparisonSnapshot? CallbackState { get; private set; }
public TaskCompletionSource ReadbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource ReleaseReadback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource CallbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource ReleaseCallback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool CallbackExited { get; private set; }
public int CallbackCalls { get; private set; }
private CancellationTokenRegistration _registration;
private int _reads;
public Task SaveAsync(TelemetryCaptureSnapshot capture, CancellationToken cancellationToken = default)
{
if (Inner.SaveCalls == 0)
_registration = cancellationToken.Register(() =>
{
CallbackCalls++;
CallbackState = ReadState!();
CallbackEntered.TrySetResult();
ReleaseCallback.Task.GetAwaiter().GetResult();
CallbackExited = true;
});
return Inner.SaveAsync(capture, cancellationToken);
}
public async Task<TelemetryCaptureSnapshot> LoadAsync(string captureId, CancellationToken cancellationToken = default)
{
var readback = await Inner.LoadAsync(captureId);
if (++_reads != 2) return readback;
ReadbackEntered.TrySetResult();
await ReleaseReadback.Task;
if (failReadback)
throw new TelemetryArchiveException(TelemetryArchiveFailure.ReadFailure, "controlled final readback failure");
return readback;
}
public Task<IReadOnlyList<TelemetrySavedCapture>> ListAsync(CancellationToken cancellationToken = default) =>
Inner.ListAsync(cancellationToken);
public void Dispose() => _registration.Dispose();
}
}