321 lines
17 KiB
C#
321 lines
17 KiB
C#
using AutoForge.Application.Features.Vehicles.Calibration;
|
|
using AutoForge.Application.Features.Vehicles.Telemetry.Recording;
|
|
using AutoForge.Simulation;
|
|
|
|
namespace AutoForge.IntegrationTests;
|
|
|
|
public sealed partial class CalibrationRehearsalTests
|
|
{
|
|
[Theory]
|
|
[InlineData(false, "evaluate")]
|
|
[InlineData(false, "save")]
|
|
[InlineData(true, "evaluate")]
|
|
[InlineData(true, "save")]
|
|
public async Task ComparisonOwnershipTail_RealCancellationCallbackReservesAdmissionUntilRelease(
|
|
bool failReadback, string contender)
|
|
{
|
|
using var fixture = new Fixture(dynamics: true);
|
|
await using var rehearsal = fixture.Session;
|
|
var (operation, head) = await PrepareComparisonAsync(fixture);
|
|
using var store = new CompletionTailStore(failReadback);
|
|
await using var comparison = CreateComparison(fixture, store);
|
|
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
var result = comparison.Snapshot.Result;
|
|
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion);
|
|
store.ReadState = () => comparison.Snapshot;
|
|
var save = comparison.SaveCapturesAsync();
|
|
Task? cancellation = null;
|
|
Task? competing = null;
|
|
try
|
|
{
|
|
await store.ReadbackEntered.Task.WaitAsync(Deadline);
|
|
cancellation = Task.Run(() => comparison.CancelAsync());
|
|
await store.CallbackEntered.Task.WaitAsync(Deadline);
|
|
Assert.True(store.CallbackState!.IsBusy);
|
|
store.ReleaseReadback.TrySetResult();
|
|
Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline));
|
|
var tail = comparison.Snapshot;
|
|
var reads = fixture.Port.ReadCount;
|
|
var writes = fixture.Port.WriteCount;
|
|
var configurations = fixture.Configurations.List().Count;
|
|
CalibrationComparisonException? rejection = null;
|
|
try
|
|
{
|
|
competing = contender == "evaluate"
|
|
? comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier)
|
|
: comparison.SaveCapturesAsync();
|
|
}
|
|
catch (CalibrationComparisonException exception) { rejection = exception; }
|
|
Assert.True(rejection?.Failure == CalibrationComparisonFailure.Busy,
|
|
$"tail busy={tail.IsBusy}; state={tail.State}; old drain completed={save.IsCompleted}; " +
|
|
$"callback exited={store.CallbackExited}; {contender} admitted={competing is not null}");
|
|
Assert.True(tail.IsBusy);
|
|
Assert.Equal(CalibrationComparisonState.Saving, tail.State);
|
|
Assert.False(save.IsCompleted);
|
|
Assert.False(store.CallbackExited);
|
|
Assert.Equal(reads, fixture.Port.ReadCount);
|
|
Assert.Equal(writes, fixture.Port.WriteCount);
|
|
Assert.Equal(configurations, fixture.Configurations.List().Count);
|
|
Assert.Equal(2, store.Inner.SaveCalls);
|
|
Assert.Same(result, comparison.Snapshot.Result);
|
|
}
|
|
finally
|
|
{
|
|
store.ReleaseReadback.TrySetResult();
|
|
store.ReleaseCallback.TrySetResult();
|
|
await ObserveTailTaskAsync(save);
|
|
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
|
|
if (competing is not null) await ObserveTailTaskAsync(competing);
|
|
}
|
|
Assert.True(store.CallbackExited);
|
|
Assert.Equal(1, store.CallbackCalls);
|
|
Assert.False(comparison.Snapshot.IsBusy);
|
|
Assert.Equal(failReadback ? CalibrationComparisonState.Failed : CalibrationComparisonState.Completed,
|
|
comparison.Snapshot.State);
|
|
Assert.True(comparison.Snapshot.Saves[0].Persisted && comparison.Snapshot.Saves[0].ReadbackVerified);
|
|
Assert.True(comparison.Snapshot.Saves[1].Persisted);
|
|
Assert.Equal(!failReadback, comparison.Snapshot.Saves[1].ReadbackVerified);
|
|
if (failReadback) Assert.Contains("controlled final readback failure", comparison.Snapshot.Error);
|
|
var retry = comparison.SaveCapturesAsync();
|
|
Assert.NotSame(save, retry);
|
|
await retry.WaitAsync(Deadline);
|
|
Assert.Same(retry, comparison.CancelAsync());
|
|
Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified));
|
|
Assert.Equal(2, store.Inner.SaveCalls);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(false)]
|
|
[InlineData(true)]
|
|
public async Task ComparisonOwnershipTail_CloseOrDisposeIncludesOldDrainAfterCompetingSave(bool dispose)
|
|
{
|
|
using var fixture = new Fixture(dynamics: true);
|
|
await using var rehearsal = fixture.Session;
|
|
var (operation, head) = await PrepareComparisonAsync(fixture);
|
|
using var store = new CompletionTailStore(false);
|
|
await using var comparison = CreateComparison(fixture, store);
|
|
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
var result = comparison.Snapshot.Result;
|
|
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var closed = new TaskCompletionSource<(bool DrainSettled, bool CallbackExited)>(
|
|
TaskCreationOptions.RunContinuationsAsynchronously);
|
|
comparison.ReleasingOwner += completion => releaseEntered.TrySetResult(completion);
|
|
store.ReadState = () => comparison.Snapshot;
|
|
var save = comparison.SaveCapturesAsync();
|
|
comparison.Changed += () =>
|
|
{
|
|
if (comparison.Snapshot.State == CalibrationComparisonState.Closed)
|
|
closed.TrySetResult((save.IsCompleted, store.CallbackExited));
|
|
};
|
|
Task? cancellation = null;
|
|
Task? competing = null;
|
|
Task? close = null;
|
|
try
|
|
{
|
|
await store.ReadbackEntered.Task.WaitAsync(Deadline);
|
|
cancellation = Task.Run(() => comparison.CancelAsync());
|
|
await store.CallbackEntered.Task.WaitAsync(Deadline);
|
|
store.ReleaseReadback.TrySetResult();
|
|
Assert.Same(save, await releaseEntered.Task.WaitAsync(Deadline));
|
|
try { competing = comparison.SaveCapturesAsync(); }
|
|
catch (CalibrationComparisonException exception) { Assert.Equal(CalibrationComparisonFailure.Busy, exception.Failure); }
|
|
if (competing is not null)
|
|
{
|
|
await competing.WaitAsync(Deadline);
|
|
close = dispose ? comparison.DisposeAsync().AsTask() : comparison.CloseAsync();
|
|
Assert.False(close.IsCompleted,
|
|
$"replacement admitted=True; old drain completed={save.IsCompleted}; " +
|
|
$"callback exited={store.CallbackExited}; Close/Dispose completed={close.IsCompleted}; state={comparison.Snapshot.State}");
|
|
}
|
|
else
|
|
{
|
|
var closeRequested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
close = Task.Run(async () =>
|
|
{
|
|
closeRequested.TrySetResult();
|
|
if (dispose) await comparison.DisposeAsync(); else await comparison.CloseAsync();
|
|
});
|
|
await closeRequested.Task.WaitAsync(Deadline);
|
|
Assert.False(save.IsCompleted);
|
|
Assert.False(closed.Task.IsCompleted);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
store.ReleaseReadback.TrySetResult();
|
|
store.ReleaseCallback.TrySetResult();
|
|
await ObserveTailTaskAsync(save);
|
|
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
|
|
if (competing is not null) await ObserveTailTaskAsync(competing);
|
|
if (close is not null) await close.WaitAsync(Deadline);
|
|
}
|
|
var terminal = await closed.Task.WaitAsync(Deadline);
|
|
Assert.True(terminal.DrainSettled);
|
|
Assert.True(terminal.CallbackExited);
|
|
Assert.Equal(1, store.CallbackCalls);
|
|
Assert.Equal(CalibrationComparisonState.Closed, comparison.Snapshot.State);
|
|
Assert.False(comparison.Snapshot.IsBusy);
|
|
Assert.Same(result, comparison.Snapshot.Result);
|
|
Assert.False(result!.IsCurrent);
|
|
Assert.Same(comparison.CloseAsync(), comparison.CloseAsync());
|
|
Assert.Throws<CalibrationComparisonException>(() => { _ = comparison.SaveCapturesAsync(); });
|
|
await using var fresh = CreateComparison(fixture, new ComparisonMemoryStore());
|
|
await fresh.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
Assert.Equal(CalibrationComparisonState.Completed, fresh.Snapshot.State);
|
|
Assert.NotSame(result, fresh.Snapshot.Result);
|
|
Assert.Equal(1, fixture.Port.WriteCount);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ComparisonOwnershipTail_CancelledEvaluationKeepsExactDrainUntilCleanupAndThenRetries()
|
|
{
|
|
var factory = new ComparisonSignalFaultFactory("held");
|
|
using var fixture = new Fixture(dynamics: true, dynamicsFactory: factory);
|
|
await using var rehearsal = fixture.Session;
|
|
var (operation, head) = await PrepareComparisonAsync(fixture);
|
|
await using var comparison = CreateComparison(fixture, new ComparisonMemoryStore());
|
|
var releaseEntered = new TaskCompletionSource<Task>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var releaseCleanup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var releaseCalls = 0;
|
|
void HoldRelease(Task completion)
|
|
{
|
|
Interlocked.Increment(ref releaseCalls);
|
|
releaseEntered.TrySetResult(completion);
|
|
releaseCleanup.Task.GetAwaiter().GetResult();
|
|
}
|
|
comparison.ReleasingOwner += HoldRelease;
|
|
var evaluation = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
Task? cancellation = null;
|
|
try
|
|
{
|
|
await factory.Entered.Task.WaitAsync(Deadline);
|
|
cancellation = comparison.CancelAsync();
|
|
Assert.Same(evaluation, cancellation);
|
|
factory.Release.TrySetResult();
|
|
Assert.Same(evaluation, await releaseEntered.Task.WaitAsync(Deadline));
|
|
var tail = comparison.Snapshot;
|
|
Assert.True(tail.IsBusy,
|
|
$"cancelled evaluation tail busy={tail.IsBusy}; state={tail.State}; old drain completed={evaluation.IsCompleted}");
|
|
Assert.Equal(CalibrationComparisonState.Evaluating, tail.State);
|
|
Assert.False(evaluation.IsCompleted);
|
|
Assert.Same(evaluation, comparison.CancelAsync());
|
|
Assert.Throws<CalibrationComparisonException>(() => { _ = comparison.SaveCapturesAsync(); });
|
|
Assert.Null(tail.Result);
|
|
}
|
|
finally
|
|
{
|
|
factory.Release.TrySetResult();
|
|
releaseCleanup.TrySetResult();
|
|
await ObserveTailTaskAsync(evaluation);
|
|
if (cancellation is not null) await ObserveTailTaskAsync(cancellation);
|
|
comparison.ReleasingOwner -= HoldRelease;
|
|
}
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => evaluation);
|
|
Assert.Equal(1, releaseCalls);
|
|
Assert.Equal(1, factory.Disposals);
|
|
Assert.Equal(CalibrationComparisonState.Cancelled, comparison.Snapshot.State);
|
|
Assert.False(comparison.Snapshot.IsBusy);
|
|
Assert.False(comparison.Snapshot.PartialRun!.IsComplete);
|
|
var next = comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
Assert.NotSame(evaluation, next);
|
|
await next.WaitAsync(Deadline);
|
|
Assert.Equal(CalibrationComparisonState.Completed, comparison.Snapshot.State);
|
|
Assert.True(comparison.Snapshot.Result!.IsCurrent);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(false)]
|
|
[InlineData(true)]
|
|
public async Task ComparisonOwnerCleanup_FailureSettlesWaitersAndPreservesPrimaryAndSaveEvidence(bool primaryFailure)
|
|
{
|
|
using var fixture = new Fixture(dynamics: true);
|
|
await using var rehearsal = fixture.Session;
|
|
var (operation, head) = await PrepareComparisonAsync(fixture);
|
|
var store = new ComparisonMemoryStore { FailSaveCall = primaryFailure ? 2 : null };
|
|
await using var comparison = CreateComparison(fixture, store);
|
|
await comparison.EvaluateAsync(operation, fixture.Workspace.Identifier, head.Identifier,
|
|
DynamicsDiagnosticTransportFactory.ProviderIdentifier);
|
|
var result = comparison.Snapshot.Result;
|
|
var failure = new IOException("controlled completion cleanup failure");
|
|
void FailRelease(Task _) => throw failure;
|
|
comparison.ReleasingOwner += FailRelease;
|
|
var save = comparison.SaveCapturesAsync();
|
|
try
|
|
{
|
|
var error = await Assert.ThrowsAnyAsync<Exception>(() => save.WaitAsync(Deadline));
|
|
Assert.IsNotType<TimeoutException>(error);
|
|
if (primaryFailure)
|
|
{
|
|
var aggregate = Assert.IsType<AggregateException>(error);
|
|
Assert.Contains(aggregate.InnerExceptions, item => item is CalibrationComparisonException);
|
|
Assert.Contains(failure, aggregate.InnerExceptions);
|
|
}
|
|
else Assert.Same(failure, error);
|
|
Assert.Equal(CalibrationComparisonState.Failed, comparison.Snapshot.State);
|
|
Assert.Contains(failure.Message, comparison.Snapshot.Error);
|
|
Assert.False(comparison.Snapshot.IsBusy);
|
|
Assert.Same(result, comparison.Snapshot.Result);
|
|
Assert.True(comparison.Snapshot.Saves[0].ReadbackVerified);
|
|
Assert.Equal(!primaryFailure, comparison.Snapshot.Saves[1].Persisted);
|
|
}
|
|
finally { comparison.ReleasingOwner -= FailRelease; }
|
|
store.FailSaveCall = null;
|
|
await comparison.SaveCapturesAsync().WaitAsync(Deadline);
|
|
Assert.All(comparison.Snapshot.Saves, outcome => Assert.True(outcome.ReadbackVerified));
|
|
}
|
|
|
|
private static async Task ObserveTailTaskAsync(Task task)
|
|
{
|
|
try { await task.WaitAsync(Deadline); }
|
|
catch (Exception exception) when (exception is CalibrationComparisonException or OperationCanceledException) { }
|
|
}
|
|
|
|
private sealed class CompletionTailStore(bool failReadback) : ITelemetryArchiveStore, IDisposable
|
|
{
|
|
public ComparisonMemoryStore Inner { get; } = new();
|
|
public Func<CalibrationComparisonSnapshot>? ReadState { get; set; }
|
|
public CalibrationComparisonSnapshot? CallbackState { get; private set; }
|
|
public TaskCompletionSource ReadbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
public TaskCompletionSource ReleaseReadback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
public TaskCompletionSource CallbackEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
public TaskCompletionSource ReleaseCallback { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
public bool CallbackExited { get; private set; }
|
|
public int CallbackCalls { get; private set; }
|
|
private CancellationTokenRegistration _registration;
|
|
private int _reads;
|
|
public Task SaveAsync(TelemetryCaptureSnapshot capture, CancellationToken cancellationToken = default)
|
|
{
|
|
if (Inner.SaveCalls == 0)
|
|
_registration = cancellationToken.Register(() =>
|
|
{
|
|
CallbackCalls++;
|
|
CallbackState = ReadState!();
|
|
CallbackEntered.TrySetResult();
|
|
ReleaseCallback.Task.GetAwaiter().GetResult();
|
|
CallbackExited = true;
|
|
});
|
|
return Inner.SaveAsync(capture, cancellationToken);
|
|
}
|
|
public async Task<TelemetryCaptureSnapshot> LoadAsync(string captureId, CancellationToken cancellationToken = default)
|
|
{
|
|
var readback = await Inner.LoadAsync(captureId);
|
|
if (++_reads != 2) return readback;
|
|
ReadbackEntered.TrySetResult();
|
|
await ReleaseReadback.Task;
|
|
if (failReadback)
|
|
throw new TelemetryArchiveException(TelemetryArchiveFailure.ReadFailure, "controlled final readback failure");
|
|
return readback;
|
|
}
|
|
public Task<IReadOnlyList<TelemetrySavedCapture>> ListAsync(CancellationToken cancellationToken = default) =>
|
|
Inner.ListAsync(cancellationToken);
|
|
public void Dispose() => _registration.Dispose();
|
|
}
|
|
}
|