using AutoForge.Application.Features.Vehicles.Profiles; using AutoForge.Application.Features.Vehicles.Simulation; using AutoForge.Core.Calibration; using AutoForge.Core.ExecutionTargets; using AutoForge.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; namespace AutoForge.Application.Features.Vehicles.Calibration; public sealed class CalibrationRehearsalFactory( CalibrationAuthoringService authoring, IDynamicsConfigurationService configurations, IVehicleProfileService profiles, IEnumerable factories, TimeProvider clock) { private readonly ISimulatedCalibrationTargetFactory[] _factories = factories.ToArray(); public CalibrationRehearsalSession Create() => new(authoring, configurations, profiles, _factories, clock); } public static class CalibrationRehearsalServiceCollectionExtensions { public static IServiceCollection AddAutoForgeCalibrationRehearsal(this IServiceCollection services) { services.TryAddSingleton(); return services; } } public sealed class CalibrationRehearsalSession( CalibrationAuthoringService authoring, IDynamicsConfigurationService configurations, IVehicleProfileService profiles, IReadOnlyList factories, TimeProvider clock) : IAsyncDisposable { public const string ProviderIdentifier = "autoforge.simulation.calibration-rehearsal"; public const int MaximumMutationOutcomes = 32; private readonly object _gate = new(); private CalibrationRehearsalSnapshot _snapshot = new(RehearsalState.Inactive, false, null, null, null, null, null, [], null); private ISimulatedCalibrationTarget? _target; private SyntheticEngineCalibration? _definition; private string? _workspaceIdentifier; private string? _issuedId; private Owner? _owner; private Owner? _lastOwner; private Task? _closeTask; private long _generation; private bool _disposed; public event Action? Changed; public IReadOnlyList Providers => Array.AsReadOnly(factories.Select(factory => factory.Descriptor).ToArray()); public CalibrationRehearsalSnapshot Snapshot { get { lock (_gate) return _snapshot; } } public string NewOperationId() { lock (_gate) { if (_disposed) throw Reject(RehearsalFailure.Closed, "This rehearsal page session was disposed."); _issuedId = Guid.NewGuid().ToString("N"); return _issuedId; } } public Task OpenAsync(string operationId, string providerIdentifier, ExecutionTargetKind kind, string workspaceIdentifier) => Start(operationId, $"open:{providerIdentifier}:{kind}:{workspaceIdentifier}", default, async owner => { lock (_gate) { if (_target is not null) throw Reject(RehearsalFailure.InvalidTarget, "Close the current isolated target before explicitly opening a fresh instance."); } if (kind != ExecutionTargetKind.Simulation || providerIdentifier != ProviderIdentifier) throw Reject(RehearsalFailure.InvalidTarget, "Only the explicitly registered Simulation calibration-rehearsal provider is eligible; physical/replay targets are prohibited."); var selected = factories.Where(factory => factory.Descriptor.ProviderIdentifier == providerIdentifier && factory.Descriptor.Kind == kind).ToArray(); if (selected.Length != 1) throw Reject(RehearsalFailure.InvalidTarget, "Exactly one matching registered Simulation rehearsal factory is required."); var workspace = RequireHead(workspaceIdentifier, null, proposed: false); var configuration = configurations.Select(workspace.Definition.Applicability.ConfigurationIdentifier); configuration.ValidateAgainst(profiles.Select(configuration.ProfileIdentifier, configuration.ProfileRevision)); owner.Token.ThrowIfCancellationRequested(); var target = selected[0].Open(configuration, workspace.Definition); try { if (target.Identity.Target.Kind != kind || target.Identity.Target.ProviderIdentifier != providerIdentifier || !target.Identity.Source.Matches(workspace.Definition.Applicability) || string.IsNullOrWhiteSpace(target.Identity.InstanceIdentifier)) throw Reject(RehearsalFailure.InvalidTarget, "The opened provider returned an incoherent target instance/source."); RequireHead(workspace.Identifier, null, proposed: false); lock (_gate) { EnsureCurrent(owner); _target = target; _definition = workspace.Definition; _workspaceIdentifier = workspace.Identifier; _snapshot = new(RehearsalState.Ready, true, target.Identity, null, null, null, null, [], null); } return true; } catch { await target.DisposeAsync().ConfigureAwait(false); throw; } }, opening: true); public Task PrepareBackupAsync(string operationId, CancellationToken cancellationToken = default) => Start(operationId, "backup", cancellationToken, async owner => { if (Snapshot.Backup is { } existing) return existing; var first = await ReadValidatedAsync(owner).ConfigureAwait(false); var second = await ReadValidatedAsync(owner).ConfigureAwait(false); if (first.Revision != 0 || !SameImage(first, second)) throw Reject(RehearsalFailure.InvalidImage, "The two independent original target reads did not match a complete revision-zero image; no verified backup was published."); var backup = new VerifiedCalibrationBackup(first, clock.GetUtcNow()); lock (_gate) { EnsureCurrent(owner); _snapshot = _snapshot with { Backup = backup, CurrentImage = second, Error = null }; } return backup; }); public Task PrepareApplyAsync(string operationId, string workspaceIdentifier, string headIdentifier, CancellationToken cancellationToken = default) => Start(operationId, $"prepare-apply:{workspaceIdentifier}:{headIdentifier}", cancellationToken, async owner => { if (Snapshot.State == RehearsalState.RecoveryRequired) throw Reject(RehearsalFailure.RecoveryRequired, "Explicit read-only reconciliation or approved Restore is required before a new Apply."); var backup = RequireBackup(); var workspace = RequireHead(workspaceIdentifier, headIdentifier, proposed: true); var before = await ReadValidatedAsync(owner).ConfigureAwait(false); RequireHead(workspaceIdentifier, headIdentifier, proposed: true); return PublishPreparation(owner, CalibrationMutationAction.Apply, backup, before, workspace.Head.Values, workspace.Identifier, workspace.Head.Identifier); }); public Task PrepareRestoreAsync(string operationId, CancellationToken cancellationToken = default) => Start(operationId, "prepare-restore", cancellationToken, async owner => { var backup = RequireBackup(); var before = await ReadValidatedAsync(owner).ConfigureAwait(false); return PublishPreparation(owner, CalibrationMutationAction.Restore, backup, before, backup.Image.Values, null, null); }); private CalibrationPreparation PublishPreparation(Owner owner, CalibrationMutationAction action, VerifiedCalibrationBackup backup, CalibrationTargetImage before, CalibrationValues candidate, string? workspaceIdentifier, string? headIdentifier) { _definition!.Validate(candidate); if (before.Values.ContentEquals(candidate)) throw Reject(RehearsalFailure.NoChange, "The actual target and candidate are identical; no changed mutation or approval was prepared."); var preparation = new CalibrationPreparation(Guid.NewGuid().ToString("N"), action, backup.Identifier, new(before.Identity, before.Revision, before.Values), new(candidate.Scalars, candidate.Tables), workspaceIdentifier, headIdentifier, CalibrationValueDifferences.Compare(before.Values, candidate)); lock (_gate) { EnsureCurrent(owner); _snapshot = _snapshot with { Preparation = preparation, Approval = null, CurrentImage = before, Error = null }; } return preparation; } public CalibrationApproval Approve(string preparationIdentifier, bool confirm) { CalibrationApproval approval; lock (_gate) { EnsureQuiescent(); var preparation = _snapshot.Preparation; if (preparation is null || preparation.Identifier != preparationIdentifier) throw Reject(RehearsalFailure.InvalidApproval, "Choose the exact current prepared candidate before confirmation."); if (!confirm) throw Reject(RehearsalFailure.ConfirmationRequired, "Explicitly confirm this exact isolated Apply or Restore action."); if (_snapshot.Approval is { Consumed: true } used && used.Preparation.Identifier == preparationIdentifier) throw Reject(RehearsalFailure.InvalidApproval, "This preparation already dispatched; prepare a fresh before-image instead of reapproving it."); ValidatePreparation(preparation); approval = new(Guid.NewGuid().ToString("N"), preparation, false, clock.GetUtcNow()); _snapshot = _snapshot with { Approval = approval, Error = null }; } Changed?.Invoke(); return approval; } public Task ExecuteAsync(string operationId, string approvalIdentifier, CalibrationMutationAction action, CancellationToken cancellationToken = default) { lock (_gate) { var completed = _snapshot.Audit.SingleOrDefault(result => result.OperationIdentifier == operationId); if (completed is not null) { if (completed.ApprovalIdentifier != approvalIdentifier || completed.Action != action) throw Reject(RehearsalFailure.OperationConflict, "An operation ID cannot be reused with different approval/action inputs."); return Task.FromResult(completed); } } return Start(operationId, $"execute:{approvalIdentifier}:{action}", cancellationToken, owner => ExecuteCoreAsync(owner, approvalIdentifier, action)); } private async Task ExecuteCoreAsync(Owner owner, string approvalIdentifier, CalibrationMutationAction action) { var startedAt = clock.GetUtcNow(); var approval = RequireApproval(approvalIdentifier, action); var preparation = approval.Preparation; ValidatePreparation(preparation); var fresh = await ReadValidatedAsync(owner).ConfigureAwait(false); if (!SameImage(preparation.Before, fresh)) throw Reject(RehearsalFailure.StaleImage, "The fresh target image/revision differs from the immutable approved before-image; zero mutation dispatched."); Task pending; DateTimeOffset dispatchedAt = default; CalibrationWriteAcknowledgement? acknowledgement = null; string? failure = null; try { Task Dispatch() { lock (_gate) { EnsureCurrent(owner); RequireApproval(approvalIdentifier, action); ValidatePreparation(preparation); if (_snapshot.Audit.Count >= MaximumMutationOutcomes) throw Reject(RehearsalFailure.AuditLimit, "At most 32 dispatched mutation outcomes per target; approval remains unconsumed and no write was sent."); dispatchedAt = clock.GetUtcNow(); _snapshot = _snapshot with { Approval = approval with { Consumed = true } }; owner.Dispatched = true; return _target!.CommitAsync(new(preparation.Before.Identity, preparation.Before.Revision, preparation.Candidate), owner.Token); } } pending = action == CalibrationMutationAction.Apply ? authoring.AtCurrentHead(preparation.WorkspaceIdentifier!, preparation.HeadIdentifier!, Dispatch) : Dispatch(); acknowledgement = await pending.ConfigureAwait(false); if (!acknowledgement.Accepted) failure = $"Negative acknowledgement: {acknowledgement.Error}"; } catch (CalibrationAuthoringException exception) when (!owner.Dispatched) { throw Reject(RehearsalFailure.StaleSource, $"The source/head changed before dispatch: {exception.Message}"); } catch (Exception exception) when (owner.Dispatched) { failure = $"Mutation dispatch outcome uncertain: {exception.Message}"; } var (actual, classification, readError) = await ReadEvidenceAsync(owner, preparation).ConfigureAwait(false); failure = Join(failure, readError); var verification = classification == RehearsalImageClassification.CandidateMatching ? action == CalibrationMutationAction.Apply ? RehearsalVerification.VerifiedApplied : RehearsalVerification.VerifiedRestored : action == CalibrationMutationAction.Apply && classification == RehearsalImageClassification.Unchanged && acknowledgement is { Accepted: false } ? RehearsalVerification.VerifiedUnchanged : RehearsalVerification.RecoveryRequired; if (verification == RehearsalVerification.RecoveryRequired) failure = Join(failure, "Independent full image/revision proof did not establish the expected changed result; explicit recovery is required."); lock (_gate) { if (!IsCurrent(owner)) { verification = RehearsalVerification.RecoveryRequired; classification = RehearsalImageClassification.Unavailable; failure = Join(failure, "Close/cancellation invalidated publication after dispatch; no late Applied/Restored result is current evidence."); } DateTimeOffset? completedAt = null; try { completedAt = clock.GetUtcNow(); } catch (Exception exception) { failure = Join(failure, $"Completion timestamp unavailable: {exception.Message}"); } var result = new CalibrationMutationResult(owner.Identifier, approvalIdentifier, action, preparation, startedAt, dispatchedAt, completedAt, acknowledgement, actual, verification, classification, failure, verification, actual); _snapshot = _snapshot with { Audit = Array.AsReadOnly(_snapshot.Audit.Append(result).ToArray()), Error = failure }; if (CanPublish(owner)) PublishEvidence(result); return result; } } public Task ReconcileAsync(string operationId, string mutationOperationIdentifier, CancellationToken cancellationToken = default) => Start(operationId, $"reconcile:{mutationOperationIdentifier}", cancellationToken, async owner => { var previous = Snapshot.Audit.LastOrDefault(); if (previous is null || previous.OperationIdentifier != mutationOperationIdentifier || previous.Verification != RehearsalVerification.RecoveryRequired) throw Reject(RehearsalFailure.InvalidCommand, "Choose the latest unresolved mutation for read-only reconciliation."); var (actual, classification, error) = await ReadEvidenceAsync(owner, previous.Preparation).ConfigureAwait(false); var verification = classification switch { RehearsalImageClassification.CandidateMatching => previous.Action == CalibrationMutationAction.Apply ? RehearsalVerification.VerifiedAppliedAfterReconciliation : RehearsalVerification.VerifiedRestoredAfterReconciliation, RehearsalImageClassification.Unchanged => RehearsalVerification.VerifiedUnchanged, _ => RehearsalVerification.RecoveryRequired }; var result = previous with { Actual = actual, Classification = classification, Verification = verification, ReconciledAt = clock.GetUtcNow(), ReconciliationError = error }; lock (_gate) { EnsureCurrent(owner); _snapshot = _snapshot with { Audit = Array.AsReadOnly(_snapshot.Audit.Select(item => item.OperationIdentifier == result.OperationIdentifier ? result : item).ToArray()) }; PublishEvidence(result); } return result; }); private async Task<(CalibrationTargetImage? Actual, RehearsalImageClassification Classification, string? Error)> ReadEvidenceAsync( Owner owner, CalibrationPreparation preparation) { CalibrationTargetImage? actual = null; try { actual = await _target!.ReadAsync(owner.Token).ConfigureAwait(false); owner.Token.ThrowIfCancellationRequested(); ValidateImage(actual); var classification = SameImage(preparation.Before, actual) ? RehearsalImageClassification.Unchanged : actual.Revision == checked(preparation.Before.Revision + 1) && actual.Values.ContentEquals(preparation.Candidate) ? RehearsalImageClassification.CandidateMatching : RehearsalImageClassification.Divergent; return (actual, classification, classification == RehearsalImageClassification.Divergent ? "Fresh readback is divergent in identity/revision or changed/unchanged canonical values." : null); } catch (Exception exception) { return (actual, RehearsalImageClassification.Unavailable, $"Independent readback unavailable/invalid: {exception.Message}"); } } private void PublishEvidence(CalibrationMutationResult result) => _snapshot = _snapshot with { State = result.Verification == RehearsalVerification.RecoveryRequired ? RehearsalState.RecoveryRequired : RehearsalState.Ready, CurrentImage = result.Classification == RehearsalImageClassification.Unavailable ? _snapshot.CurrentImage : result.Actual, Error = Join(result.OriginalFailure, result.ReconciliationError) }; private async Task ReadValidatedAsync(Owner owner) { var image = await _target!.ReadAsync(owner.Token).ConfigureAwait(false); lock (_gate) EnsureCurrent(owner); ValidateImage(image); return image; } private void ValidateImage(CalibrationTargetImage image) { if (image is null || !_snapshot.Identity!.Matches(image.Identity) || image.Revision < 0) throw Reject(RehearsalFailure.InvalidImage, "The target read returned the wrong instance/provider/kind/source/revision."); try { _definition!.Validate(image.Values); } catch (CalibrationAuthoringException exception) { throw Reject(RehearsalFailure.InvalidImage, $"The actual target read is not the complete supported image: {exception.Message}"); } } private VerifiedCalibrationBackup RequireBackup() { var backup = Snapshot.Backup ?? throw Reject(RehearsalFailure.BackupRequired, "Prepare a verified original target backup with two independent reads first."); if (!_snapshot.Identity!.Matches(backup.Image.Identity) || backup.Image.Revision != 0) throw Reject(RehearsalFailure.BackupRequired, "The original verified backup does not belong to this target instance/source."); _definition!.Validate(backup.Image.Values); return backup; } private CalibrationWorkspace RequireHead(string workspaceIdentifier, string? headIdentifier, bool proposed) { var current = authoring.Snapshot; if (current is null || current.Identifier != workspaceIdentifier || (_workspaceIdentifier is not null && workspaceIdentifier != _workspaceIdentifier && _target is not null)) throw Reject(RehearsalFailure.StaleSource, "The selected authoring workspace differs from the opened target source."); if (headIdentifier is not null && current.Head.Identifier != headIdentifier) throw Reject(RehearsalFailure.StaleSource, $"Selected saved head '{headIdentifier}' is not current head '{current.Head.Identifier}'; select the exact current Proposed revision."); if (proposed && current.Head.Kind != CalibrationVersionKind.Proposed) throw Reject(RehearsalFailure.StaleSource, "Only a saved current Proposed revision can prepare Apply; baseline is not an apply proposal."); if (_target is not null && !_target.Identity.Source.Matches(current.Definition.Applicability)) throw Reject(RehearsalFailure.StaleSource, "The current authoring source/definition differs from the opened target source."); return current; } private void ValidatePreparation(CalibrationPreparation preparation) { var backup = RequireBackup(); if (preparation.BackupIdentifier != backup.Identifier || !_snapshot.Identity!.Matches(preparation.Before.Identity)) throw Reject(RehearsalFailure.InvalidApproval, "Prepared target/backup identity no longer matches this rehearsal."); _definition!.Validate(preparation.Candidate); if (preparation.Action == CalibrationMutationAction.Apply) { var current = RequireHead(preparation.WorkspaceIdentifier!, preparation.HeadIdentifier, true); if (!current.Head.Values.ContentEquals(preparation.Candidate)) throw Reject(RehearsalFailure.StaleSource, "The approved candidate is not the exact saved head."); } else if (!backup.Image.Values.ContentEquals(preparation.Candidate)) throw Reject(RehearsalFailure.InvalidApproval, "Restore candidate is not the full original verified backup."); } private CalibrationApproval RequireApproval(string identifier, CalibrationMutationAction action) { var approval = Snapshot.Approval; if (!Enum.IsDefined(action) || approval is null || approval.Identifier != identifier || approval.Consumed || approval.Preparation.Action != action) throw Reject(RehearsalFailure.InvalidApproval, "A matching explicit single-use approval is required before dispatch."); return approval; } private Task Start(string identifier, string key, CancellationToken cancellationToken, Func, Task> run, bool opening = false) { Owner owner; lock (_gate) { if (_owner is { } active) { if (active.Identifier == identifier && active.Key == key && active is Owner same) return same.Task; throw Reject(active.Identifier == identifier ? RehearsalFailure.OperationConflict : RehearsalFailure.Busy, "Another rehearsal operation/cleanup owns the target; conflicting requests cannot mutate it."); } if (_snapshot.State == RehearsalState.Closing) throw Reject(RehearsalFailure.Busy, "The previous target is draining cleanup."); if (_disposed || (!opening && (_target is null || _snapshot.State == RehearsalState.Closed))) throw Reject(RehearsalFailure.Closed, "No open isolated rehearsal target is available."); if (_lastOwner is { } previous && previous.Identifier == identifier) { if (previous.Key == key && previous is Owner same) return same.Task; throw Reject(RehearsalFailure.OperationConflict, "The exact previous operation ID cannot change inputs."); } if (identifier != _issuedId) throw Reject(RehearsalFailure.OperationConflict, "Use a fresh rehearsal-issued opaque operation ID; expired IDs cannot authorize a new request."); cancellationToken.ThrowIfCancellationRequested(); _issuedId = null; owner = new(identifier, key, _generation, cancellationToken); owner.Opening = opening; // A pending reopen owns a new lifetime before its target is published. if (opening && _target is null) _closeTask = null; _owner = owner; _snapshot = _snapshot with { IsBusy = true }; } _ = RunAsync(owner, run); return owner.Task; } private async Task RunAsync(Owner owner, Func, Task> run) { T? result = default; Exception? failure = null; try { result = await run(owner).ConfigureAwait(false); } catch (Exception exception) { failure = exception; lock (_gate) _snapshot = _snapshot with { Error = exception is OperationCanceledException ? "Rehearsal operation cancelled before mutation dispatch or publication." : exception.Message }; } finally { owner.Release(); lock (_gate) { _owner = null; _lastOwner = owner; _snapshot = _snapshot with { IsBusy = false }; if (failure is OperationCanceledException) owner.Completion.TrySetCanceled(); else if (failure is not null) owner.Completion.TrySetException(failure); else owner.Completion.TrySetResult(result!); } Changed?.Invoke(); } } public Task CloseAsync() { Owner? owner; ISimulatedCalibrationTarget? target; TaskCompletionSource completion; lock (_gate) { if (_closeTask is not null) return _closeTask; completion = new(TaskCreationOptions.RunContinuationsAsynchronously); _closeTask = completion.Task; ++_generation; owner = _owner; target = _target; _snapshot = _snapshot with { State = RehearsalState.Closing }; } _ = CloseCoreAsync(owner, target, completion); return completion.Task; } private async Task CloseCoreAsync(Owner? owner, ISimulatedCalibrationTarget? target, TaskCompletionSource completion) { Exception? failure = null; try { owner?.Cancel(); if (owner is not null) { try { await owner.UntypedTask.ConfigureAwait(false); } catch (OperationCanceledException) { } catch (Exception exception) { failure = exception; } } if (target is not null) await target.DisposeAsync().ConfigureAwait(false); } catch (Exception exception) { failure = failure is null ? exception : new AggregateException(failure, exception); } lock (_gate) { _target = null; _snapshot = _snapshot with { State = RehearsalState.Closed, IsBusy = false, Error = failure is null ? _snapshot.Error : $"Target cleanup failed: {failure.Message}" }; if (failure is null) completion.TrySetResult(); else completion.TrySetException(failure); } Changed?.Invoke(); } public async ValueTask DisposeAsync() { lock (_gate) _disposed = true; try { await CloseAsync().ConfigureAwait(false); } finally { Changed = null; } } private void EnsureQuiescent() { if (_owner is not null || _snapshot.State == RehearsalState.Closing) throw Reject(RehearsalFailure.Busy, "Wait for target operation/cleanup before approval."); if (_target is null) throw Reject(RehearsalFailure.Closed, "This isolated target is closed."); } private bool CanPublish(Owner owner) => !_disposed && owner.Generation == _generation && _snapshot.State != RehearsalState.Closing && _snapshot.State != RehearsalState.Closed; private bool IsCurrent(Owner owner) => CanPublish(owner) && !owner.Token.IsCancellationRequested; private void EnsureCurrent(Owner owner) { owner.Token.ThrowIfCancellationRequested(); if (_disposed || owner.Generation != _generation || _snapshot.State == RehearsalState.Closing || (_snapshot.State == RehearsalState.Closed && !owner.Opening)) throw new OperationCanceledException("Old rehearsal generation cannot publish after close."); } private CalibrationRehearsalException Reject(RehearsalFailure failure, string message) { lock (_gate) _snapshot = _snapshot with { Error = message }; return new(failure, message); } private static bool SameImage(CalibrationTargetImage left, CalibrationTargetImage right) => left.Identity.Matches(right.Identity) && left.Revision == right.Revision && left.Values.ContentEquals(right.Values); private static string? Join(string? first, string? second) => first is null ? second : second is null ? first : $"{first} {second}"; private abstract class Owner(string identifier, string key, long generation, CancellationToken external) { private readonly object _gate = new(); private readonly CancellationTokenSource _cancellation = CancellationTokenSource.CreateLinkedTokenSource(external); private bool _released; public string Identifier { get; } = identifier; public string Key { get; } = key; public long Generation { get; } = generation; public CancellationToken Token => _cancellation.Token; public bool Dispatched { get; set; } public bool Opening { get; set; } public abstract Task UntypedTask { get; } public void Cancel() { lock (_gate) { if (!_released) _cancellation.Cancel(); } } public void Release() { lock (_gate) { _released = true; _cancellation.Dispose(); } } } private sealed class Owner(string identifier, string key, long generation, CancellationToken external) : Owner(identifier, key, generation, external) { public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task Task => Completion.Task; public override Task UntypedTask => Task; } }