using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.Logging; using System.Globalization; using System.Runtime.ExceptionServices; using System.Security.Cryptography; using System.Text; using System.Threading.Channels; using RLHF.Generator.LmStudio; using RLHF.Generator.Persistence; using RLHF.Generator.TrainingData; namespace RLHF.Generator.Generation; public sealed class TrainingGenerationService { private readonly TrainingDataReader _trainingDataReader; private readonly LmStudioClient _lmStudioClient; private readonly RlhfGeneratorDbContext _dbContext; private readonly GenerationOptions _options; private readonly LmStudioOptions _lmStudioOptions; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; public TrainingGenerationService( TrainingDataReader trainingDataReader, LmStudioClient lmStudioClient, RlhfGeneratorDbContext dbContext, GenerationOptions options, LmStudioOptions lmStudioOptions, ILogger logger, TimeProvider? timeProvider = null) { ArgumentNullException.ThrowIfNull(trainingDataReader); ArgumentNullException.ThrowIfNull(lmStudioClient); ArgumentNullException.ThrowIfNull(dbContext); ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(lmStudioOptions); ArgumentNullException.ThrowIfNull(logger); if (options.MaxDegreeOfParallelism <= 0) { throw new InvalidOperationException( "Generation MaxDegreeOfParallelism must be greater than zero."); } if (options.SaveBatchSize <= 0) { throw new InvalidOperationException( "Generation SaveBatchSize must be greater than zero."); } if (options.ProgressLogIntervalSeconds <= 0) { throw new InvalidOperationException( "Generation ProgressLogIntervalSeconds must be greater than zero."); } _trainingDataReader = trainingDataReader; _lmStudioClient = lmStudioClient; _dbContext = dbContext; _options = options; _lmStudioOptions = lmStudioOptions; _logger = logger; _timeProvider = timeProvider ?? TimeProvider.System; } public async Task GenerateAsync( CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); await _trainingDataReader.ValidateAsync(cancellationToken); string inputFingerprint = await CalculateInputFingerprintAsync(cancellationToken); string configurationFingerprint = CalculateConfigurationFingerprint(); int totalExpectedRowCount = await CalculateExpectedRowCountAsync(cancellationToken); (GenerationRun run, bool resumed) = await GetOrCreateRunAsync( inputFingerprint, configurationFingerprint, totalExpectedRowCount, cancellationToken); int initiallyCommittedRowCount = run.TotalCommittedRowCount; GenerationProgressState progress = new( run.TotalExpectedRowCount, initiallyCommittedRowCount, _options.MaxDegreeOfParallelism, _timeProvider); _logger.LogInformation( "Generation execution started: total {TotalExpected}, already committed " + "{AlreadyCommitted}, remaining {Remaining}, parallelism {ConfiguredParallelism}, " + "batch size {SaveBatchSize}, progress interval {ProgressLogIntervalSeconds}s.", run.TotalExpectedRowCount, initiallyCommittedRowCount, Math.Max(0, run.TotalExpectedRowCount - initiallyCommittedRowCount), _options.MaxDegreeOfParallelism, _options.SaveBatchSize, _options.ProgressLogIntervalSeconds); using CancellationTokenSource reporterCancellation = new(); Task reporterTask = ReportProgressAsync(progress, reporterCancellation.Token); try { if (run.TotalCommittedRowCount < run.TotalExpectedRowCount) { await GenerateAndPersistAsync(run, progress, cancellationToken); } GenerationRun completedRun = await MarkCompletedAsync(run.Id, cancellationToken); await reporterCancellation.CancelAsync(); await reporterTask; LogFinalProgress(progress, completedRun.TotalCommittedRowCount, false); return new GenerationRunResult( completedRun.Id, completedRun.Status, completedRun.TotalCommittedRowCount - initiallyCommittedRowCount, completedRun.TotalCommittedRowCount, completedRun.TotalExpectedRowCount, resumed); } catch (Exception exception) { await reporterCancellation.CancelAsync(); await reporterTask; LogFinalProgress(progress, progress.CommittedCount, true); await TryMarkInterruptedAsync( run.Id, exception, cancellationToken.IsCancellationRequested); throw; } finally { await reporterCancellation.CancelAsync(); await reporterTask; } } private async Task GenerateAndPersistAsync( GenerationRun run, GenerationProgressState progress, CancellationToken cancellationToken) { Random random = new(_options.RandomSeed); int channelCapacity = Math.Max( _options.SaveBatchSize, _options.MaxDegreeOfParallelism); Channel channel = Channel.CreateBounded( new BoundedChannelOptions(channelCapacity) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true, SingleWriter = false }); using CancellationTokenSource writerFailureCancellation = new(); using CancellationTokenSource generationCancellation = CancellationTokenSource.CreateLinkedTokenSource( cancellationToken, writerFailureCancellation.Token); Task writerTask = PersistGeneratedItemsAsync( channel, run.Id, run.LastCommittedPlanOrdinal + 1, progress, writerFailureCancellation); ExceptionDispatchInfo? generationFailure = null; long nextPlanOrdinal = 0; try { foreach (TrainingCategoryDefinition category in TrainingCategories.All) { IReadOnlyList categoryValues = await _trainingDataReader.ReadCategoryValuesAsync( category, generationCancellation.Token); IReadOnlyList positivePatterns = await _trainingDataReader.ReadPatternsAsync( category, true, generationCancellation.Token); nextPlanOrdinal += await ProcessPassAsync( category, categoryValues, positivePatterns, true, random, run.Id, nextPlanOrdinal, run.LastCommittedPlanOrdinal, progress, channel.Writer, writerFailureCancellation.Token, generationCancellation.Token); IReadOnlyList negativePatterns = await _trainingDataReader.ReadPatternsAsync( category, false, generationCancellation.Token); nextPlanOrdinal += await ProcessPassAsync( category, categoryValues, negativePatterns, false, random, run.Id, nextPlanOrdinal, run.LastCommittedPlanOrdinal, progress, channel.Writer, writerFailureCancellation.Token, generationCancellation.Token); } } catch (Exception exception) { generationFailure = ExceptionDispatchInfo.Capture(exception); await generationCancellation.CancelAsync(); } finally { channel.Writer.TryComplete(); } ExceptionDispatchInfo? writerFailure = null; try { await writerTask; } catch (Exception exception) { writerFailure = ExceptionDispatchInfo.Capture(exception); } writerFailure?.Throw(); generationFailure?.Throw(); } private async Task PersistGeneratedItemsAsync( Channel channel, Guid runId, long nextPlanOrdinal, GenerationProgressState progress, CancellationTokenSource writerFailureCancellation) { List batch = new(_options.SaveBatchSize); SortedDictionary pendingItems = []; int persistedRowCount = 0; try { await foreach (GeneratedItem item in channel.Reader.ReadAllAsync()) { pendingItems.Add(item.PlanOrdinal, item); while (pendingItems.Remove(nextPlanOrdinal, out GeneratedItem? nextItem)) { batch.Add(nextItem); nextPlanOrdinal++; if (batch.Count == _options.SaveBatchSize) { persistedRowCount += await SaveBatchAsync(runId, batch, progress); } } } if (batch.Count > 0) { persistedRowCount += await SaveBatchAsync(runId, batch, progress); } return persistedRowCount; } catch (Exception exception) { channel.Writer.TryComplete(exception); await writerFailureCancellation.CancelAsync(); throw; } } private async Task SaveBatchAsync( Guid runId, List batch, GenerationProgressState progress) { int expectedRowCount = batch.Count; GeneratedItem checkpoint = batch[^1]; await using IDbContextTransaction transaction = await _dbContext.Database.BeginTransactionAsync(CancellationToken.None); try { GenerationRun run = await _dbContext.GenerationRuns.SingleAsync( candidate => candidate.Id == runId, CancellationToken.None); _dbContext.TrainingItems.AddRange(batch.Select(item => item.Item)); run.CurrentCategoryId = checkpoint.CategoryId; run.CurrentIsPositive = checkpoint.IsPositive; run.LastCommittedPlanOrdinal = checkpoint.PlanOrdinal; run.TotalCommittedRowCount += expectedRowCount; run.LastCheckpointAt = DateTimeOffset.UtcNow; await _dbContext.SaveChangesAsync(CancellationToken.None); await transaction.CommitAsync(CancellationToken.None); progress.SetCommittedCount(run.TotalCommittedRowCount); _logger.LogInformation( "Run {RunId} committed batch {BatchNumber}: category {CategoryId}, pass {Pass}, " + "rows {BatchRowCount}, cumulative {CommittedRowCount}, checkpoint {CheckpointOrdinal}.", runId, ((run.TotalCommittedRowCount - 1) / _options.SaveBatchSize) + 1, checkpoint.CategoryId, checkpoint.IsPositive ? "positive" : "negative", expectedRowCount, run.TotalCommittedRowCount, checkpoint.PlanOrdinal); } catch { await transaction.RollbackAsync(CancellationToken.None); _dbContext.ChangeTracker.Clear(); throw; } _dbContext.ChangeTracker.Clear(); batch.Clear(); return expectedRowCount; } private async Task ProcessPassAsync( TrainingCategoryDefinition category, IReadOnlyList categoryValues, IReadOnlyList patterns, bool isPositive, Random random, Guid runId, long firstPlanOrdinal, long lastCommittedPlanOrdinal, GenerationProgressState progress, ChannelWriter writer, CancellationToken writerCancellationToken, CancellationToken cancellationToken) { IReadOnlyList plan = CreatePlan( category, categoryValues, patterns, isPositive, random); ExceptionDispatchInfo? failure = null; using CancellationTokenSource linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); ParallelOptions parallelOptions = new() { CancellationToken = linkedCancellation.Token, MaxDegreeOfParallelism = _options.MaxDegreeOfParallelism }; try { int firstUncommittedIndex = (int)Math.Clamp( lastCommittedPlanOrdinal - firstPlanOrdinal + 1, 0, plan.Count); await Parallel.ForEachAsync( Enumerable.Range(firstUncommittedIndex, plan.Count - firstUncommittedIndex), parallelOptions, async (index, itemCancellationToken) => { try { PlannedItem plannedItem = plan[index]; string value; if (plannedItem.RequiresLmStudio) { progress.LmRequestStarted(); try { value = await _lmStudioClient.CompleteAsync( plannedItem.SubstitutedText, plannedItem.Tense, plannedItem.IsFragment, plannedItem.SelectedSubject, plannedItem.ProtectedMedicalTerm, plannedItem.AllowedMedicalForms, itemCancellationToken); ValidateLmStudioResponse(value, category, isPositive); } finally { progress.LmRequestFinished(); } } else { value = plannedItem.SubstitutedText; } progress.ItemProcessed(); long planOrdinal = firstPlanOrdinal + index; TrainingItem item = new() { GenerationRunId = runId, PlanOrdinal = planOrdinal, RecordKey = CalculateRecordKey( runId, category.Id, isPositive, planOrdinal), CategoryId = category.Id, IsPositive = isPositive, Value = value }; await writer.WriteAsync( new GeneratedItem( planOrdinal, category.Id, isPositive, item), writerCancellationToken); } catch (Exception exception) { Interlocked.CompareExchange( ref failure, ExceptionDispatchInfo.Capture(exception), null); linkedCancellation.Cancel(); throw; } }); } catch { await linkedCancellation.CancelAsync(); failure?.Throw(); throw; } return plan.Count; } private async Task ReportProgressAsync( GenerationProgressState progress, CancellationToken cancellationToken) { using PeriodicTimer timer = new( TimeSpan.FromSeconds(_options.ProgressLogIntervalSeconds), _timeProvider); try { while (await timer.WaitForNextTickAsync(cancellationToken)) { LogPeriodicProgress(progress); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } } private void LogPeriodicProgress(GenerationProgressState progress) { ProgressSnapshot snapshot = progress.TakeSnapshot(); _logger.LogInformation( "Generation progress: {OverallCompleted}/{TotalExpected} ({Percent:F1}%) | " + "session {SessionProcessed} | {CurrentRate:F1} items/s current | " + "{AverageRate:F1} items/s average | committed {CommittedCount} | " + "LM active {ActiveLmRequests}/{ConfiguredParallelism} | LM peak {PeakLmRequests} | " + "ETA {Eta}", snapshot.OverallCompleted, snapshot.TotalExpected, snapshot.Percent, snapshot.SessionProcessed, snapshot.CurrentRate, snapshot.AverageRate, snapshot.CommittedCount, snapshot.ActiveLmRequests, snapshot.ConfiguredParallelism, snapshot.PeakLmRequests, snapshot.Eta); } private void LogFinalProgress( GenerationProgressState progress, int committedCount, bool interrupted) { ProgressSnapshot snapshot = progress.TakeSnapshot(); string message = interrupted ? "Generation execution interrupted: session processed {SessionProcessed}, committed " + "{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " + "LM active {ActiveLmRequests}, LM peak {PeakLmRequests}." : "Generation execution completed: session processed {SessionProcessed}, committed " + "{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " + "LM active {ActiveLmRequests}, LM peak {PeakLmRequests}."; if (interrupted) { _logger.LogWarning( message, snapshot.SessionProcessed, committedCount, snapshot.Elapsed, snapshot.AverageRate, snapshot.ActiveLmRequests, snapshot.PeakLmRequests); } else { _logger.LogInformation( message, snapshot.SessionProcessed, committedCount, snapshot.Elapsed, snapshot.AverageRate, snapshot.ActiveLmRequests, snapshot.PeakLmRequests); } } private async Task<(GenerationRun Run, bool Resumed)> GetOrCreateRunAsync( string inputFingerprint, string configurationFingerprint, int totalExpectedRowCount, CancellationToken cancellationToken) { List incompleteRuns = await _dbContext.GenerationRuns .Where(run => GenerationRunStatuses.Incomplete.Contains(run.Status)) .ToListAsync(cancellationToken); incompleteRuns = incompleteRuns .OrderByDescending(run => run.StartedAt) .ToList(); if (incompleteRuns.Count > 1) { throw new InvalidOperationException( "Multiple incomplete generation runs exist. Automatic resume was rejected."); } GenerationRun? run = incompleteRuns.FirstOrDefault(candidate => candidate.RandomSeed == _options.RandomSeed && candidate.InputFingerprint == inputFingerprint && candidate.ConfigurationFingerprint == configurationFingerprint); if (run is null && incompleteRuns.Count > 0) { throw new InvalidOperationException( "An incomplete generation run exists, but its input data, configuration, or random seed changed. Unsafe resume was rejected."); } bool resumed = run is not null; DateTimeOffset now = DateTimeOffset.UtcNow; if (run is null) { run = new GenerationRun { Id = Guid.NewGuid(), Status = GenerationRunStatuses.Running, RandomSeed = _options.RandomSeed, InputFingerprint = inputFingerprint, ConfigurationFingerprint = configurationFingerprint, TotalExpectedRowCount = totalExpectedRowCount, StartedAt = now, LastCheckpointAt = now }; _dbContext.GenerationRuns.Add(run); } else { if (run.TotalExpectedRowCount != totalExpectedRowCount) { throw new InvalidOperationException( "The resumed generation plan count does not match its durable run state."); } int durableRowCount = await _dbContext.TrainingItems.CountAsync( item => item.GenerationRunId == run.Id, cancellationToken); long durableLastOrdinal = await _dbContext.TrainingItems .Where(item => item.GenerationRunId == run.Id) .MaxAsync(item => (long?)item.PlanOrdinal, cancellationToken) ?? -1; if (durableRowCount != run.TotalCommittedRowCount || durableLastOrdinal != run.LastCommittedPlanOrdinal) { throw new InvalidOperationException( "The resumed generation rows do not match the durable checkpoint."); } run.Status = GenerationRunStatuses.Running; run.FailureType = null; run.FailureMessage = null; } await _dbContext.SaveChangesAsync(cancellationToken); _logger.LogInformation( "Run {RunId} {RunAction} with {CommittedRowCount} of {ExpectedRowCount} rows committed.", run.Id, resumed ? "resumed" : "started", run.TotalCommittedRowCount, run.TotalExpectedRowCount); _dbContext.ChangeTracker.Clear(); return (run, resumed); } private async Task MarkCompletedAsync( Guid runId, CancellationToken cancellationToken) { GenerationRun run = await _dbContext.GenerationRuns.SingleAsync( candidate => candidate.Id == runId, cancellationToken); if (run.TotalCommittedRowCount != run.TotalExpectedRowCount) { throw new InvalidOperationException( $"Run committed {run.TotalCommittedRowCount} rows but expected {run.TotalExpectedRowCount}."); } run.Status = GenerationRunStatuses.Completed; run.CompletedAt = DateTimeOffset.UtcNow; run.LastCheckpointAt = run.CompletedAt.Value; await _dbContext.SaveChangesAsync(cancellationToken); _logger.LogInformation( "Run {RunId} completed with {CommittedRowCount} rows.", run.Id, run.TotalCommittedRowCount); return run; } private async Task TryMarkInterruptedAsync( Guid runId, Exception exception, bool cancellationRequested) { try { _dbContext.ChangeTracker.Clear(); GenerationRun run = await _dbContext.GenerationRuns.SingleAsync( candidate => candidate.Id == runId, CancellationToken.None); run.Status = cancellationRequested ? GenerationRunStatuses.Cancelled : GenerationRunStatuses.Failed; string failureType = exception.GetType().FullName ?? exception.GetType().Name; run.FailureType = failureType[..Math.Min(failureType.Length, 256)]; run.FailureMessage = GetSanitizedFailureMessage(exception); run.LastCheckpointAt = DateTimeOffset.UtcNow; await _dbContext.SaveChangesAsync(CancellationToken.None); _logger.LogWarning( "Run {RunId} {Status} after {CommittedRowCount} committed rows at checkpoint " + "{CheckpointOrdinal}. Failure type: {FailureType}.", run.Id, run.Status, run.TotalCommittedRowCount, run.LastCommittedPlanOrdinal, run.FailureType); } catch (Exception statusException) { _logger.LogError( statusException, "Run {RunId} failure status could not be persisted.", runId); } } private async Task CalculateExpectedRowCountAsync( CancellationToken cancellationToken) { Random random = new(_options.RandomSeed); int total = 0; foreach (TrainingCategoryDefinition category in TrainingCategories.All) { IReadOnlyList values = await _trainingDataReader.ReadCategoryValuesAsync(category, cancellationToken); foreach (bool isPositive in new[] { true, false }) { IReadOnlyList patterns = await _trainingDataReader.ReadPatternsAsync( category, isPositive, cancellationToken); total = checked(total + CreatePlan( category, values, patterns, isPositive, random).Count); } } return total; } private async Task CalculateInputFingerprintAsync( CancellationToken cancellationToken) { using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); IEnumerable paths = Directory .EnumerateFiles(_trainingDataReader.DataDirectory, "*.txt", SearchOption.AllDirectories) .Where(path => { string relativePath = Path.GetRelativePath( _trainingDataReader.DataDirectory, path); return relativePath.Equals("Subjects.txt", StringComparison.Ordinal) || relativePath.StartsWith($"Categories{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || relativePath.StartsWith($"Patterns{Path.DirectorySeparatorChar}", StringComparison.Ordinal); }) .OrderBy(path => Path.GetRelativePath( _trainingDataReader.DataDirectory, path), StringComparer.Ordinal); foreach (string path in paths) { string relativePath = Path.GetRelativePath( _trainingDataReader.DataDirectory, path).Replace('\\', '/'); hash.AppendData(Encoding.UTF8.GetBytes(relativePath)); hash.AppendData([0]); hash.AppendData(await File.ReadAllBytesAsync(path, cancellationToken)); hash.AppendData([0]); } return Convert.ToHexString(hash.GetHashAndReset()); } private string CalculateConfigurationFingerprint() { string configuration = string.Join('\n', _options.RandomSeed.ToString(CultureInfo.InvariantCulture), _options.MaxDegreeOfParallelism.ToString(CultureInfo.InvariantCulture), _options.SaveBatchSize.ToString(CultureInfo.InvariantCulture), _lmStudioOptions.ChatCompletionsUrl, _lmStudioOptions.ModelName, _lmStudioOptions.SystemPrompt, _lmStudioOptions.Temperature.ToString("R", CultureInfo.InvariantCulture), _lmStudioOptions.TopP.ToString("R", CultureInfo.InvariantCulture), _lmStudioOptions.MaxTokens.ToString(CultureInfo.InvariantCulture), _lmStudioOptions.Seed.ToString(CultureInfo.InvariantCulture), _lmStudioOptions.TimeoutSeconds.ToString(CultureInfo.InvariantCulture)); return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(configuration))); } private static string CalculateRecordKey( Guid runId, int categoryId, bool isPositive, long planOrdinal) { string identity = $"{runId:N}|{categoryId}|{isPositive}|{planOrdinal}"; return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(identity))); } private static string GetSanitizedFailureMessage(Exception exception) => exception switch { OperationCanceledException => "Generation was cancelled.", HttpRequestException => "The LM Studio request failed.", InvalidDataException => "Generation input or model output failed validation.", DbUpdateException => "A database batch could not be committed.", _ => "Generation failed." }; private static IReadOnlyList CreatePlan( TrainingCategoryDefinition category, IReadOnlyList categoryValues, IReadOnlyList patterns, bool isPositive, Random random) { List categoryValueDeck = [.. categoryValues]; List valueBoundPatternDeck = []; List genericPatternDeck = []; foreach (string pattern in patterns) { bool containsCategoryValuePlaceholder = pattern.Contains("[x]", StringComparison.Ordinal); if (containsCategoryValuePlaceholder) { valueBoundPatternDeck.Add(pattern); } else { genericPatternDeck.Add(pattern); } } string passName = isPositive ? "positive" : "negative"; if (categoryValueDeck.Count > 0 && valueBoundPatternDeck.Count == 0) { throw new InvalidDataException( $"Category '{category.Name}' {passName} pass has category values but no pattern containing '[x]'."); } if (categoryValueDeck.Count == 0 && valueBoundPatternDeck.Count > 0) { throw new InvalidDataException( $"Category '{category.Name}' {passName} pass has patterns containing '[x]' but no category values."); } Shuffle(categoryValueDeck, random); Shuffle(valueBoundPatternDeck, random); Shuffle(genericPatternDeck, random); List subjectDeck = [.. TrainingSubjects.All]; Shuffle(subjectDeck, random); int subjectIndex = 0; List tenseDeck = [ null, SentenceTenses.Past, SentenceTenses.Present, SentenceTenses.Future ]; Shuffle(tenseDeck, random); int tenseIndex = 0; int valueBoundItemCount = Math.Max( valueBoundPatternDeck.Count, categoryValueDeck.Count); List plan = new( genericPatternDeck.Count + valueBoundItemCount); foreach (string genericPattern in genericPatternDeck) { AddPlannedItem(genericPattern, null); } for (int itemIndex = 0; itemIndex < valueBoundItemCount; itemIndex++) { if (itemIndex > 0 && itemIndex % valueBoundPatternDeck.Count == 0) { Shuffle(valueBoundPatternDeck, random); } if (itemIndex > 0 && itemIndex % categoryValueDeck.Count == 0) { Shuffle(categoryValueDeck, random); } string pattern = valueBoundPatternDeck[itemIndex % valueBoundPatternDeck.Count]; TrainingCategoryValue categoryValue = categoryValueDeck[itemIndex % categoryValueDeck.Count]; AddPlannedItem(pattern, categoryValue); } return plan; void AddPlannedItem(string pattern, TrainingCategoryValue? categoryValue) { string substitutedText = categoryValue is null ? pattern : pattern.Replace("[x]", categoryValue.CanonicalForm, StringComparison.Ordinal); string? tense = null; if (pattern.Contains("[is|are|was|were]", StringComparison.Ordinal)) { if (tenseIndex == tenseDeck.Count) { Shuffle(tenseDeck, random); tenseIndex = 0; } tense = tenseDeck[tenseIndex]; tenseIndex++; } bool containsSubject = pattern.Contains("[sub]", StringComparison.Ordinal); bool isFragment = !containsSubject; string? selectedSubject = null; if (containsSubject) { if (subjectIndex == subjectDeck.Count) { Shuffle(subjectDeck, random); subjectIndex = 0; } selectedSubject = subjectDeck[subjectIndex]; substitutedText = substitutedText.Replace( "[sub]", selectedSubject, StringComparison.Ordinal); subjectIndex++; } substitutedText = substitutedText.Trim(); if (string.IsNullOrWhiteSpace(substitutedText)) { throw new InvalidDataException( $"Substituted text is empty for category '{category.Name}' {passName} pass."); } if (substitutedText.Contains("[x]", StringComparison.Ordinal) || substitutedText.Contains("[sub]", StringComparison.Ordinal)) { throw new InvalidDataException( $"Unresolved placeholder remains for category '{category.Name}' {passName} pass."); } bool requiresLmStudio = containsSubject || substitutedText.Contains('[', StringComparison.Ordinal) || substitutedText.Contains(']', StringComparison.Ordinal) || tense is not null || categoryValue?.AllowedForms.Count > 1; plan.Add(new PlannedItem( substitutedText, requiresLmStudio, categoryValue?.CanonicalForm, categoryValue?.AllowedForms, selectedSubject, tense, isFragment)); } } private static void ValidateLmStudioResponse( string value, TrainingCategoryDefinition category, bool isPositive) { if (string.IsNullOrWhiteSpace(value) || value.Contains('\r', StringComparison.Ordinal) || value.Contains('\n', StringComparison.Ordinal) || value.Contains('[', StringComparison.Ordinal) || value.Contains(']', StringComparison.Ordinal) || IsResponseWrapper(value)) { string passName = isPositive ? "positive" : "negative"; throw new InvalidDataException( $"LM Studio returned invalid output for category '{category.Name}' {passName} pass."); } } private static bool IsResponseWrapper(string value) { bool isDoubleQuoted = value.Length >= 2 && value[0] == '"' && value[^1] == '"'; bool isSingleQuoted = value.Length >= 2 && value[0] == '\'' && value[^1] == '\''; bool isJsonObject = value.Length >= 2 && value[0] == '{' && value[^1] == '}'; bool isJsonArray = value.Length >= 2 && value[0] == '[' && value[^1] == ']'; if (isDoubleQuoted || isSingleQuoted || isJsonObject || isJsonArray || value.StartsWith("```", StringComparison.Ordinal)) { return true; } return ResponsePrefixes.Any(prefix => value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); } private static void Shuffle(IList items, Random random) { for (int index = items.Count - 1; index > 0; index--) { int swapIndex = random.Next(index + 1); (items[index], items[swapIndex]) = (items[swapIndex], items[index]); } } private sealed record PlannedItem( string SubstitutedText, bool RequiresLmStudio, string? ProtectedMedicalTerm, IReadOnlyList? AllowedMedicalForms, string? SelectedSubject, string? Tense, bool IsFragment); private sealed record GeneratedItem( long PlanOrdinal, int CategoryId, bool IsPositive, TrainingItem Item); private sealed class GenerationProgressState { private readonly int _totalExpected; private readonly int _initiallyCommitted; private readonly int _configuredParallelism; private readonly TimeProvider _timeProvider; private readonly long _startedTimestamp; private long _lastSnapshotTimestamp; private long _sessionProcessed; private long _lastSnapshotProcessed; private int _committedCount; private int _activeLmRequests; private int _peakLmRequests; public GenerationProgressState( int totalExpected, int initiallyCommitted, int configuredParallelism, TimeProvider timeProvider) { _totalExpected = totalExpected; _initiallyCommitted = initiallyCommitted; _committedCount = initiallyCommitted; _configuredParallelism = configuredParallelism; _timeProvider = timeProvider; _startedTimestamp = timeProvider.GetTimestamp(); _lastSnapshotTimestamp = _startedTimestamp; } public int CommittedCount => Volatile.Read(ref _committedCount); public void ItemProcessed() => Interlocked.Increment(ref _sessionProcessed); public void SetCommittedCount(int count) => Volatile.Write(ref _committedCount, count); public void LmRequestStarted() { int active = Interlocked.Increment(ref _activeLmRequests); int peak = Volatile.Read(ref _peakLmRequests); while (active > peak) { int observed = Interlocked.CompareExchange( ref _peakLmRequests, active, peak); if (observed == peak) { break; } peak = observed; } } public void LmRequestFinished() => Interlocked.Decrement(ref _activeLmRequests); public ProgressSnapshot TakeSnapshot() { long timestamp = _timeProvider.GetTimestamp(); long processed = Volatile.Read(ref _sessionProcessed); long previousProcessed = Interlocked.Exchange( ref _lastSnapshotProcessed, processed); long previousTimestamp = Interlocked.Exchange( ref _lastSnapshotTimestamp, timestamp); TimeSpan elapsed = _timeProvider.GetElapsedTime(_startedTimestamp, timestamp); TimeSpan interval = _timeProvider.GetElapsedTime(previousTimestamp, timestamp); double currentRate = interval.TotalSeconds > 0 ? (processed - previousProcessed) / interval.TotalSeconds : 0; double averageRate = elapsed.TotalSeconds > 0 ? processed / elapsed.TotalSeconds : 0; long overallCompleted = Math.Min( _totalExpected, _initiallyCommitted + processed); long remaining = Math.Max(0, _totalExpected - overallCompleted); double percent = _totalExpected > 0 ? overallCompleted * 100.0 / _totalExpected : 100; string eta = averageRate > 0 ? FormatEta(TimeSpan.FromSeconds(remaining / averageRate)) : "calculating"; return new ProgressSnapshot( overallCompleted, _totalExpected, percent, processed, currentRate, averageRate, CommittedCount, Volatile.Read(ref _activeLmRequests), _configuredParallelism, Volatile.Read(ref _peakLmRequests), eta, elapsed); } } private sealed record ProgressSnapshot( long OverallCompleted, int TotalExpected, double Percent, long SessionProcessed, double CurrentRate, double AverageRate, int CommittedCount, int ActiveLmRequests, int ConfiguredParallelism, int PeakLmRequests, string Eta, TimeSpan Elapsed); private static string FormatEta(TimeSpan eta) { int totalHours = Math.Max(0, (int)eta.TotalHours); if (totalHours > 0) { return $"{totalHours}h {eta.Minutes}m"; } if (eta.Minutes > 0) { return $"{eta.Minutes}m {eta.Seconds}s"; } return $"{Math.Max(0, eta.Seconds)}s"; } private static readonly string[] ResponsePrefixes = [ "Output:", "Result:", "Completed text:", "Corrected sentence:" ]; }