Models-Phi-8B/PHI.Data.Generator/RLHF.Generator/Generation/TrainingGenerationService.cs

1127 lines
42 KiB
C#

using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging;
using System.Globalization;
using System.Runtime.ExceptionServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Channels;
using RLHF.Generator.LmStudio;
using RLHF.Generator.Persistence;
using RLHF.Generator.TrainingData;
namespace RLHF.Generator.Generation;
public sealed class TrainingGenerationService
{
private readonly TrainingDataReader _trainingDataReader;
private readonly LmStudioClient _lmStudioClient;
private readonly RlhfGeneratorDbContext _dbContext;
private readonly GenerationOptions _options;
private readonly LmStudioOptions _lmStudioOptions;
private readonly ILogger<TrainingGenerationService> _logger;
private readonly TimeProvider _timeProvider;
public TrainingGenerationService(
TrainingDataReader trainingDataReader,
LmStudioClient lmStudioClient,
RlhfGeneratorDbContext dbContext,
GenerationOptions options,
LmStudioOptions lmStudioOptions,
ILogger<TrainingGenerationService> logger,
TimeProvider? timeProvider = null)
{
ArgumentNullException.ThrowIfNull(trainingDataReader);
ArgumentNullException.ThrowIfNull(lmStudioClient);
ArgumentNullException.ThrowIfNull(dbContext);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(lmStudioOptions);
ArgumentNullException.ThrowIfNull(logger);
if (options.MaxDegreeOfParallelism <= 0)
{
throw new InvalidOperationException(
"Generation MaxDegreeOfParallelism must be greater than zero.");
}
if (options.SaveBatchSize <= 0)
{
throw new InvalidOperationException(
"Generation SaveBatchSize must be greater than zero.");
}
if (options.ProgressLogIntervalSeconds <= 0)
{
throw new InvalidOperationException(
"Generation ProgressLogIntervalSeconds must be greater than zero.");
}
_trainingDataReader = trainingDataReader;
_lmStudioClient = lmStudioClient;
_dbContext = dbContext;
_options = options;
_lmStudioOptions = lmStudioOptions;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<GenerationRunResult> GenerateAsync(
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
await _trainingDataReader.ValidateAsync(cancellationToken);
string inputFingerprint = await CalculateInputFingerprintAsync(cancellationToken);
string configurationFingerprint = CalculateConfigurationFingerprint();
int totalExpectedRowCount = await CalculateExpectedRowCountAsync(cancellationToken);
(GenerationRun run, bool resumed) = await GetOrCreateRunAsync(
inputFingerprint,
configurationFingerprint,
totalExpectedRowCount,
cancellationToken);
int initiallyCommittedRowCount = run.TotalCommittedRowCount;
GenerationProgressState progress = new(
run.TotalExpectedRowCount,
initiallyCommittedRowCount,
_options.MaxDegreeOfParallelism,
_timeProvider);
_logger.LogInformation(
"Generation execution started: total {TotalExpected}, already committed " +
"{AlreadyCommitted}, remaining {Remaining}, parallelism {ConfiguredParallelism}, " +
"batch size {SaveBatchSize}, progress interval {ProgressLogIntervalSeconds}s.",
run.TotalExpectedRowCount,
initiallyCommittedRowCount,
Math.Max(0, run.TotalExpectedRowCount - initiallyCommittedRowCount),
_options.MaxDegreeOfParallelism,
_options.SaveBatchSize,
_options.ProgressLogIntervalSeconds);
using CancellationTokenSource reporterCancellation = new();
Task reporterTask = ReportProgressAsync(progress, reporterCancellation.Token);
try
{
if (run.TotalCommittedRowCount < run.TotalExpectedRowCount)
{
await GenerateAndPersistAsync(run, progress, cancellationToken);
}
GenerationRun completedRun = await MarkCompletedAsync(run.Id, cancellationToken);
await reporterCancellation.CancelAsync();
await reporterTask;
LogFinalProgress(progress, completedRun.TotalCommittedRowCount, false);
return new GenerationRunResult(
completedRun.Id,
completedRun.Status,
completedRun.TotalCommittedRowCount - initiallyCommittedRowCount,
completedRun.TotalCommittedRowCount,
completedRun.TotalExpectedRowCount,
resumed);
}
catch (Exception exception)
{
await reporterCancellation.CancelAsync();
await reporterTask;
LogFinalProgress(progress, progress.CommittedCount, true);
await TryMarkInterruptedAsync(
run.Id,
exception,
cancellationToken.IsCancellationRequested);
throw;
}
finally
{
await reporterCancellation.CancelAsync();
await reporterTask;
}
}
private async Task GenerateAndPersistAsync(
GenerationRun run,
GenerationProgressState progress,
CancellationToken cancellationToken)
{
Random random = new(_options.RandomSeed);
int channelCapacity = Math.Max(
_options.SaveBatchSize,
_options.MaxDegreeOfParallelism);
Channel<GeneratedItem> channel = Channel.CreateBounded<GeneratedItem>(
new BoundedChannelOptions(channelCapacity)
{
FullMode = BoundedChannelFullMode.Wait,
SingleReader = true,
SingleWriter = false
});
using CancellationTokenSource writerFailureCancellation = new();
using CancellationTokenSource generationCancellation =
CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken,
writerFailureCancellation.Token);
Task<int> writerTask = PersistGeneratedItemsAsync(
channel,
run.Id,
run.LastCommittedPlanOrdinal + 1,
progress,
writerFailureCancellation);
ExceptionDispatchInfo? generationFailure = null;
long nextPlanOrdinal = 0;
try
{
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
IReadOnlyList<TrainingCategoryValue> categoryValues =
await _trainingDataReader.ReadCategoryValuesAsync(
category,
generationCancellation.Token);
IReadOnlyList<string> positivePatterns =
await _trainingDataReader.ReadPatternsAsync(
category,
true,
generationCancellation.Token);
nextPlanOrdinal += await ProcessPassAsync(
category,
categoryValues,
positivePatterns,
true,
random,
run.Id,
nextPlanOrdinal,
run.LastCommittedPlanOrdinal,
progress,
channel.Writer,
writerFailureCancellation.Token,
generationCancellation.Token);
IReadOnlyList<string> negativePatterns =
await _trainingDataReader.ReadPatternsAsync(
category,
false,
generationCancellation.Token);
nextPlanOrdinal += await ProcessPassAsync(
category,
categoryValues,
negativePatterns,
false,
random,
run.Id,
nextPlanOrdinal,
run.LastCommittedPlanOrdinal,
progress,
channel.Writer,
writerFailureCancellation.Token,
generationCancellation.Token);
}
}
catch (Exception exception)
{
generationFailure = ExceptionDispatchInfo.Capture(exception);
await generationCancellation.CancelAsync();
}
finally
{
channel.Writer.TryComplete();
}
ExceptionDispatchInfo? writerFailure = null;
try
{
await writerTask;
}
catch (Exception exception)
{
writerFailure = ExceptionDispatchInfo.Capture(exception);
}
writerFailure?.Throw();
generationFailure?.Throw();
}
private async Task<int> PersistGeneratedItemsAsync(
Channel<GeneratedItem> channel,
Guid runId,
long nextPlanOrdinal,
GenerationProgressState progress,
CancellationTokenSource writerFailureCancellation)
{
List<GeneratedItem> batch = new(_options.SaveBatchSize);
SortedDictionary<long, GeneratedItem> pendingItems = [];
int persistedRowCount = 0;
try
{
await foreach (GeneratedItem item in channel.Reader.ReadAllAsync())
{
pendingItems.Add(item.PlanOrdinal, item);
while (pendingItems.Remove(nextPlanOrdinal, out GeneratedItem? nextItem))
{
batch.Add(nextItem);
nextPlanOrdinal++;
if (batch.Count == _options.SaveBatchSize)
{
persistedRowCount += await SaveBatchAsync(runId, batch, progress);
}
}
}
if (batch.Count > 0)
{
persistedRowCount += await SaveBatchAsync(runId, batch, progress);
}
return persistedRowCount;
}
catch (Exception exception)
{
channel.Writer.TryComplete(exception);
await writerFailureCancellation.CancelAsync();
throw;
}
}
private async Task<int> SaveBatchAsync(
Guid runId,
List<GeneratedItem> batch,
GenerationProgressState progress)
{
int expectedRowCount = batch.Count;
GeneratedItem checkpoint = batch[^1];
await using IDbContextTransaction transaction =
await _dbContext.Database.BeginTransactionAsync(CancellationToken.None);
try
{
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
candidate => candidate.Id == runId,
CancellationToken.None);
_dbContext.TrainingItems.AddRange(batch.Select(item => item.Item));
run.CurrentCategoryId = checkpoint.CategoryId;
run.CurrentIsPositive = checkpoint.IsPositive;
run.LastCommittedPlanOrdinal = checkpoint.PlanOrdinal;
run.TotalCommittedRowCount += expectedRowCount;
run.LastCheckpointAt = DateTimeOffset.UtcNow;
await _dbContext.SaveChangesAsync(CancellationToken.None);
await transaction.CommitAsync(CancellationToken.None);
progress.SetCommittedCount(run.TotalCommittedRowCount);
_logger.LogInformation(
"Run {RunId} committed batch {BatchNumber}: category {CategoryId}, pass {Pass}, " +
"rows {BatchRowCount}, cumulative {CommittedRowCount}, checkpoint {CheckpointOrdinal}.",
runId,
((run.TotalCommittedRowCount - 1) / _options.SaveBatchSize) + 1,
checkpoint.CategoryId,
checkpoint.IsPositive ? "positive" : "negative",
expectedRowCount,
run.TotalCommittedRowCount,
checkpoint.PlanOrdinal);
}
catch
{
await transaction.RollbackAsync(CancellationToken.None);
_dbContext.ChangeTracker.Clear();
throw;
}
_dbContext.ChangeTracker.Clear();
batch.Clear();
return expectedRowCount;
}
private async Task<int> ProcessPassAsync(
TrainingCategoryDefinition category,
IReadOnlyList<TrainingCategoryValue> categoryValues,
IReadOnlyList<string> patterns,
bool isPositive,
Random random,
Guid runId,
long firstPlanOrdinal,
long lastCommittedPlanOrdinal,
GenerationProgressState progress,
ChannelWriter<GeneratedItem> writer,
CancellationToken writerCancellationToken,
CancellationToken cancellationToken)
{
IReadOnlyList<PlannedItem> plan = CreatePlan(
category,
categoryValues,
patterns,
isPositive,
random);
ExceptionDispatchInfo? failure = null;
using CancellationTokenSource linkedCancellation =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
ParallelOptions parallelOptions = new()
{
CancellationToken = linkedCancellation.Token,
MaxDegreeOfParallelism = _options.MaxDegreeOfParallelism
};
try
{
int firstUncommittedIndex = (int)Math.Clamp(
lastCommittedPlanOrdinal - firstPlanOrdinal + 1,
0,
plan.Count);
await Parallel.ForEachAsync(
Enumerable.Range(firstUncommittedIndex, plan.Count - firstUncommittedIndex),
parallelOptions,
async (index, itemCancellationToken) =>
{
try
{
PlannedItem plannedItem = plan[index];
string value;
if (plannedItem.RequiresLmStudio)
{
progress.LmRequestStarted();
try
{
value = await _lmStudioClient.CompleteAsync(
plannedItem.SubstitutedText,
plannedItem.Tense,
plannedItem.IsFragment,
plannedItem.SelectedSubject,
plannedItem.ProtectedMedicalTerm,
plannedItem.AllowedMedicalForms,
itemCancellationToken);
ValidateLmStudioResponse(value, category, isPositive);
}
finally
{
progress.LmRequestFinished();
}
}
else
{
value = plannedItem.SubstitutedText;
}
progress.ItemProcessed();
long planOrdinal = firstPlanOrdinal + index;
TrainingItem item = new()
{
GenerationRunId = runId,
PlanOrdinal = planOrdinal,
RecordKey = CalculateRecordKey(
runId,
category.Id,
isPositive,
planOrdinal),
CategoryId = category.Id,
IsPositive = isPositive,
Value = value
};
await writer.WriteAsync(
new GeneratedItem(
planOrdinal,
category.Id,
isPositive,
item),
writerCancellationToken);
}
catch (Exception exception)
{
Interlocked.CompareExchange(
ref failure,
ExceptionDispatchInfo.Capture(exception),
null);
linkedCancellation.Cancel();
throw;
}
});
}
catch
{
await linkedCancellation.CancelAsync();
failure?.Throw();
throw;
}
return plan.Count;
}
private async Task ReportProgressAsync(
GenerationProgressState progress,
CancellationToken cancellationToken)
{
using PeriodicTimer timer = new(
TimeSpan.FromSeconds(_options.ProgressLogIntervalSeconds),
_timeProvider);
try
{
while (await timer.WaitForNextTickAsync(cancellationToken))
{
LogPeriodicProgress(progress);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
}
private void LogPeriodicProgress(GenerationProgressState progress)
{
ProgressSnapshot snapshot = progress.TakeSnapshot();
_logger.LogInformation(
"Generation progress: {OverallCompleted}/{TotalExpected} ({Percent:F1}%) | " +
"session {SessionProcessed} | {CurrentRate:F1} items/s current | " +
"{AverageRate:F1} items/s average | committed {CommittedCount} | " +
"LM active {ActiveLmRequests}/{ConfiguredParallelism} | LM peak {PeakLmRequests} | " +
"ETA {Eta}",
snapshot.OverallCompleted,
snapshot.TotalExpected,
snapshot.Percent,
snapshot.SessionProcessed,
snapshot.CurrentRate,
snapshot.AverageRate,
snapshot.CommittedCount,
snapshot.ActiveLmRequests,
snapshot.ConfiguredParallelism,
snapshot.PeakLmRequests,
snapshot.Eta);
}
private void LogFinalProgress(
GenerationProgressState progress,
int committedCount,
bool interrupted)
{
ProgressSnapshot snapshot = progress.TakeSnapshot();
string message = interrupted
? "Generation execution interrupted: session processed {SessionProcessed}, committed " +
"{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " +
"LM active {ActiveLmRequests}, LM peak {PeakLmRequests}."
: "Generation execution completed: session processed {SessionProcessed}, committed " +
"{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " +
"LM active {ActiveLmRequests}, LM peak {PeakLmRequests}.";
if (interrupted)
{
_logger.LogWarning(
message,
snapshot.SessionProcessed,
committedCount,
snapshot.Elapsed,
snapshot.AverageRate,
snapshot.ActiveLmRequests,
snapshot.PeakLmRequests);
}
else
{
_logger.LogInformation(
message,
snapshot.SessionProcessed,
committedCount,
snapshot.Elapsed,
snapshot.AverageRate,
snapshot.ActiveLmRequests,
snapshot.PeakLmRequests);
}
}
private async Task<(GenerationRun Run, bool Resumed)> GetOrCreateRunAsync(
string inputFingerprint,
string configurationFingerprint,
int totalExpectedRowCount,
CancellationToken cancellationToken)
{
List<GenerationRun> incompleteRuns = await _dbContext.GenerationRuns
.Where(run => GenerationRunStatuses.Incomplete.Contains(run.Status))
.ToListAsync(cancellationToken);
incompleteRuns = incompleteRuns
.OrderByDescending(run => run.StartedAt)
.ToList();
if (incompleteRuns.Count > 1)
{
throw new InvalidOperationException(
"Multiple incomplete generation runs exist. Automatic resume was rejected.");
}
GenerationRun? run = incompleteRuns.FirstOrDefault(candidate =>
candidate.RandomSeed == _options.RandomSeed
&& candidate.InputFingerprint == inputFingerprint
&& candidate.ConfigurationFingerprint == configurationFingerprint);
if (run is null && incompleteRuns.Count > 0)
{
throw new InvalidOperationException(
"An incomplete generation run exists, but its input data, configuration, or random seed changed. Unsafe resume was rejected.");
}
bool resumed = run is not null;
DateTimeOffset now = DateTimeOffset.UtcNow;
if (run is null)
{
run = new GenerationRun
{
Id = Guid.NewGuid(),
Status = GenerationRunStatuses.Running,
RandomSeed = _options.RandomSeed,
InputFingerprint = inputFingerprint,
ConfigurationFingerprint = configurationFingerprint,
TotalExpectedRowCount = totalExpectedRowCount,
StartedAt = now,
LastCheckpointAt = now
};
_dbContext.GenerationRuns.Add(run);
}
else
{
if (run.TotalExpectedRowCount != totalExpectedRowCount)
{
throw new InvalidOperationException(
"The resumed generation plan count does not match its durable run state.");
}
int durableRowCount = await _dbContext.TrainingItems.CountAsync(
item => item.GenerationRunId == run.Id,
cancellationToken);
long durableLastOrdinal = await _dbContext.TrainingItems
.Where(item => item.GenerationRunId == run.Id)
.MaxAsync(item => (long?)item.PlanOrdinal, cancellationToken)
?? -1;
if (durableRowCount != run.TotalCommittedRowCount
|| durableLastOrdinal != run.LastCommittedPlanOrdinal)
{
throw new InvalidOperationException(
"The resumed generation rows do not match the durable checkpoint.");
}
run.Status = GenerationRunStatuses.Running;
run.FailureType = null;
run.FailureMessage = null;
}
await _dbContext.SaveChangesAsync(cancellationToken);
_logger.LogInformation(
"Run {RunId} {RunAction} with {CommittedRowCount} of {ExpectedRowCount} rows committed.",
run.Id,
resumed ? "resumed" : "started",
run.TotalCommittedRowCount,
run.TotalExpectedRowCount);
_dbContext.ChangeTracker.Clear();
return (run, resumed);
}
private async Task<GenerationRun> MarkCompletedAsync(
Guid runId,
CancellationToken cancellationToken)
{
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
candidate => candidate.Id == runId,
cancellationToken);
if (run.TotalCommittedRowCount != run.TotalExpectedRowCount)
{
throw new InvalidOperationException(
$"Run committed {run.TotalCommittedRowCount} rows but expected {run.TotalExpectedRowCount}.");
}
run.Status = GenerationRunStatuses.Completed;
run.CompletedAt = DateTimeOffset.UtcNow;
run.LastCheckpointAt = run.CompletedAt.Value;
await _dbContext.SaveChangesAsync(cancellationToken);
_logger.LogInformation(
"Run {RunId} completed with {CommittedRowCount} rows.",
run.Id,
run.TotalCommittedRowCount);
return run;
}
private async Task TryMarkInterruptedAsync(
Guid runId,
Exception exception,
bool cancellationRequested)
{
try
{
_dbContext.ChangeTracker.Clear();
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
candidate => candidate.Id == runId,
CancellationToken.None);
run.Status = cancellationRequested
? GenerationRunStatuses.Cancelled
: GenerationRunStatuses.Failed;
string failureType = exception.GetType().FullName
?? exception.GetType().Name;
run.FailureType = failureType[..Math.Min(failureType.Length, 256)];
run.FailureMessage = GetSanitizedFailureMessage(exception);
run.LastCheckpointAt = DateTimeOffset.UtcNow;
await _dbContext.SaveChangesAsync(CancellationToken.None);
_logger.LogWarning(
"Run {RunId} {Status} after {CommittedRowCount} committed rows at checkpoint " +
"{CheckpointOrdinal}. Failure type: {FailureType}.",
run.Id,
run.Status,
run.TotalCommittedRowCount,
run.LastCommittedPlanOrdinal,
run.FailureType);
}
catch (Exception statusException)
{
_logger.LogError(
statusException,
"Run {RunId} failure status could not be persisted.",
runId);
}
}
private async Task<int> CalculateExpectedRowCountAsync(
CancellationToken cancellationToken)
{
Random random = new(_options.RandomSeed);
int total = 0;
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
IReadOnlyList<TrainingCategoryValue> values =
await _trainingDataReader.ReadCategoryValuesAsync(category, cancellationToken);
foreach (bool isPositive in new[] { true, false })
{
IReadOnlyList<string> patterns =
await _trainingDataReader.ReadPatternsAsync(
category,
isPositive,
cancellationToken);
total = checked(total + CreatePlan(
category,
values,
patterns,
isPositive,
random).Count);
}
}
return total;
}
private async Task<string> CalculateInputFingerprintAsync(
CancellationToken cancellationToken)
{
using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
IEnumerable<string> paths = Directory
.EnumerateFiles(_trainingDataReader.DataDirectory, "*.txt", SearchOption.AllDirectories)
.Where(path =>
{
string relativePath = Path.GetRelativePath(
_trainingDataReader.DataDirectory,
path);
return relativePath.Equals("Subjects.txt", StringComparison.Ordinal)
|| relativePath.StartsWith($"Categories{Path.DirectorySeparatorChar}", StringComparison.Ordinal)
|| relativePath.StartsWith($"Patterns{Path.DirectorySeparatorChar}", StringComparison.Ordinal);
})
.OrderBy(path => Path.GetRelativePath(
_trainingDataReader.DataDirectory,
path), StringComparer.Ordinal);
foreach (string path in paths)
{
string relativePath = Path.GetRelativePath(
_trainingDataReader.DataDirectory,
path).Replace('\\', '/');
hash.AppendData(Encoding.UTF8.GetBytes(relativePath));
hash.AppendData([0]);
hash.AppendData(await File.ReadAllBytesAsync(path, cancellationToken));
hash.AppendData([0]);
}
return Convert.ToHexString(hash.GetHashAndReset());
}
private string CalculateConfigurationFingerprint()
{
string configuration = string.Join('\n',
_options.RandomSeed.ToString(CultureInfo.InvariantCulture),
_options.MaxDegreeOfParallelism.ToString(CultureInfo.InvariantCulture),
_options.SaveBatchSize.ToString(CultureInfo.InvariantCulture),
_lmStudioOptions.ChatCompletionsUrl,
_lmStudioOptions.ModelName,
_lmStudioOptions.SystemPrompt,
_lmStudioOptions.Temperature.ToString("R", CultureInfo.InvariantCulture),
_lmStudioOptions.TopP.ToString("R", CultureInfo.InvariantCulture),
_lmStudioOptions.MaxTokens.ToString(CultureInfo.InvariantCulture),
_lmStudioOptions.Seed.ToString(CultureInfo.InvariantCulture),
_lmStudioOptions.TimeoutSeconds.ToString(CultureInfo.InvariantCulture));
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(configuration)));
}
private static string CalculateRecordKey(
Guid runId,
int categoryId,
bool isPositive,
long planOrdinal)
{
string identity = $"{runId:N}|{categoryId}|{isPositive}|{planOrdinal}";
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(identity)));
}
private static string GetSanitizedFailureMessage(Exception exception) => exception switch
{
OperationCanceledException => "Generation was cancelled.",
HttpRequestException => "The LM Studio request failed.",
InvalidDataException => "Generation input or model output failed validation.",
DbUpdateException => "A database batch could not be committed.",
_ => "Generation failed."
};
private static IReadOnlyList<PlannedItem> CreatePlan(
TrainingCategoryDefinition category,
IReadOnlyList<TrainingCategoryValue> categoryValues,
IReadOnlyList<string> patterns,
bool isPositive,
Random random)
{
List<TrainingCategoryValue> categoryValueDeck = [.. categoryValues];
List<string> valueBoundPatternDeck = [];
List<string> genericPatternDeck = [];
foreach (string pattern in patterns)
{
bool containsCategoryValuePlaceholder =
pattern.Contains("[x]", StringComparison.Ordinal);
if (containsCategoryValuePlaceholder)
{
valueBoundPatternDeck.Add(pattern);
}
else
{
genericPatternDeck.Add(pattern);
}
}
string passName = isPositive ? "positive" : "negative";
if (categoryValueDeck.Count > 0 && valueBoundPatternDeck.Count == 0)
{
throw new InvalidDataException(
$"Category '{category.Name}' {passName} pass has category values but no pattern containing '[x]'.");
}
if (categoryValueDeck.Count == 0 && valueBoundPatternDeck.Count > 0)
{
throw new InvalidDataException(
$"Category '{category.Name}' {passName} pass has patterns containing '[x]' but no category values.");
}
Shuffle(categoryValueDeck, random);
Shuffle(valueBoundPatternDeck, random);
Shuffle(genericPatternDeck, random);
List<string> subjectDeck = [.. TrainingSubjects.All];
Shuffle(subjectDeck, random);
int subjectIndex = 0;
List<string?> tenseDeck =
[
null,
SentenceTenses.Past,
SentenceTenses.Present,
SentenceTenses.Future
];
Shuffle(tenseDeck, random);
int tenseIndex = 0;
int valueBoundItemCount = Math.Max(
valueBoundPatternDeck.Count,
categoryValueDeck.Count);
List<PlannedItem> plan = new(
genericPatternDeck.Count + valueBoundItemCount);
foreach (string genericPattern in genericPatternDeck)
{
AddPlannedItem(genericPattern, null);
}
for (int itemIndex = 0; itemIndex < valueBoundItemCount; itemIndex++)
{
if (itemIndex > 0 && itemIndex % valueBoundPatternDeck.Count == 0)
{
Shuffle(valueBoundPatternDeck, random);
}
if (itemIndex > 0 && itemIndex % categoryValueDeck.Count == 0)
{
Shuffle(categoryValueDeck, random);
}
string pattern = valueBoundPatternDeck[itemIndex % valueBoundPatternDeck.Count];
TrainingCategoryValue categoryValue =
categoryValueDeck[itemIndex % categoryValueDeck.Count];
AddPlannedItem(pattern, categoryValue);
}
return plan;
void AddPlannedItem(string pattern, TrainingCategoryValue? categoryValue)
{
string substitutedText = categoryValue is null
? pattern
: pattern.Replace("[x]", categoryValue.CanonicalForm, StringComparison.Ordinal);
string? tense = null;
if (pattern.Contains("[is|are|was|were]", StringComparison.Ordinal))
{
if (tenseIndex == tenseDeck.Count)
{
Shuffle(tenseDeck, random);
tenseIndex = 0;
}
tense = tenseDeck[tenseIndex];
tenseIndex++;
}
bool containsSubject = pattern.Contains("[sub]", StringComparison.Ordinal);
bool isFragment = !containsSubject;
string? selectedSubject = null;
if (containsSubject)
{
if (subjectIndex == subjectDeck.Count)
{
Shuffle(subjectDeck, random);
subjectIndex = 0;
}
selectedSubject = subjectDeck[subjectIndex];
substitutedText = substitutedText.Replace(
"[sub]",
selectedSubject,
StringComparison.Ordinal);
subjectIndex++;
}
substitutedText = substitutedText.Trim();
if (string.IsNullOrWhiteSpace(substitutedText))
{
throw new InvalidDataException(
$"Substituted text is empty for category '{category.Name}' {passName} pass.");
}
if (substitutedText.Contains("[x]", StringComparison.Ordinal)
|| substitutedText.Contains("[sub]", StringComparison.Ordinal))
{
throw new InvalidDataException(
$"Unresolved placeholder remains for category '{category.Name}' {passName} pass.");
}
bool requiresLmStudio = containsSubject
|| substitutedText.Contains('[', StringComparison.Ordinal)
|| substitutedText.Contains(']', StringComparison.Ordinal)
|| tense is not null
|| categoryValue?.AllowedForms.Count > 1;
plan.Add(new PlannedItem(
substitutedText,
requiresLmStudio,
categoryValue?.CanonicalForm,
categoryValue?.AllowedForms,
selectedSubject,
tense,
isFragment));
}
}
private static void ValidateLmStudioResponse(
string value,
TrainingCategoryDefinition category,
bool isPositive)
{
if (string.IsNullOrWhiteSpace(value)
|| value.Contains('\r', StringComparison.Ordinal)
|| value.Contains('\n', StringComparison.Ordinal)
|| value.Contains('[', StringComparison.Ordinal)
|| value.Contains(']', StringComparison.Ordinal)
|| IsResponseWrapper(value))
{
string passName = isPositive ? "positive" : "negative";
throw new InvalidDataException(
$"LM Studio returned invalid output for category '{category.Name}' {passName} pass.");
}
}
private static bool IsResponseWrapper(string value)
{
bool isDoubleQuoted = value.Length >= 2
&& value[0] == '"'
&& value[^1] == '"';
bool isSingleQuoted = value.Length >= 2
&& value[0] == '\''
&& value[^1] == '\'';
bool isJsonObject = value.Length >= 2
&& value[0] == '{'
&& value[^1] == '}';
bool isJsonArray = value.Length >= 2
&& value[0] == '['
&& value[^1] == ']';
if (isDoubleQuoted
|| isSingleQuoted
|| isJsonObject
|| isJsonArray
|| value.StartsWith("```", StringComparison.Ordinal))
{
return true;
}
return ResponsePrefixes.Any(prefix =>
value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
}
private static void Shuffle<T>(IList<T> items, Random random)
{
for (int index = items.Count - 1; index > 0; index--)
{
int swapIndex = random.Next(index + 1);
(items[index], items[swapIndex]) = (items[swapIndex], items[index]);
}
}
private sealed record PlannedItem(
string SubstitutedText,
bool RequiresLmStudio,
string? ProtectedMedicalTerm,
IReadOnlyList<string>? AllowedMedicalForms,
string? SelectedSubject,
string? Tense,
bool IsFragment);
private sealed record GeneratedItem(
long PlanOrdinal,
int CategoryId,
bool IsPositive,
TrainingItem Item);
private sealed class GenerationProgressState
{
private readonly int _totalExpected;
private readonly int _initiallyCommitted;
private readonly int _configuredParallelism;
private readonly TimeProvider _timeProvider;
private readonly long _startedTimestamp;
private long _lastSnapshotTimestamp;
private long _sessionProcessed;
private long _lastSnapshotProcessed;
private int _committedCount;
private int _activeLmRequests;
private int _peakLmRequests;
public GenerationProgressState(
int totalExpected,
int initiallyCommitted,
int configuredParallelism,
TimeProvider timeProvider)
{
_totalExpected = totalExpected;
_initiallyCommitted = initiallyCommitted;
_committedCount = initiallyCommitted;
_configuredParallelism = configuredParallelism;
_timeProvider = timeProvider;
_startedTimestamp = timeProvider.GetTimestamp();
_lastSnapshotTimestamp = _startedTimestamp;
}
public int CommittedCount => Volatile.Read(ref _committedCount);
public void ItemProcessed() => Interlocked.Increment(ref _sessionProcessed);
public void SetCommittedCount(int count) => Volatile.Write(ref _committedCount, count);
public void LmRequestStarted()
{
int active = Interlocked.Increment(ref _activeLmRequests);
int peak = Volatile.Read(ref _peakLmRequests);
while (active > peak)
{
int observed = Interlocked.CompareExchange(
ref _peakLmRequests,
active,
peak);
if (observed == peak)
{
break;
}
peak = observed;
}
}
public void LmRequestFinished() => Interlocked.Decrement(ref _activeLmRequests);
public ProgressSnapshot TakeSnapshot()
{
long timestamp = _timeProvider.GetTimestamp();
long processed = Volatile.Read(ref _sessionProcessed);
long previousProcessed = Interlocked.Exchange(
ref _lastSnapshotProcessed,
processed);
long previousTimestamp = Interlocked.Exchange(
ref _lastSnapshotTimestamp,
timestamp);
TimeSpan elapsed = _timeProvider.GetElapsedTime(_startedTimestamp, timestamp);
TimeSpan interval = _timeProvider.GetElapsedTime(previousTimestamp, timestamp);
double currentRate = interval.TotalSeconds > 0
? (processed - previousProcessed) / interval.TotalSeconds
: 0;
double averageRate = elapsed.TotalSeconds > 0
? processed / elapsed.TotalSeconds
: 0;
long overallCompleted = Math.Min(
_totalExpected,
_initiallyCommitted + processed);
long remaining = Math.Max(0, _totalExpected - overallCompleted);
double percent = _totalExpected > 0
? overallCompleted * 100.0 / _totalExpected
: 100;
string eta = averageRate > 0
? FormatEta(TimeSpan.FromSeconds(remaining / averageRate))
: "calculating";
return new ProgressSnapshot(
overallCompleted,
_totalExpected,
percent,
processed,
currentRate,
averageRate,
CommittedCount,
Volatile.Read(ref _activeLmRequests),
_configuredParallelism,
Volatile.Read(ref _peakLmRequests),
eta,
elapsed);
}
}
private sealed record ProgressSnapshot(
long OverallCompleted,
int TotalExpected,
double Percent,
long SessionProcessed,
double CurrentRate,
double AverageRate,
int CommittedCount,
int ActiveLmRequests,
int ConfiguredParallelism,
int PeakLmRequests,
string Eta,
TimeSpan Elapsed);
private static string FormatEta(TimeSpan eta)
{
int totalHours = Math.Max(0, (int)eta.TotalHours);
if (totalHours > 0)
{
return $"{totalHours}h {eta.Minutes}m";
}
if (eta.Minutes > 0)
{
return $"{eta.Minutes}m {eta.Seconds}s";
}
return $"{Math.Max(0, eta.Seconds)}s";
}
private static readonly string[] ResponsePrefixes =
[
"Output:",
"Result:",
"Completed text:",
"Corrected sentence:"
];
}