1127 lines
42 KiB
C#
1127 lines
42 KiB
C#
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.EntityFrameworkCore.Storage;
|
|
using Microsoft.Extensions.Logging;
|
|
using System.Globalization;
|
|
using System.Runtime.ExceptionServices;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Threading.Channels;
|
|
using RLHF.Generator.LmStudio;
|
|
using RLHF.Generator.Persistence;
|
|
using RLHF.Generator.TrainingData;
|
|
|
|
namespace RLHF.Generator.Generation;
|
|
|
|
public sealed class TrainingGenerationService
|
|
{
|
|
private readonly TrainingDataReader _trainingDataReader;
|
|
private readonly LmStudioClient _lmStudioClient;
|
|
private readonly RlhfGeneratorDbContext _dbContext;
|
|
private readonly GenerationOptions _options;
|
|
private readonly LmStudioOptions _lmStudioOptions;
|
|
private readonly ILogger<TrainingGenerationService> _logger;
|
|
private readonly TimeProvider _timeProvider;
|
|
|
|
public TrainingGenerationService(
|
|
TrainingDataReader trainingDataReader,
|
|
LmStudioClient lmStudioClient,
|
|
RlhfGeneratorDbContext dbContext,
|
|
GenerationOptions options,
|
|
LmStudioOptions lmStudioOptions,
|
|
ILogger<TrainingGenerationService> logger,
|
|
TimeProvider? timeProvider = null)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(trainingDataReader);
|
|
ArgumentNullException.ThrowIfNull(lmStudioClient);
|
|
ArgumentNullException.ThrowIfNull(dbContext);
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
ArgumentNullException.ThrowIfNull(lmStudioOptions);
|
|
ArgumentNullException.ThrowIfNull(logger);
|
|
|
|
if (options.MaxDegreeOfParallelism <= 0)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Generation MaxDegreeOfParallelism must be greater than zero.");
|
|
}
|
|
|
|
if (options.SaveBatchSize <= 0)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Generation SaveBatchSize must be greater than zero.");
|
|
}
|
|
|
|
if (options.ProgressLogIntervalSeconds <= 0)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Generation ProgressLogIntervalSeconds must be greater than zero.");
|
|
}
|
|
|
|
_trainingDataReader = trainingDataReader;
|
|
_lmStudioClient = lmStudioClient;
|
|
_dbContext = dbContext;
|
|
_options = options;
|
|
_lmStudioOptions = lmStudioOptions;
|
|
_logger = logger;
|
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
|
}
|
|
|
|
public async Task<GenerationRunResult> GenerateAsync(
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
await _trainingDataReader.ValidateAsync(cancellationToken);
|
|
|
|
string inputFingerprint = await CalculateInputFingerprintAsync(cancellationToken);
|
|
string configurationFingerprint = CalculateConfigurationFingerprint();
|
|
int totalExpectedRowCount = await CalculateExpectedRowCountAsync(cancellationToken);
|
|
(GenerationRun run, bool resumed) = await GetOrCreateRunAsync(
|
|
inputFingerprint,
|
|
configurationFingerprint,
|
|
totalExpectedRowCount,
|
|
cancellationToken);
|
|
int initiallyCommittedRowCount = run.TotalCommittedRowCount;
|
|
GenerationProgressState progress = new(
|
|
run.TotalExpectedRowCount,
|
|
initiallyCommittedRowCount,
|
|
_options.MaxDegreeOfParallelism,
|
|
_timeProvider);
|
|
_logger.LogInformation(
|
|
"Generation execution started: total {TotalExpected}, already committed " +
|
|
"{AlreadyCommitted}, remaining {Remaining}, parallelism {ConfiguredParallelism}, " +
|
|
"batch size {SaveBatchSize}, progress interval {ProgressLogIntervalSeconds}s.",
|
|
run.TotalExpectedRowCount,
|
|
initiallyCommittedRowCount,
|
|
Math.Max(0, run.TotalExpectedRowCount - initiallyCommittedRowCount),
|
|
_options.MaxDegreeOfParallelism,
|
|
_options.SaveBatchSize,
|
|
_options.ProgressLogIntervalSeconds);
|
|
using CancellationTokenSource reporterCancellation = new();
|
|
Task reporterTask = ReportProgressAsync(progress, reporterCancellation.Token);
|
|
|
|
try
|
|
{
|
|
if (run.TotalCommittedRowCount < run.TotalExpectedRowCount)
|
|
{
|
|
await GenerateAndPersistAsync(run, progress, cancellationToken);
|
|
}
|
|
|
|
GenerationRun completedRun = await MarkCompletedAsync(run.Id, cancellationToken);
|
|
await reporterCancellation.CancelAsync();
|
|
await reporterTask;
|
|
LogFinalProgress(progress, completedRun.TotalCommittedRowCount, false);
|
|
return new GenerationRunResult(
|
|
completedRun.Id,
|
|
completedRun.Status,
|
|
completedRun.TotalCommittedRowCount - initiallyCommittedRowCount,
|
|
completedRun.TotalCommittedRowCount,
|
|
completedRun.TotalExpectedRowCount,
|
|
resumed);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
await reporterCancellation.CancelAsync();
|
|
await reporterTask;
|
|
LogFinalProgress(progress, progress.CommittedCount, true);
|
|
await TryMarkInterruptedAsync(
|
|
run.Id,
|
|
exception,
|
|
cancellationToken.IsCancellationRequested);
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
await reporterCancellation.CancelAsync();
|
|
await reporterTask;
|
|
}
|
|
}
|
|
|
|
private async Task GenerateAndPersistAsync(
|
|
GenerationRun run,
|
|
GenerationProgressState progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
Random random = new(_options.RandomSeed);
|
|
int channelCapacity = Math.Max(
|
|
_options.SaveBatchSize,
|
|
_options.MaxDegreeOfParallelism);
|
|
Channel<GeneratedItem> channel = Channel.CreateBounded<GeneratedItem>(
|
|
new BoundedChannelOptions(channelCapacity)
|
|
{
|
|
FullMode = BoundedChannelFullMode.Wait,
|
|
SingleReader = true,
|
|
SingleWriter = false
|
|
});
|
|
using CancellationTokenSource writerFailureCancellation = new();
|
|
using CancellationTokenSource generationCancellation =
|
|
CancellationTokenSource.CreateLinkedTokenSource(
|
|
cancellationToken,
|
|
writerFailureCancellation.Token);
|
|
Task<int> writerTask = PersistGeneratedItemsAsync(
|
|
channel,
|
|
run.Id,
|
|
run.LastCommittedPlanOrdinal + 1,
|
|
progress,
|
|
writerFailureCancellation);
|
|
ExceptionDispatchInfo? generationFailure = null;
|
|
long nextPlanOrdinal = 0;
|
|
|
|
try
|
|
{
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
IReadOnlyList<TrainingCategoryValue> categoryValues =
|
|
await _trainingDataReader.ReadCategoryValuesAsync(
|
|
category,
|
|
generationCancellation.Token);
|
|
|
|
IReadOnlyList<string> positivePatterns =
|
|
await _trainingDataReader.ReadPatternsAsync(
|
|
category,
|
|
true,
|
|
generationCancellation.Token);
|
|
nextPlanOrdinal += await ProcessPassAsync(
|
|
category,
|
|
categoryValues,
|
|
positivePatterns,
|
|
true,
|
|
random,
|
|
run.Id,
|
|
nextPlanOrdinal,
|
|
run.LastCommittedPlanOrdinal,
|
|
progress,
|
|
channel.Writer,
|
|
writerFailureCancellation.Token,
|
|
generationCancellation.Token);
|
|
|
|
IReadOnlyList<string> negativePatterns =
|
|
await _trainingDataReader.ReadPatternsAsync(
|
|
category,
|
|
false,
|
|
generationCancellation.Token);
|
|
nextPlanOrdinal += await ProcessPassAsync(
|
|
category,
|
|
categoryValues,
|
|
negativePatterns,
|
|
false,
|
|
random,
|
|
run.Id,
|
|
nextPlanOrdinal,
|
|
run.LastCommittedPlanOrdinal,
|
|
progress,
|
|
channel.Writer,
|
|
writerFailureCancellation.Token,
|
|
generationCancellation.Token);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
generationFailure = ExceptionDispatchInfo.Capture(exception);
|
|
await generationCancellation.CancelAsync();
|
|
}
|
|
finally
|
|
{
|
|
channel.Writer.TryComplete();
|
|
}
|
|
|
|
ExceptionDispatchInfo? writerFailure = null;
|
|
try
|
|
{
|
|
await writerTask;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
writerFailure = ExceptionDispatchInfo.Capture(exception);
|
|
}
|
|
|
|
writerFailure?.Throw();
|
|
generationFailure?.Throw();
|
|
}
|
|
|
|
private async Task<int> PersistGeneratedItemsAsync(
|
|
Channel<GeneratedItem> channel,
|
|
Guid runId,
|
|
long nextPlanOrdinal,
|
|
GenerationProgressState progress,
|
|
CancellationTokenSource writerFailureCancellation)
|
|
{
|
|
List<GeneratedItem> batch = new(_options.SaveBatchSize);
|
|
SortedDictionary<long, GeneratedItem> pendingItems = [];
|
|
int persistedRowCount = 0;
|
|
|
|
try
|
|
{
|
|
await foreach (GeneratedItem item in channel.Reader.ReadAllAsync())
|
|
{
|
|
pendingItems.Add(item.PlanOrdinal, item);
|
|
while (pendingItems.Remove(nextPlanOrdinal, out GeneratedItem? nextItem))
|
|
{
|
|
batch.Add(nextItem);
|
|
nextPlanOrdinal++;
|
|
if (batch.Count == _options.SaveBatchSize)
|
|
{
|
|
persistedRowCount += await SaveBatchAsync(runId, batch, progress);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (batch.Count > 0)
|
|
{
|
|
persistedRowCount += await SaveBatchAsync(runId, batch, progress);
|
|
}
|
|
|
|
return persistedRowCount;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
channel.Writer.TryComplete(exception);
|
|
await writerFailureCancellation.CancelAsync();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private async Task<int> SaveBatchAsync(
|
|
Guid runId,
|
|
List<GeneratedItem> batch,
|
|
GenerationProgressState progress)
|
|
{
|
|
int expectedRowCount = batch.Count;
|
|
GeneratedItem checkpoint = batch[^1];
|
|
await using IDbContextTransaction transaction =
|
|
await _dbContext.Database.BeginTransactionAsync(CancellationToken.None);
|
|
|
|
try
|
|
{
|
|
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
|
|
candidate => candidate.Id == runId,
|
|
CancellationToken.None);
|
|
_dbContext.TrainingItems.AddRange(batch.Select(item => item.Item));
|
|
run.CurrentCategoryId = checkpoint.CategoryId;
|
|
run.CurrentIsPositive = checkpoint.IsPositive;
|
|
run.LastCommittedPlanOrdinal = checkpoint.PlanOrdinal;
|
|
run.TotalCommittedRowCount += expectedRowCount;
|
|
run.LastCheckpointAt = DateTimeOffset.UtcNow;
|
|
await _dbContext.SaveChangesAsync(CancellationToken.None);
|
|
await transaction.CommitAsync(CancellationToken.None);
|
|
progress.SetCommittedCount(run.TotalCommittedRowCount);
|
|
|
|
_logger.LogInformation(
|
|
"Run {RunId} committed batch {BatchNumber}: category {CategoryId}, pass {Pass}, " +
|
|
"rows {BatchRowCount}, cumulative {CommittedRowCount}, checkpoint {CheckpointOrdinal}.",
|
|
runId,
|
|
((run.TotalCommittedRowCount - 1) / _options.SaveBatchSize) + 1,
|
|
checkpoint.CategoryId,
|
|
checkpoint.IsPositive ? "positive" : "negative",
|
|
expectedRowCount,
|
|
run.TotalCommittedRowCount,
|
|
checkpoint.PlanOrdinal);
|
|
}
|
|
catch
|
|
{
|
|
await transaction.RollbackAsync(CancellationToken.None);
|
|
_dbContext.ChangeTracker.Clear();
|
|
throw;
|
|
}
|
|
|
|
_dbContext.ChangeTracker.Clear();
|
|
batch.Clear();
|
|
return expectedRowCount;
|
|
}
|
|
|
|
private async Task<int> ProcessPassAsync(
|
|
TrainingCategoryDefinition category,
|
|
IReadOnlyList<TrainingCategoryValue> categoryValues,
|
|
IReadOnlyList<string> patterns,
|
|
bool isPositive,
|
|
Random random,
|
|
Guid runId,
|
|
long firstPlanOrdinal,
|
|
long lastCommittedPlanOrdinal,
|
|
GenerationProgressState progress,
|
|
ChannelWriter<GeneratedItem> writer,
|
|
CancellationToken writerCancellationToken,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
IReadOnlyList<PlannedItem> plan = CreatePlan(
|
|
category,
|
|
categoryValues,
|
|
patterns,
|
|
isPositive,
|
|
random);
|
|
ExceptionDispatchInfo? failure = null;
|
|
using CancellationTokenSource linkedCancellation =
|
|
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
|
|
ParallelOptions parallelOptions = new()
|
|
{
|
|
CancellationToken = linkedCancellation.Token,
|
|
MaxDegreeOfParallelism = _options.MaxDegreeOfParallelism
|
|
};
|
|
|
|
try
|
|
{
|
|
int firstUncommittedIndex = (int)Math.Clamp(
|
|
lastCommittedPlanOrdinal - firstPlanOrdinal + 1,
|
|
0,
|
|
plan.Count);
|
|
await Parallel.ForEachAsync(
|
|
Enumerable.Range(firstUncommittedIndex, plan.Count - firstUncommittedIndex),
|
|
parallelOptions,
|
|
async (index, itemCancellationToken) =>
|
|
{
|
|
try
|
|
{
|
|
PlannedItem plannedItem = plan[index];
|
|
string value;
|
|
if (plannedItem.RequiresLmStudio)
|
|
{
|
|
progress.LmRequestStarted();
|
|
try
|
|
{
|
|
value = await _lmStudioClient.CompleteAsync(
|
|
plannedItem.SubstitutedText,
|
|
plannedItem.Tense,
|
|
plannedItem.IsFragment,
|
|
plannedItem.SelectedSubject,
|
|
plannedItem.ProtectedMedicalTerm,
|
|
plannedItem.AllowedMedicalForms,
|
|
itemCancellationToken);
|
|
ValidateLmStudioResponse(value, category, isPositive);
|
|
}
|
|
finally
|
|
{
|
|
progress.LmRequestFinished();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
value = plannedItem.SubstitutedText;
|
|
}
|
|
|
|
progress.ItemProcessed();
|
|
|
|
long planOrdinal = firstPlanOrdinal + index;
|
|
TrainingItem item = new()
|
|
{
|
|
GenerationRunId = runId,
|
|
PlanOrdinal = planOrdinal,
|
|
RecordKey = CalculateRecordKey(
|
|
runId,
|
|
category.Id,
|
|
isPositive,
|
|
planOrdinal),
|
|
CategoryId = category.Id,
|
|
IsPositive = isPositive,
|
|
Value = value
|
|
};
|
|
await writer.WriteAsync(
|
|
new GeneratedItem(
|
|
planOrdinal,
|
|
category.Id,
|
|
isPositive,
|
|
item),
|
|
writerCancellationToken);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
Interlocked.CompareExchange(
|
|
ref failure,
|
|
ExceptionDispatchInfo.Capture(exception),
|
|
null);
|
|
linkedCancellation.Cancel();
|
|
throw;
|
|
}
|
|
});
|
|
}
|
|
catch
|
|
{
|
|
await linkedCancellation.CancelAsync();
|
|
failure?.Throw();
|
|
throw;
|
|
}
|
|
|
|
return plan.Count;
|
|
}
|
|
|
|
private async Task ReportProgressAsync(
|
|
GenerationProgressState progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
using PeriodicTimer timer = new(
|
|
TimeSpan.FromSeconds(_options.ProgressLogIntervalSeconds),
|
|
_timeProvider);
|
|
try
|
|
{
|
|
while (await timer.WaitForNextTickAsync(cancellationToken))
|
|
{
|
|
LogPeriodicProgress(progress);
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
}
|
|
}
|
|
|
|
private void LogPeriodicProgress(GenerationProgressState progress)
|
|
{
|
|
ProgressSnapshot snapshot = progress.TakeSnapshot();
|
|
_logger.LogInformation(
|
|
"Generation progress: {OverallCompleted}/{TotalExpected} ({Percent:F1}%) | " +
|
|
"session {SessionProcessed} | {CurrentRate:F1} items/s current | " +
|
|
"{AverageRate:F1} items/s average | committed {CommittedCount} | " +
|
|
"LM active {ActiveLmRequests}/{ConfiguredParallelism} | LM peak {PeakLmRequests} | " +
|
|
"ETA {Eta}",
|
|
snapshot.OverallCompleted,
|
|
snapshot.TotalExpected,
|
|
snapshot.Percent,
|
|
snapshot.SessionProcessed,
|
|
snapshot.CurrentRate,
|
|
snapshot.AverageRate,
|
|
snapshot.CommittedCount,
|
|
snapshot.ActiveLmRequests,
|
|
snapshot.ConfiguredParallelism,
|
|
snapshot.PeakLmRequests,
|
|
snapshot.Eta);
|
|
}
|
|
|
|
private void LogFinalProgress(
|
|
GenerationProgressState progress,
|
|
int committedCount,
|
|
bool interrupted)
|
|
{
|
|
ProgressSnapshot snapshot = progress.TakeSnapshot();
|
|
string message = interrupted
|
|
? "Generation execution interrupted: session processed {SessionProcessed}, committed " +
|
|
"{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " +
|
|
"LM active {ActiveLmRequests}, LM peak {PeakLmRequests}."
|
|
: "Generation execution completed: session processed {SessionProcessed}, committed " +
|
|
"{CommittedCount}, elapsed {Elapsed}, average {AverageRate:F1} items/s, " +
|
|
"LM active {ActiveLmRequests}, LM peak {PeakLmRequests}.";
|
|
if (interrupted)
|
|
{
|
|
_logger.LogWarning(
|
|
message,
|
|
snapshot.SessionProcessed,
|
|
committedCount,
|
|
snapshot.Elapsed,
|
|
snapshot.AverageRate,
|
|
snapshot.ActiveLmRequests,
|
|
snapshot.PeakLmRequests);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogInformation(
|
|
message,
|
|
snapshot.SessionProcessed,
|
|
committedCount,
|
|
snapshot.Elapsed,
|
|
snapshot.AverageRate,
|
|
snapshot.ActiveLmRequests,
|
|
snapshot.PeakLmRequests);
|
|
}
|
|
}
|
|
|
|
private async Task<(GenerationRun Run, bool Resumed)> GetOrCreateRunAsync(
|
|
string inputFingerprint,
|
|
string configurationFingerprint,
|
|
int totalExpectedRowCount,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
List<GenerationRun> incompleteRuns = await _dbContext.GenerationRuns
|
|
.Where(run => GenerationRunStatuses.Incomplete.Contains(run.Status))
|
|
.ToListAsync(cancellationToken);
|
|
incompleteRuns = incompleteRuns
|
|
.OrderByDescending(run => run.StartedAt)
|
|
.ToList();
|
|
if (incompleteRuns.Count > 1)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Multiple incomplete generation runs exist. Automatic resume was rejected.");
|
|
}
|
|
|
|
GenerationRun? run = incompleteRuns.FirstOrDefault(candidate =>
|
|
candidate.RandomSeed == _options.RandomSeed
|
|
&& candidate.InputFingerprint == inputFingerprint
|
|
&& candidate.ConfigurationFingerprint == configurationFingerprint);
|
|
|
|
if (run is null && incompleteRuns.Count > 0)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"An incomplete generation run exists, but its input data, configuration, or random seed changed. Unsafe resume was rejected.");
|
|
}
|
|
|
|
bool resumed = run is not null;
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
if (run is null)
|
|
{
|
|
run = new GenerationRun
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Status = GenerationRunStatuses.Running,
|
|
RandomSeed = _options.RandomSeed,
|
|
InputFingerprint = inputFingerprint,
|
|
ConfigurationFingerprint = configurationFingerprint,
|
|
TotalExpectedRowCount = totalExpectedRowCount,
|
|
StartedAt = now,
|
|
LastCheckpointAt = now
|
|
};
|
|
_dbContext.GenerationRuns.Add(run);
|
|
}
|
|
else
|
|
{
|
|
if (run.TotalExpectedRowCount != totalExpectedRowCount)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"The resumed generation plan count does not match its durable run state.");
|
|
}
|
|
|
|
int durableRowCount = await _dbContext.TrainingItems.CountAsync(
|
|
item => item.GenerationRunId == run.Id,
|
|
cancellationToken);
|
|
long durableLastOrdinal = await _dbContext.TrainingItems
|
|
.Where(item => item.GenerationRunId == run.Id)
|
|
.MaxAsync(item => (long?)item.PlanOrdinal, cancellationToken)
|
|
?? -1;
|
|
if (durableRowCount != run.TotalCommittedRowCount
|
|
|| durableLastOrdinal != run.LastCommittedPlanOrdinal)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"The resumed generation rows do not match the durable checkpoint.");
|
|
}
|
|
|
|
run.Status = GenerationRunStatuses.Running;
|
|
run.FailureType = null;
|
|
run.FailureMessage = null;
|
|
}
|
|
|
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
|
_logger.LogInformation(
|
|
"Run {RunId} {RunAction} with {CommittedRowCount} of {ExpectedRowCount} rows committed.",
|
|
run.Id,
|
|
resumed ? "resumed" : "started",
|
|
run.TotalCommittedRowCount,
|
|
run.TotalExpectedRowCount);
|
|
_dbContext.ChangeTracker.Clear();
|
|
return (run, resumed);
|
|
}
|
|
|
|
private async Task<GenerationRun> MarkCompletedAsync(
|
|
Guid runId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
|
|
candidate => candidate.Id == runId,
|
|
cancellationToken);
|
|
if (run.TotalCommittedRowCount != run.TotalExpectedRowCount)
|
|
{
|
|
throw new InvalidOperationException(
|
|
$"Run committed {run.TotalCommittedRowCount} rows but expected {run.TotalExpectedRowCount}.");
|
|
}
|
|
|
|
run.Status = GenerationRunStatuses.Completed;
|
|
run.CompletedAt = DateTimeOffset.UtcNow;
|
|
run.LastCheckpointAt = run.CompletedAt.Value;
|
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
|
_logger.LogInformation(
|
|
"Run {RunId} completed with {CommittedRowCount} rows.",
|
|
run.Id,
|
|
run.TotalCommittedRowCount);
|
|
return run;
|
|
}
|
|
|
|
private async Task TryMarkInterruptedAsync(
|
|
Guid runId,
|
|
Exception exception,
|
|
bool cancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
_dbContext.ChangeTracker.Clear();
|
|
GenerationRun run = await _dbContext.GenerationRuns.SingleAsync(
|
|
candidate => candidate.Id == runId,
|
|
CancellationToken.None);
|
|
run.Status = cancellationRequested
|
|
? GenerationRunStatuses.Cancelled
|
|
: GenerationRunStatuses.Failed;
|
|
string failureType = exception.GetType().FullName
|
|
?? exception.GetType().Name;
|
|
run.FailureType = failureType[..Math.Min(failureType.Length, 256)];
|
|
run.FailureMessage = GetSanitizedFailureMessage(exception);
|
|
run.LastCheckpointAt = DateTimeOffset.UtcNow;
|
|
await _dbContext.SaveChangesAsync(CancellationToken.None);
|
|
_logger.LogWarning(
|
|
"Run {RunId} {Status} after {CommittedRowCount} committed rows at checkpoint " +
|
|
"{CheckpointOrdinal}. Failure type: {FailureType}.",
|
|
run.Id,
|
|
run.Status,
|
|
run.TotalCommittedRowCount,
|
|
run.LastCommittedPlanOrdinal,
|
|
run.FailureType);
|
|
}
|
|
catch (Exception statusException)
|
|
{
|
|
_logger.LogError(
|
|
statusException,
|
|
"Run {RunId} failure status could not be persisted.",
|
|
runId);
|
|
}
|
|
}
|
|
|
|
private async Task<int> CalculateExpectedRowCountAsync(
|
|
CancellationToken cancellationToken)
|
|
{
|
|
Random random = new(_options.RandomSeed);
|
|
int total = 0;
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
IReadOnlyList<TrainingCategoryValue> values =
|
|
await _trainingDataReader.ReadCategoryValuesAsync(category, cancellationToken);
|
|
foreach (bool isPositive in new[] { true, false })
|
|
{
|
|
IReadOnlyList<string> patterns =
|
|
await _trainingDataReader.ReadPatternsAsync(
|
|
category,
|
|
isPositive,
|
|
cancellationToken);
|
|
total = checked(total + CreatePlan(
|
|
category,
|
|
values,
|
|
patterns,
|
|
isPositive,
|
|
random).Count);
|
|
}
|
|
}
|
|
|
|
return total;
|
|
}
|
|
|
|
private async Task<string> CalculateInputFingerprintAsync(
|
|
CancellationToken cancellationToken)
|
|
{
|
|
using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
|
IEnumerable<string> paths = Directory
|
|
.EnumerateFiles(_trainingDataReader.DataDirectory, "*.txt", SearchOption.AllDirectories)
|
|
.Where(path =>
|
|
{
|
|
string relativePath = Path.GetRelativePath(
|
|
_trainingDataReader.DataDirectory,
|
|
path);
|
|
return relativePath.Equals("Subjects.txt", StringComparison.Ordinal)
|
|
|| relativePath.StartsWith($"Categories{Path.DirectorySeparatorChar}", StringComparison.Ordinal)
|
|
|| relativePath.StartsWith($"Patterns{Path.DirectorySeparatorChar}", StringComparison.Ordinal);
|
|
})
|
|
.OrderBy(path => Path.GetRelativePath(
|
|
_trainingDataReader.DataDirectory,
|
|
path), StringComparer.Ordinal);
|
|
|
|
foreach (string path in paths)
|
|
{
|
|
string relativePath = Path.GetRelativePath(
|
|
_trainingDataReader.DataDirectory,
|
|
path).Replace('\\', '/');
|
|
hash.AppendData(Encoding.UTF8.GetBytes(relativePath));
|
|
hash.AppendData([0]);
|
|
hash.AppendData(await File.ReadAllBytesAsync(path, cancellationToken));
|
|
hash.AppendData([0]);
|
|
}
|
|
|
|
return Convert.ToHexString(hash.GetHashAndReset());
|
|
}
|
|
|
|
private string CalculateConfigurationFingerprint()
|
|
{
|
|
string configuration = string.Join('\n',
|
|
_options.RandomSeed.ToString(CultureInfo.InvariantCulture),
|
|
_options.MaxDegreeOfParallelism.ToString(CultureInfo.InvariantCulture),
|
|
_options.SaveBatchSize.ToString(CultureInfo.InvariantCulture),
|
|
_lmStudioOptions.ChatCompletionsUrl,
|
|
_lmStudioOptions.ModelName,
|
|
_lmStudioOptions.SystemPrompt,
|
|
_lmStudioOptions.Temperature.ToString("R", CultureInfo.InvariantCulture),
|
|
_lmStudioOptions.TopP.ToString("R", CultureInfo.InvariantCulture),
|
|
_lmStudioOptions.MaxTokens.ToString(CultureInfo.InvariantCulture),
|
|
_lmStudioOptions.Seed.ToString(CultureInfo.InvariantCulture),
|
|
_lmStudioOptions.TimeoutSeconds.ToString(CultureInfo.InvariantCulture));
|
|
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(configuration)));
|
|
}
|
|
|
|
private static string CalculateRecordKey(
|
|
Guid runId,
|
|
int categoryId,
|
|
bool isPositive,
|
|
long planOrdinal)
|
|
{
|
|
string identity = $"{runId:N}|{categoryId}|{isPositive}|{planOrdinal}";
|
|
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(identity)));
|
|
}
|
|
|
|
private static string GetSanitizedFailureMessage(Exception exception) => exception switch
|
|
{
|
|
OperationCanceledException => "Generation was cancelled.",
|
|
HttpRequestException => "The LM Studio request failed.",
|
|
InvalidDataException => "Generation input or model output failed validation.",
|
|
DbUpdateException => "A database batch could not be committed.",
|
|
_ => "Generation failed."
|
|
};
|
|
|
|
private static IReadOnlyList<PlannedItem> CreatePlan(
|
|
TrainingCategoryDefinition category,
|
|
IReadOnlyList<TrainingCategoryValue> categoryValues,
|
|
IReadOnlyList<string> patterns,
|
|
bool isPositive,
|
|
Random random)
|
|
{
|
|
List<TrainingCategoryValue> categoryValueDeck = [.. categoryValues];
|
|
List<string> valueBoundPatternDeck = [];
|
|
List<string> genericPatternDeck = [];
|
|
foreach (string pattern in patterns)
|
|
{
|
|
bool containsCategoryValuePlaceholder =
|
|
pattern.Contains("[x]", StringComparison.Ordinal);
|
|
if (containsCategoryValuePlaceholder)
|
|
{
|
|
valueBoundPatternDeck.Add(pattern);
|
|
}
|
|
else
|
|
{
|
|
genericPatternDeck.Add(pattern);
|
|
}
|
|
}
|
|
|
|
string passName = isPositive ? "positive" : "negative";
|
|
if (categoryValueDeck.Count > 0 && valueBoundPatternDeck.Count == 0)
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Category '{category.Name}' {passName} pass has category values but no pattern containing '[x]'.");
|
|
}
|
|
|
|
if (categoryValueDeck.Count == 0 && valueBoundPatternDeck.Count > 0)
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Category '{category.Name}' {passName} pass has patterns containing '[x]' but no category values.");
|
|
}
|
|
|
|
Shuffle(categoryValueDeck, random);
|
|
Shuffle(valueBoundPatternDeck, random);
|
|
Shuffle(genericPatternDeck, random);
|
|
|
|
List<string> subjectDeck = [.. TrainingSubjects.All];
|
|
Shuffle(subjectDeck, random);
|
|
|
|
int subjectIndex = 0;
|
|
List<string?> tenseDeck =
|
|
[
|
|
null,
|
|
SentenceTenses.Past,
|
|
SentenceTenses.Present,
|
|
SentenceTenses.Future
|
|
];
|
|
Shuffle(tenseDeck, random);
|
|
int tenseIndex = 0;
|
|
int valueBoundItemCount = Math.Max(
|
|
valueBoundPatternDeck.Count,
|
|
categoryValueDeck.Count);
|
|
List<PlannedItem> plan = new(
|
|
genericPatternDeck.Count + valueBoundItemCount);
|
|
|
|
foreach (string genericPattern in genericPatternDeck)
|
|
{
|
|
AddPlannedItem(genericPattern, null);
|
|
}
|
|
|
|
for (int itemIndex = 0; itemIndex < valueBoundItemCount; itemIndex++)
|
|
{
|
|
if (itemIndex > 0 && itemIndex % valueBoundPatternDeck.Count == 0)
|
|
{
|
|
Shuffle(valueBoundPatternDeck, random);
|
|
}
|
|
|
|
if (itemIndex > 0 && itemIndex % categoryValueDeck.Count == 0)
|
|
{
|
|
Shuffle(categoryValueDeck, random);
|
|
}
|
|
|
|
string pattern = valueBoundPatternDeck[itemIndex % valueBoundPatternDeck.Count];
|
|
TrainingCategoryValue categoryValue =
|
|
categoryValueDeck[itemIndex % categoryValueDeck.Count];
|
|
AddPlannedItem(pattern, categoryValue);
|
|
}
|
|
|
|
return plan;
|
|
|
|
void AddPlannedItem(string pattern, TrainingCategoryValue? categoryValue)
|
|
{
|
|
string substitutedText = categoryValue is null
|
|
? pattern
|
|
: pattern.Replace("[x]", categoryValue.CanonicalForm, StringComparison.Ordinal);
|
|
|
|
string? tense = null;
|
|
if (pattern.Contains("[is|are|was|were]", StringComparison.Ordinal))
|
|
{
|
|
if (tenseIndex == tenseDeck.Count)
|
|
{
|
|
Shuffle(tenseDeck, random);
|
|
tenseIndex = 0;
|
|
}
|
|
|
|
tense = tenseDeck[tenseIndex];
|
|
tenseIndex++;
|
|
}
|
|
|
|
bool containsSubject = pattern.Contains("[sub]", StringComparison.Ordinal);
|
|
bool isFragment = !containsSubject;
|
|
string? selectedSubject = null;
|
|
|
|
if (containsSubject)
|
|
{
|
|
if (subjectIndex == subjectDeck.Count)
|
|
{
|
|
Shuffle(subjectDeck, random);
|
|
subjectIndex = 0;
|
|
}
|
|
|
|
selectedSubject = subjectDeck[subjectIndex];
|
|
substitutedText = substitutedText.Replace(
|
|
"[sub]",
|
|
selectedSubject,
|
|
StringComparison.Ordinal);
|
|
subjectIndex++;
|
|
}
|
|
|
|
substitutedText = substitutedText.Trim();
|
|
|
|
if (string.IsNullOrWhiteSpace(substitutedText))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Substituted text is empty for category '{category.Name}' {passName} pass.");
|
|
}
|
|
|
|
if (substitutedText.Contains("[x]", StringComparison.Ordinal)
|
|
|| substitutedText.Contains("[sub]", StringComparison.Ordinal))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Unresolved placeholder remains for category '{category.Name}' {passName} pass.");
|
|
}
|
|
|
|
bool requiresLmStudio = containsSubject
|
|
|| substitutedText.Contains('[', StringComparison.Ordinal)
|
|
|| substitutedText.Contains(']', StringComparison.Ordinal)
|
|
|| tense is not null
|
|
|| categoryValue?.AllowedForms.Count > 1;
|
|
plan.Add(new PlannedItem(
|
|
substitutedText,
|
|
requiresLmStudio,
|
|
categoryValue?.CanonicalForm,
|
|
categoryValue?.AllowedForms,
|
|
selectedSubject,
|
|
tense,
|
|
isFragment));
|
|
}
|
|
}
|
|
|
|
private static void ValidateLmStudioResponse(
|
|
string value,
|
|
TrainingCategoryDefinition category,
|
|
bool isPositive)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(value)
|
|
|| value.Contains('\r', StringComparison.Ordinal)
|
|
|| value.Contains('\n', StringComparison.Ordinal)
|
|
|| value.Contains('[', StringComparison.Ordinal)
|
|
|| value.Contains(']', StringComparison.Ordinal)
|
|
|| IsResponseWrapper(value))
|
|
{
|
|
string passName = isPositive ? "positive" : "negative";
|
|
throw new InvalidDataException(
|
|
$"LM Studio returned invalid output for category '{category.Name}' {passName} pass.");
|
|
}
|
|
}
|
|
|
|
private static bool IsResponseWrapper(string value)
|
|
{
|
|
bool isDoubleQuoted = value.Length >= 2
|
|
&& value[0] == '"'
|
|
&& value[^1] == '"';
|
|
bool isSingleQuoted = value.Length >= 2
|
|
&& value[0] == '\''
|
|
&& value[^1] == '\'';
|
|
bool isJsonObject = value.Length >= 2
|
|
&& value[0] == '{'
|
|
&& value[^1] == '}';
|
|
bool isJsonArray = value.Length >= 2
|
|
&& value[0] == '['
|
|
&& value[^1] == ']';
|
|
if (isDoubleQuoted
|
|
|| isSingleQuoted
|
|
|| isJsonObject
|
|
|| isJsonArray
|
|
|| value.StartsWith("```", StringComparison.Ordinal))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return ResponsePrefixes.Any(prefix =>
|
|
value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
private static void Shuffle<T>(IList<T> items, Random random)
|
|
{
|
|
for (int index = items.Count - 1; index > 0; index--)
|
|
{
|
|
int swapIndex = random.Next(index + 1);
|
|
(items[index], items[swapIndex]) = (items[swapIndex], items[index]);
|
|
}
|
|
}
|
|
|
|
private sealed record PlannedItem(
|
|
string SubstitutedText,
|
|
bool RequiresLmStudio,
|
|
string? ProtectedMedicalTerm,
|
|
IReadOnlyList<string>? AllowedMedicalForms,
|
|
string? SelectedSubject,
|
|
string? Tense,
|
|
bool IsFragment);
|
|
|
|
private sealed record GeneratedItem(
|
|
long PlanOrdinal,
|
|
int CategoryId,
|
|
bool IsPositive,
|
|
TrainingItem Item);
|
|
|
|
private sealed class GenerationProgressState
|
|
{
|
|
private readonly int _totalExpected;
|
|
private readonly int _initiallyCommitted;
|
|
private readonly int _configuredParallelism;
|
|
private readonly TimeProvider _timeProvider;
|
|
private readonly long _startedTimestamp;
|
|
private long _lastSnapshotTimestamp;
|
|
private long _sessionProcessed;
|
|
private long _lastSnapshotProcessed;
|
|
private int _committedCount;
|
|
private int _activeLmRequests;
|
|
private int _peakLmRequests;
|
|
|
|
public GenerationProgressState(
|
|
int totalExpected,
|
|
int initiallyCommitted,
|
|
int configuredParallelism,
|
|
TimeProvider timeProvider)
|
|
{
|
|
_totalExpected = totalExpected;
|
|
_initiallyCommitted = initiallyCommitted;
|
|
_committedCount = initiallyCommitted;
|
|
_configuredParallelism = configuredParallelism;
|
|
_timeProvider = timeProvider;
|
|
_startedTimestamp = timeProvider.GetTimestamp();
|
|
_lastSnapshotTimestamp = _startedTimestamp;
|
|
}
|
|
|
|
public int CommittedCount => Volatile.Read(ref _committedCount);
|
|
|
|
public void ItemProcessed() => Interlocked.Increment(ref _sessionProcessed);
|
|
|
|
public void SetCommittedCount(int count) => Volatile.Write(ref _committedCount, count);
|
|
|
|
public void LmRequestStarted()
|
|
{
|
|
int active = Interlocked.Increment(ref _activeLmRequests);
|
|
int peak = Volatile.Read(ref _peakLmRequests);
|
|
while (active > peak)
|
|
{
|
|
int observed = Interlocked.CompareExchange(
|
|
ref _peakLmRequests,
|
|
active,
|
|
peak);
|
|
if (observed == peak)
|
|
{
|
|
break;
|
|
}
|
|
|
|
peak = observed;
|
|
}
|
|
}
|
|
|
|
public void LmRequestFinished() => Interlocked.Decrement(ref _activeLmRequests);
|
|
|
|
public ProgressSnapshot TakeSnapshot()
|
|
{
|
|
long timestamp = _timeProvider.GetTimestamp();
|
|
long processed = Volatile.Read(ref _sessionProcessed);
|
|
long previousProcessed = Interlocked.Exchange(
|
|
ref _lastSnapshotProcessed,
|
|
processed);
|
|
long previousTimestamp = Interlocked.Exchange(
|
|
ref _lastSnapshotTimestamp,
|
|
timestamp);
|
|
TimeSpan elapsed = _timeProvider.GetElapsedTime(_startedTimestamp, timestamp);
|
|
TimeSpan interval = _timeProvider.GetElapsedTime(previousTimestamp, timestamp);
|
|
double currentRate = interval.TotalSeconds > 0
|
|
? (processed - previousProcessed) / interval.TotalSeconds
|
|
: 0;
|
|
double averageRate = elapsed.TotalSeconds > 0
|
|
? processed / elapsed.TotalSeconds
|
|
: 0;
|
|
long overallCompleted = Math.Min(
|
|
_totalExpected,
|
|
_initiallyCommitted + processed);
|
|
long remaining = Math.Max(0, _totalExpected - overallCompleted);
|
|
double percent = _totalExpected > 0
|
|
? overallCompleted * 100.0 / _totalExpected
|
|
: 100;
|
|
string eta = averageRate > 0
|
|
? FormatEta(TimeSpan.FromSeconds(remaining / averageRate))
|
|
: "calculating";
|
|
return new ProgressSnapshot(
|
|
overallCompleted,
|
|
_totalExpected,
|
|
percent,
|
|
processed,
|
|
currentRate,
|
|
averageRate,
|
|
CommittedCount,
|
|
Volatile.Read(ref _activeLmRequests),
|
|
_configuredParallelism,
|
|
Volatile.Read(ref _peakLmRequests),
|
|
eta,
|
|
elapsed);
|
|
}
|
|
}
|
|
|
|
private sealed record ProgressSnapshot(
|
|
long OverallCompleted,
|
|
int TotalExpected,
|
|
double Percent,
|
|
long SessionProcessed,
|
|
double CurrentRate,
|
|
double AverageRate,
|
|
int CommittedCount,
|
|
int ActiveLmRequests,
|
|
int ConfiguredParallelism,
|
|
int PeakLmRequests,
|
|
string Eta,
|
|
TimeSpan Elapsed);
|
|
|
|
private static string FormatEta(TimeSpan eta)
|
|
{
|
|
int totalHours = Math.Max(0, (int)eta.TotalHours);
|
|
if (totalHours > 0)
|
|
{
|
|
return $"{totalHours}h {eta.Minutes}m";
|
|
}
|
|
|
|
if (eta.Minutes > 0)
|
|
{
|
|
return $"{eta.Minutes}m {eta.Seconds}s";
|
|
}
|
|
|
|
return $"{Math.Max(0, eta.Seconds)}s";
|
|
}
|
|
|
|
private static readonly string[] ResponsePrefixes =
|
|
[
|
|
"Output:",
|
|
"Result:",
|
|
"Completed text:",
|
|
"Corrected sentence:"
|
|
];
|
|
} |