Models-Phi-8B/PHI.Data.Generator/RLHF.Generator.Tests/CommittedBatchGenerationTes...

743 lines
28 KiB
C#

using System.Net;
using System.Text.Json;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using RLHF.Generator.Generation;
using RLHF.Generator.LmStudio;
using RLHF.Generator.Persistence;
using RLHF.Generator.TrainingData;
using Xunit;
namespace RLHF.Generator.Tests;
public sealed class CommittedBatchGenerationTests
{
[Fact]
public async Task SuccessfulLargeRun_CommitsConfiguredBatchesAndFinalPartialBatch()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1_001);
BatchObservationInterceptor observer = new();
await using ServiceInstance instance = fixture.CreateService(interceptor: observer);
GenerationRunResult result = await instance.Service.GenerateAsync();
Assert.Equal(2_024, result.TotalCommittedRows);
Assert.Equal(2_024, result.RowsCommittedThisRequest);
Assert.False(result.Resumed);
Assert.Equal(Enumerable.Repeat(10, 202).Append(4), observer.CommittedBatchSizes);
Assert.Equal(2_024, await fixture.CountRowsAsync());
}
[Fact]
public async Task SeparateConnection_ObservesCommittedBatchBeforeRunCompletes()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
1,
requiresLmStudio: true);
GatedLmStudioHandler handler = new(allowBeforeGate: 12);
await using ServiceInstance instance = fixture.CreateService(handler);
Task<GenerationRunResult> generationTask = instance.Service.GenerateAsync();
await handler.GateReached.WaitAsync(TimeSpan.FromSeconds(10));
int visibleRowCount = await fixture.WaitForVisibleRowsAsync();
Assert.Equal(10, visibleRowCount);
Assert.False(generationTask.IsCompleted);
handler.Release();
GenerationRunResult result = await generationTask;
Assert.Equal(24, result.TotalCommittedRows);
}
[Fact]
public async Task LmStudioCapitalization_IsStoredExactlyAsReturned()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
1,
requiresLmStudio: true);
fixture.ReplaceFirstCategoryValues("klippel-trenaunay syndrome");
CapitalizingLmStudioHandler handler = new();
await using ServiceInstance instance = fixture.CreateService(handler);
await instance.Service.GenerateAsync();
IReadOnlyList<string> storedValues = await fixture.GetValuesAsync();
Assert.NotEmpty(handler.CapitalizedResponses);
Assert.All(
handler.CapitalizedResponses,
returnedValue => Assert.Contains(returnedValue, storedValues));
}
[Fact]
public async Task FailureAfterOneThousandCommittedRows_ResumesWithoutDuplicates()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1_001);
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(103);
await using (ServiceInstance failedInstance = fixture.CreateService(interceptor: failure))
{
await Assert.ThrowsAsync<InvalidOperationException>(
() => failedInstance.Service.GenerateAsync());
}
GenerationRun failedRun = await fixture.GetSingleRunAsync();
Assert.Equal(GenerationRunStatuses.Failed, failedRun.Status);
Assert.Equal(1_010, failedRun.TotalCommittedRowCount);
Assert.Equal(1_009, failedRun.LastCommittedPlanOrdinal);
Assert.Equal(1_010, await fixture.CountRowsAsync());
await using ServiceInstance resumedInstance = fixture.CreateService();
GenerationRunResult resumed = await resumedInstance.Service.GenerateAsync();
Assert.True(resumed.Resumed);
Assert.Equal(1_014, resumed.RowsCommittedThisRequest);
Assert.Equal(2_024, resumed.TotalCommittedRows);
Assert.Equal(0, await fixture.CountDuplicateRecordKeysAsync());
Assert.Equal(2_024, await fixture.CountDistinctOrdinalsAsync(resumed.RunId));
}
[Fact]
public async Task Cancellation_PreservesCommittedRowsAndPermitsResume()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1_001);
using CancellationTokenSource cancellation = new();
BatchFaultInterceptor failure = BatchFaultInterceptor.CancelBeforeSave(4, cancellation);
await using (ServiceInstance failedInstance = fixture.CreateService(interceptor: failure))
{
await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => failedInstance.Service.GenerateAsync(cancellation.Token));
}
GenerationRun cancelledRun = await fixture.GetSingleRunAsync();
Assert.Equal(GenerationRunStatuses.Cancelled, cancelledRun.Status);
Assert.Equal(20, cancelledRun.TotalCommittedRowCount);
Assert.Equal(20, await fixture.CountRowsAsync());
await using ServiceInstance resumedInstance = fixture.CreateService();
GenerationRunResult resumed = await resumedInstance.Service.GenerateAsync();
Assert.True(resumed.Resumed);
Assert.Equal(2_024, resumed.TotalCommittedRows);
Assert.Equal(0, await fixture.CountDuplicateRecordKeysAsync());
}
[Fact]
public async Task LmStudioTimeout_PreservesCommittedBatchAndPermitsResume()
{
await AssertLmFailureResumesAsync(new TimeoutAfterResponsesHandler(12));
}
[Fact]
public async Task InvalidLmStudioOutput_PreservesCommittedBatchAndPermitsResume()
{
await AssertLmFailureResumesAsync(new InvalidOutputAfterResponsesHandler(12));
}
[Fact]
public async Task FailedBatch_DoesNotAdvanceCheckpointOrLeaveInsertedRows()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(20);
BatchFaultInterceptor failure = BatchFaultInterceptor.AfterSave(3);
await using ServiceInstance instance = fixture.CreateService(interceptor: failure);
await Assert.ThrowsAsync<InvalidOperationException>(
() => instance.Service.GenerateAsync());
GenerationRun run = await fixture.GetSingleRunAsync();
Assert.Equal(10, run.TotalCommittedRowCount);
Assert.Equal(9, run.LastCommittedPlanOrdinal);
Assert.Equal(10, await fixture.CountRowsAsync());
}
[Fact]
public async Task ChangedInputFiles_RejectUnsafeResume()
{
await using GenerationFixture fixture = await CreateFailedFixtureAsync();
fixture.AppendFirstCategoryValue("changed-input");
await using ServiceInstance instance = fixture.CreateService();
InvalidOperationException exception = await Assert.ThrowsAsync<InvalidOperationException>(
() => instance.Service.GenerateAsync());
Assert.Contains("Unsafe resume", exception.Message, StringComparison.Ordinal);
Assert.Equal(10, await fixture.CountRowsAsync());
}
[Fact]
public async Task ChangedRelevantConfiguration_RejectsUnsafeResume()
{
await using GenerationFixture fixture = await CreateFailedFixtureAsync();
LmStudioOptions changedOptions = fixture.CreateLmStudioOptions();
changedOptions.MaxTokens++;
await using ServiceInstance instance = fixture.CreateService(
lmStudioOptions: changedOptions);
InvalidOperationException exception = await Assert.ThrowsAsync<InvalidOperationException>(
() => instance.Service.GenerateAsync());
Assert.Contains("Unsafe resume", exception.Message, StringComparison.Ordinal);
Assert.Equal(10, await fixture.CountRowsAsync());
}
[Fact]
public async Task CompletedRun_IsNotResumedAsIncomplete()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1);
await using (ServiceInstance firstInstance = fixture.CreateService())
{
GenerationRunResult first = await firstInstance.Service.GenerateAsync();
Assert.False(first.Resumed);
}
await using ServiceInstance secondInstance = fixture.CreateService();
GenerationRunResult second = await secondInstance.Service.GenerateAsync();
Assert.False(second.Resumed);
Assert.Equal(2, await fixture.CountRunsAsync());
Assert.Equal(48, await fixture.CountRowsAsync());
Assert.Equal(0, await fixture.CountDuplicateRecordKeysAsync());
}
private static async Task AssertLmFailureResumesAsync(HttpMessageHandler failureHandler)
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
1,
requiresLmStudio: true);
await using (ServiceInstance failedInstance = fixture.CreateService(failureHandler))
{
await Assert.ThrowsAnyAsync<Exception>(
() => failedInstance.Service.GenerateAsync());
}
GenerationRun failedRun = await fixture.GetSingleRunAsync();
Assert.Equal(GenerationRunStatuses.Failed, failedRun.Status);
Assert.Equal(12, failedRun.TotalCommittedRowCount);
Assert.Equal(12, await fixture.CountRowsAsync());
await using ServiceInstance resumedInstance = fixture.CreateService(
new SuccessfulLmStudioHandler());
GenerationRunResult resumed = await resumedInstance.Service.GenerateAsync();
Assert.True(resumed.Resumed);
Assert.Equal(24, resumed.TotalCommittedRows);
Assert.Equal(0, await fixture.CountDuplicateRecordKeysAsync());
}
private static async Task<GenerationFixture> CreateFailedFixtureAsync()
{
GenerationFixture fixture = await GenerationFixture.CreateAsync(20);
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(3);
await using ServiceInstance instance = fixture.CreateService(interceptor: failure);
await Assert.ThrowsAsync<InvalidOperationException>(
() => instance.Service.GenerateAsync());
return fixture;
}
}
internal sealed class GenerationFixture : IAsyncDisposable
{
private readonly string _rootDirectory;
private readonly string _databasePath;
private readonly string _connectionString;
private GenerationFixture(string rootDirectory, string databasePath)
{
_rootDirectory = rootDirectory;
_databasePath = databasePath;
_connectionString = $"Data Source={databasePath};Pooling=False";
}
public string DataDirectory => Path.Combine(_rootDirectory, "Data");
public static async Task<GenerationFixture> CreateAsync(
int firstCategoryValueCount,
bool requiresLmStudio = false)
{
string rootDirectory = Path.Combine(
Path.GetTempPath(),
$"RLHF.Generator.Tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(rootDirectory);
GenerationFixture fixture = new(
rootDirectory,
Path.Combine(rootDirectory, "generation.db"));
fixture.WriteTrainingData(firstCategoryValueCount, requiresLmStudio);
await fixture.CreateDatabaseAsync();
return fixture;
}
public ServiceInstance CreateService(
HttpMessageHandler? handler = null,
SaveChangesInterceptor? interceptor = null,
GenerationOptions? generationOptions = null,
LmStudioOptions? lmStudioOptions = null,
ILogger<TrainingGenerationService>? logger = null,
TimeProvider? timeProvider = null)
{
DbContextOptionsBuilder<RlhfGeneratorDbContext> builder = new();
builder.UseSqlite(_connectionString);
if (interceptor is not null)
{
builder.AddInterceptors(interceptor);
}
RlhfGeneratorDbContext dbContext = new(builder.Options);
LmStudioOptions resolvedLmOptions = lmStudioOptions ?? CreateLmStudioOptions();
HttpClient httpClient = new(handler ?? new SuccessfulLmStudioHandler())
{
Timeout = TimeSpan.FromSeconds(resolvedLmOptions.TimeoutSeconds)
};
TrainingGenerationService service = new(
new TrainingDataReader(DataDirectory),
new LmStudioClient(httpClient, resolvedLmOptions),
dbContext,
generationOptions ?? CreateGenerationOptions(),
resolvedLmOptions,
logger ?? NullLogger<TrainingGenerationService>.Instance,
timeProvider);
return new ServiceInstance(dbContext, httpClient, service);
}
public GenerationOptions CreateGenerationOptions() => new()
{
RandomSeed = 42,
MaxDegreeOfParallelism = 4,
SaveBatchSize = 10,
ProgressLogIntervalSeconds = 5
};
public LmStudioOptions CreateLmStudioOptions() => new()
{
ChatCompletionsUrl = "http://localhost/v1/chat/completions",
ModelName = "test-model",
SystemPrompt = "test-system-prompt",
Temperature = 0,
TopP = 1,
MaxTokens = 32,
Seed = 42,
TimeoutSeconds = 30
};
public void AppendFirstCategoryValue(string value) => File.AppendAllText(
Path.Combine(DataDirectory, "Categories", "MEDICAL_ALLERGY.txt"),
$"{Environment.NewLine}{value}");
public void ReplaceFirstCategoryValues(params string[] values) => File.WriteAllLines(
Path.Combine(DataDirectory, "Categories", "MEDICAL_ALLERGY.txt"),
values);
public async Task<int> CountRowsAsync()
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.TrainingItems.CountAsync();
}
public async Task<int> CountRunsAsync()
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.GenerationRuns.CountAsync();
}
public async Task<IReadOnlyList<string>> GetValuesAsync()
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.TrainingItems
.AsNoTracking()
.Select(item => item.Value)
.ToListAsync();
}
public async Task<GenerationRun> GetSingleRunAsync()
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.GenerationRuns.AsNoTracking().SingleAsync();
}
public async Task<int> CountDuplicateRecordKeysAsync()
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.TrainingItems
.Where(item => item.RecordKey != null)
.GroupBy(item => item.RecordKey)
.CountAsync(group => group.Count() > 1);
}
public async Task<int> CountDistinctOrdinalsAsync(Guid runId)
{
await using RlhfGeneratorDbContext context = CreateContext();
return await context.TrainingItems
.Where(item => item.GenerationRunId == runId)
.Select(item => item.PlanOrdinal)
.Distinct()
.CountAsync();
}
public async Task<int> WaitForVisibleRowsAsync()
{
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10));
while (!timeout.IsCancellationRequested)
{
int count = await CountRowsAsync();
if (count > 0)
{
return count;
}
await Task.Delay(20, timeout.Token);
}
throw new TimeoutException("No committed batch became visible.");
}
public async ValueTask DisposeAsync()
{
SqliteConnection.ClearAllPools();
await Task.Yield();
Directory.Delete(_rootDirectory, true);
}
private RlhfGeneratorDbContext CreateContext()
{
DbContextOptions<RlhfGeneratorDbContext> options =
new DbContextOptionsBuilder<RlhfGeneratorDbContext>()
.UseSqlite(_connectionString)
.Options;
return new RlhfGeneratorDbContext(options);
}
private async Task CreateDatabaseAsync()
{
await using SqliteConnection connection = new(_connectionString);
await connection.OpenAsync();
await using SqliteCommand command = connection.CreateCommand();
command.CommandText =
"""
PRAGMA foreign_keys = ON;
CREATE TABLE Category (
Id INTEGER NOT NULL PRIMARY KEY,
Name TEXT NOT NULL
);
CREATE TABLE GenerationRun (
Id TEXT NOT NULL PRIMARY KEY,
Status TEXT NOT NULL,
RandomSeed INTEGER NOT NULL,
InputFingerprint TEXT NOT NULL,
ConfigurationFingerprint TEXT NOT NULL,
CurrentCategoryId INTEGER NULL,
CurrentIsPositive INTEGER NULL,
LastCommittedPlanOrdinal INTEGER NOT NULL,
TotalExpectedRowCount INTEGER NOT NULL,
TotalCommittedRowCount INTEGER NOT NULL,
StartedAt TEXT NOT NULL,
LastCheckpointAt TEXT NOT NULL,
CompletedAt TEXT NULL,
FailureType TEXT NULL,
FailureMessage TEXT NULL,
FOREIGN KEY (CurrentCategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
);
CREATE TABLE TrainingItem (
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
GenerationRunId TEXT NULL,
PlanOrdinal INTEGER NULL,
RecordKey TEXT NULL,
CategoryId INTEGER NOT NULL,
KeywordId INTEGER NULL,
IsPositive INTEGER NOT NULL,
Value TEXT NOT NULL,
FOREIGN KEY (GenerationRunId) REFERENCES GenerationRun (Id) ON DELETE RESTRICT,
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
);
CREATE UNIQUE INDEX IX_TrainingItem_RecordKey
ON TrainingItem (RecordKey) WHERE RecordKey IS NOT NULL;
CREATE UNIQUE INDEX IX_TrainingItem_GenerationRunId_PlanOrdinal
ON TrainingItem (GenerationRunId, PlanOrdinal)
WHERE GenerationRunId IS NOT NULL AND PlanOrdinal IS NOT NULL;
""";
await command.ExecuteNonQueryAsync();
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
command.Parameters.Clear();
command.CommandText = "INSERT INTO Category (Id, Name) VALUES ($id, $name);";
command.Parameters.AddWithValue("$id", category.Id);
command.Parameters.AddWithValue("$name", category.Name);
await command.ExecuteNonQueryAsync();
}
}
private void WriteTrainingData(int firstCategoryValueCount, bool requiresLmStudio)
{
string categoriesDirectory = Path.Combine(DataDirectory, "Categories");
string patternsDirectory = Path.Combine(DataDirectory, "Patterns");
Directory.CreateDirectory(categoriesDirectory);
Directory.CreateDirectory(patternsDirectory);
File.WriteAllLines(Path.Combine(DataDirectory, "Subjects.txt"), TrainingSubjects.All);
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
int valueCount = category.Id == 1 ? firstCategoryValueCount : 1;
File.WriteAllLines(
Path.Combine(categoriesDirectory, $"{category.Name}.txt"),
Enumerable.Range(1, valueCount).Select(index => $"value-{category.Id}-{index}"));
string subject = requiresLmStudio ? "[sub] " : string.Empty;
File.WriteAllText(
Path.Combine(patternsDirectory, $"{category.Name}.positive.txt"),
$"{subject}confirms [x]");
File.WriteAllText(
Path.Combine(patternsDirectory, $"{category.Name}.negative.txt"),
$"{subject}denies [x]");
}
}
}
internal sealed class ServiceInstance : IAsyncDisposable
{
private readonly RlhfGeneratorDbContext _dbContext;
private readonly HttpClient _httpClient;
public ServiceInstance(
RlhfGeneratorDbContext dbContext,
HttpClient httpClient,
TrainingGenerationService service)
{
_dbContext = dbContext;
_httpClient = httpClient;
Service = service;
}
public TrainingGenerationService Service { get; }
public async ValueTask DisposeAsync()
{
await _dbContext.DisposeAsync();
_httpClient.Dispose();
}
}
internal sealed class BatchObservationInterceptor : SaveChangesInterceptor
{
public List<int> CommittedBatchSizes { get; } = [];
public override ValueTask<int> SavedChangesAsync(
SaveChangesCompletedEventData eventData,
int result,
CancellationToken cancellationToken = default)
{
int batchSize = eventData.Context!.ChangeTracker
.Entries<TrainingItem>()
.Count(entry => entry.State == EntityState.Unchanged);
if (batchSize > 0)
{
CommittedBatchSizes.Add(batchSize);
}
return base.SavedChangesAsync(eventData, result, cancellationToken);
}
}
internal sealed class BatchFaultInterceptor : SaveChangesInterceptor
{
private readonly int _failureCall;
private readonly bool _failAfterSave;
private readonly CancellationTokenSource? _cancellation;
private int _saveCallCount;
private bool _failed;
private BatchFaultInterceptor(
int failureCall,
bool failAfterSave,
CancellationTokenSource? cancellation)
{
_failureCall = failureCall;
_failAfterSave = failAfterSave;
_cancellation = cancellation;
}
public static BatchFaultInterceptor BeforeSave(int failureCall) =>
new(failureCall, false, null);
public static BatchFaultInterceptor AfterSave(int failureCall) =>
new(failureCall, true, null);
public static BatchFaultInterceptor CancelBeforeSave(
int failureCall,
CancellationTokenSource cancellation) =>
new(failureCall, false, cancellation);
public override ValueTask<InterceptionResult<int>> SavingChangesAsync(
DbContextEventData eventData,
InterceptionResult<int> result,
CancellationToken cancellationToken = default)
{
int call = Interlocked.Increment(ref _saveCallCount);
if (!_failed && !_failAfterSave && call == _failureCall)
{
_failed = true;
if (_cancellation is not null)
{
_cancellation.Cancel();
throw new OperationCanceledException(_cancellation.Token);
}
throw new InvalidOperationException("Injected batch persistence failure.");
}
return base.SavingChangesAsync(eventData, result, cancellationToken);
}
public override ValueTask<int> SavedChangesAsync(
SaveChangesCompletedEventData eventData,
int result,
CancellationToken cancellationToken = default)
{
if (!_failed && _failAfterSave && _saveCallCount == _failureCall)
{
_failed = true;
throw new InvalidOperationException("Injected failure before batch commit.");
}
return base.SavedChangesAsync(eventData, result, cancellationToken);
}
}
internal class SuccessfulLmStudioHandler : HttpMessageHandler
{
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
string sentence = await ReadSentenceAsync(request, cancellationToken);
return CreateResponse(sentence);
}
protected static async Task<string> ReadSentenceAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
string requestBody = await request.Content!.ReadAsStringAsync(cancellationToken);
using JsonDocument requestJson = JsonDocument.Parse(requestBody);
string userContent = requestJson.RootElement
.GetProperty("messages")[1]
.GetProperty("content")
.GetString()!;
return userContent
.Split('\n')
.Single(line => line.StartsWith("Sentence: ", StringComparison.Ordinal))[10..];
}
protected static HttpResponseMessage CreateResponse(string content)
{
string responseBody = JsonSerializer.Serialize(new
{
choices = new[] { new { message = new { content } } }
});
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(responseBody)
};
}
}
internal sealed class GatedLmStudioHandler : SuccessfulLmStudioHandler
{
private readonly int _allowBeforeGate;
private readonly TaskCompletionSource _gateReached =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _requestCount;
public GatedLmStudioHandler(int allowBeforeGate)
{
_allowBeforeGate = allowBeforeGate;
}
public Task GateReached => _gateReached.Task;
public void Release() => _release.TrySetResult();
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _requestCount) > _allowBeforeGate)
{
_gateReached.TrySetResult();
await _release.Task.WaitAsync(cancellationToken);
}
string sentence = await ReadSentenceAsync(request, cancellationToken);
return CreateResponse(sentence);
}
}
internal sealed class CapitalizingLmStudioHandler : SuccessfulLmStudioHandler
{
private readonly System.Collections.Concurrent.ConcurrentBag<string>
_capitalizedResponses = [];
public IReadOnlyCollection<string> CapitalizedResponses => _capitalizedResponses;
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
string sentence = await ReadSentenceAsync(request, cancellationToken);
string returnedValue = sentence.Replace(
"klippel-trenaunay syndrome",
"Klippel-Trenaunay syndrome",
StringComparison.Ordinal);
if (!string.Equals(sentence, returnedValue, StringComparison.Ordinal))
{
_capitalizedResponses.Add(returnedValue);
}
return CreateResponse(returnedValue);
}
}
internal sealed class TimeoutAfterResponsesHandler : SuccessfulLmStudioHandler
{
private readonly int _successfulResponses;
private int _requestCount;
public TimeoutAfterResponsesHandler(int successfulResponses)
{
_successfulResponses = successfulResponses;
}
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _requestCount) > _successfulResponses)
{
throw new TaskCanceledException("Injected LM Studio timeout.");
}
string sentence = await ReadSentenceAsync(request, cancellationToken);
return CreateResponse(sentence);
}
}
internal sealed class InvalidOutputAfterResponsesHandler : SuccessfulLmStudioHandler
{
private readonly int _successfulResponses;
private int _requestCount;
public InvalidOutputAfterResponsesHandler(int successfulResponses)
{
_successfulResponses = successfulResponses;
}
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
int requestNumber = Interlocked.Increment(ref _requestCount);
string sentence = await ReadSentenceAsync(request, cancellationToken);
return CreateResponse(requestNumber > _successfulResponses
? $"\"{sentence}\""
: sentence);
}
}