PHI-Model-8B/PHI.Data.Generator/RLHF.Generator.Tests/GenerationEndpointAwaitingT...

314 lines
12 KiB
C#

using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using RLHF.Generator.Api.Contracts;
using RLHF.Generator.Generation;
using RLHF.Generator.LmStudio;
using RLHF.Generator.Persistence;
using RLHF.Generator.TrainingData;
using Xunit;
namespace RLHF.Generator.Tests;
public sealed class GenerationEndpointAwaitingTests
{
[Fact]
public async Task ManualEndpoint_AcceptsCaseInsensitiveCanonicalFormMatch()
{
await using EndpointFixture fixture = new();
fixture.Handler.ReleaseResponses();
using HttpClient client = fixture.Factory.CreateClient();
using HttpResponseMessage response = await client.PostAsJsonAsync(
"/api/lm-studio/complete",
new CompleteTextRequest(
"She received a Klippel-Trenaunay syndrome diagnosis.",
IsFragment: false,
ProtectedMedicalTerm: "klippel-trenaunay syndrome",
AllowedMedicalForms: ["Klippel-Trenaunay syndrome"]));
response.EnsureSuccessStatusCode();
Assert.Equal(
"She received a Klippel-Trenaunay syndrome diagnosis.",
await response.Content.ReadAsStringAsync());
}
[Fact]
public async Task ManualEndpoint_RejectsCaseInsensitiveDuplicateForms()
{
await using EndpointFixture fixture = new();
using HttpClient client = fixture.Factory.CreateClient();
using HttpResponseMessage response = await client.PostAsJsonAsync(
"/api/lm-studio/complete",
new CompleteTextRequest(
"Member is taking Aleve.",
IsFragment: false,
ProtectedMedicalTerm: "Aleve",
AllowedMedicalForms: ["Aleve", "aleve"]));
Assert.Equal(System.Net.HttpStatusCode.BadRequest, response.StatusCode);
}
[Fact]
public async Task RunEndpoint_AwaitsGenerationAndCommitBeforeResponding()
{
await using EndpointFixture fixture = new();
using HttpClient client = fixture.Factory.CreateClient();
Task<HttpResponseMessage> responseTask = client.PostAsync(
"/api/generation/run",
null);
await fixture.Handler.RequestStarted.WaitAsync(TimeSpan.FromSeconds(10));
Assert.False(responseTask.IsCompleted);
fixture.Handler.ReleaseResponses();
using HttpResponseMessage response = await responseTask;
response.EnsureSuccessStatusCode();
GenerationRunResponse? result =
await response.Content.ReadFromJsonAsync<GenerationRunResponse>();
Assert.NotNull(result);
Assert.NotEqual(Guid.Empty, result.RunId);
Assert.Equal(GenerationRunStatuses.Completed, result.Status);
Assert.Equal(24, result.TotalCommittedRows);
Assert.Equal(24, result.RowsCommittedThisRequest);
Assert.Equal(24, result.TotalExpectedRows);
Assert.False(result.Resumed);
Assert.Equal(
24,
await fixture.DbContext.TrainingItems.AsNoTracking().CountAsync());
}
private sealed class EndpointFixture : IAsyncDisposable
{
private readonly string _dataDirectory;
private readonly SqliteConnection _connection;
public EndpointFixture()
{
_dataDirectory = CreateTrainingData();
_connection = new SqliteConnection("Data Source=:memory:");
_connection.Open();
CreateDatabase(_connection);
DbContextOptions<RlhfGeneratorDbContext> dbOptions =
new DbContextOptionsBuilder<RlhfGeneratorDbContext>()
.UseSqlite(_connection)
.Options;
DbContext = new RlhfGeneratorDbContext(dbOptions);
Handler = new GatedLmStudioHandler();
Factory = new AtomicGenerationApiFactory(
dbOptions,
_dataDirectory,
Handler);
}
public RlhfGeneratorDbContext DbContext { get; }
public GatedLmStudioHandler Handler { get; }
public WebApplicationFactory<Program> Factory { get; }
public async ValueTask DisposeAsync()
{
await Factory.DisposeAsync();
await DbContext.DisposeAsync();
await _connection.DisposeAsync();
Directory.Delete(_dataDirectory, true);
}
private static void CreateDatabase(SqliteConnection connection)
{
using SqliteCommand command = connection.CreateCommand();
command.CommandText =
"""
CREATE TABLE Category (
Id INTEGER NOT NULL PRIMARY KEY,
Name TEXT NOT NULL
);
CREATE TABLE TrainingItem (
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
GenerationRunId TEXT NULL,
PlanOrdinal INTEGER NULL,
RecordKey TEXT NULL,
CategoryId INTEGER NOT NULL,
KeywordId INTEGER NULL,
IsPositive INTEGER NOT NULL,
Value TEXT NOT NULL,
FOREIGN KEY (GenerationRunId) REFERENCES GenerationRun (Id) ON DELETE RESTRICT,
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
);
CREATE TABLE GenerationRun (
Id TEXT NOT NULL PRIMARY KEY,
Status TEXT NOT NULL,
RandomSeed INTEGER NOT NULL,
InputFingerprint TEXT NOT NULL,
ConfigurationFingerprint TEXT NOT NULL,
CurrentCategoryId INTEGER NULL,
CurrentIsPositive INTEGER NULL,
LastCommittedPlanOrdinal INTEGER NOT NULL,
TotalExpectedRowCount INTEGER NOT NULL,
TotalCommittedRowCount INTEGER NOT NULL,
StartedAt TEXT NOT NULL,
LastCheckpointAt TEXT NOT NULL,
CompletedAt TEXT NULL,
FailureType TEXT NULL,
FailureMessage TEXT NULL,
FOREIGN KEY (CurrentCategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
);
CREATE INDEX IX_TrainingItem_CategoryId ON TrainingItem (CategoryId);
CREATE UNIQUE INDEX IX_TrainingItem_RecordKey
ON TrainingItem (RecordKey) WHERE RecordKey IS NOT NULL;
CREATE UNIQUE INDEX IX_TrainingItem_GenerationRunId_PlanOrdinal
ON TrainingItem (GenerationRunId, PlanOrdinal)
WHERE GenerationRunId IS NOT NULL AND PlanOrdinal IS NOT NULL;
""";
command.ExecuteNonQuery();
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
command.Parameters.Clear();
command.CommandText =
"INSERT INTO Category (Id, Name) VALUES ($id, $name);";
command.Parameters.AddWithValue("$id", category.Id);
command.Parameters.AddWithValue("$name", category.Name);
command.ExecuteNonQuery();
}
}
private static string CreateTrainingData()
{
string dataDirectory = Path.Combine(
Path.GetTempPath(),
$"RLHF.Generator.EndpointTests-{Guid.NewGuid():N}");
string categoriesDirectory = Path.Combine(dataDirectory, "Categories");
string patternsDirectory = Path.Combine(dataDirectory, "Patterns");
Directory.CreateDirectory(categoriesDirectory);
Directory.CreateDirectory(patternsDirectory);
File.WriteAllLines(
Path.Combine(dataDirectory, "Subjects.txt"),
TrainingSubjects.All);
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
File.WriteAllText(
Path.Combine(categoriesDirectory, $"{category.Name}.txt"),
$"value-{category.Id}");
File.WriteAllText(
Path.Combine(patternsDirectory, $"{category.Name}.positive.txt"),
"[sub] confirms [x]");
File.WriteAllText(
Path.Combine(patternsDirectory, $"{category.Name}.negative.txt"),
"[sub] denies [x]");
}
return dataDirectory;
}
}
private sealed class AtomicGenerationApiFactory : WebApplicationFactory<Program>
{
private readonly DbContextOptions<RlhfGeneratorDbContext> _dbOptions;
private readonly string _dataDirectory;
private readonly GatedLmStudioHandler _handler;
public AtomicGenerationApiFactory(
DbContextOptions<RlhfGeneratorDbContext> dbOptions,
string dataDirectory,
GatedLmStudioHandler handler)
{
_dbOptions = dbOptions;
_dataDirectory = dataDirectory;
_handler = handler;
}
protected override void ConfigureWebHost(IWebHostBuilder builder)
{
builder.ConfigureTestServices(services =>
{
services.RemoveAll<RlhfGeneratorDbContext>();
services.RemoveAll<DbContextOptions<RlhfGeneratorDbContext>>();
services.AddScoped(_ => new RlhfGeneratorDbContext(_dbOptions));
services.RemoveAll<TrainingDataReader>();
services.AddSingleton(new TrainingDataReader(_dataDirectory));
GenerationOptions generationOptions = new()
{
RandomSeed = 42,
MaxDegreeOfParallelism = 4,
SaveBatchSize = 10,
ProgressLogIntervalSeconds = 5
};
services.RemoveAll<GenerationOptions>();
services.AddSingleton(generationOptions);
LmStudioOptions lmStudioOptions = new()
{
ChatCompletionsUrl = "http://localhost/v1/chat/completions",
ModelName = "endpoint-test",
SystemPrompt = "endpoint-test",
Temperature = 0,
TopP = 1,
MaxTokens = 32,
Seed = 42,
TimeoutSeconds = 30
};
services.RemoveAll<LmStudioOptions>();
services.AddSingleton(lmStudioOptions);
services.RemoveAll<LmStudioClient>();
services.AddSingleton(
new LmStudioClient(new HttpClient(_handler), lmStudioOptions));
});
}
}
private sealed class GatedLmStudioHandler : HttpMessageHandler
{
private readonly TaskCompletionSource _requestStarted =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task RequestStarted => _requestStarted.Task;
public void ReleaseResponses() => _release.TrySetResult();
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
_requestStarted.TrySetResult();
await _release.Task.WaitAsync(cancellationToken);
string requestBody = await request.Content!.ReadAsStringAsync(cancellationToken);
using JsonDocument requestJson = JsonDocument.Parse(requestBody);
string userContent = requestJson.RootElement
.GetProperty("messages")[1]
.GetProperty("content")
.GetString()!;
string sentence = userContent
.Split('\n')
.Single(line => line.StartsWith("Sentence: ", StringComparison.Ordinal))[10..];
string responseBody = JsonSerializer.Serialize(new
{
choices = new[]
{
new { message = new { content = sentence } }
}
});
return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(responseBody)
};
}
}
}