314 lines
12 KiB
C#
314 lines
12 KiB
C#
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.Mvc.Testing;
|
|
using Microsoft.AspNetCore.TestHost;
|
|
using Microsoft.Data.Sqlite;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.DependencyInjection.Extensions;
|
|
using RLHF.Generator.Api.Contracts;
|
|
using RLHF.Generator.Generation;
|
|
using RLHF.Generator.LmStudio;
|
|
using RLHF.Generator.Persistence;
|
|
using RLHF.Generator.TrainingData;
|
|
using Xunit;
|
|
|
|
namespace RLHF.Generator.Tests;
|
|
|
|
public sealed class GenerationEndpointAwaitingTests
|
|
{
|
|
[Fact]
|
|
public async Task ManualEndpoint_AcceptsCaseInsensitiveCanonicalFormMatch()
|
|
{
|
|
await using EndpointFixture fixture = new();
|
|
fixture.Handler.ReleaseResponses();
|
|
using HttpClient client = fixture.Factory.CreateClient();
|
|
|
|
using HttpResponseMessage response = await client.PostAsJsonAsync(
|
|
"/api/lm-studio/complete",
|
|
new CompleteTextRequest(
|
|
"She received a Klippel-Trenaunay syndrome diagnosis.",
|
|
IsFragment: false,
|
|
ProtectedMedicalTerm: "klippel-trenaunay syndrome",
|
|
AllowedMedicalForms: ["Klippel-Trenaunay syndrome"]));
|
|
|
|
response.EnsureSuccessStatusCode();
|
|
Assert.Equal(
|
|
"She received a Klippel-Trenaunay syndrome diagnosis.",
|
|
await response.Content.ReadAsStringAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ManualEndpoint_RejectsCaseInsensitiveDuplicateForms()
|
|
{
|
|
await using EndpointFixture fixture = new();
|
|
using HttpClient client = fixture.Factory.CreateClient();
|
|
|
|
using HttpResponseMessage response = await client.PostAsJsonAsync(
|
|
"/api/lm-studio/complete",
|
|
new CompleteTextRequest(
|
|
"Member is taking Aleve.",
|
|
IsFragment: false,
|
|
ProtectedMedicalTerm: "Aleve",
|
|
AllowedMedicalForms: ["Aleve", "aleve"]));
|
|
|
|
Assert.Equal(System.Net.HttpStatusCode.BadRequest, response.StatusCode);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunEndpoint_AwaitsGenerationAndCommitBeforeResponding()
|
|
{
|
|
await using EndpointFixture fixture = new();
|
|
using HttpClient client = fixture.Factory.CreateClient();
|
|
|
|
Task<HttpResponseMessage> responseTask = client.PostAsync(
|
|
"/api/generation/run",
|
|
null);
|
|
await fixture.Handler.RequestStarted.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
Assert.False(responseTask.IsCompleted);
|
|
|
|
fixture.Handler.ReleaseResponses();
|
|
using HttpResponseMessage response = await responseTask;
|
|
response.EnsureSuccessStatusCode();
|
|
GenerationRunResponse? result =
|
|
await response.Content.ReadFromJsonAsync<GenerationRunResponse>();
|
|
|
|
Assert.NotNull(result);
|
|
Assert.NotEqual(Guid.Empty, result.RunId);
|
|
Assert.Equal(GenerationRunStatuses.Completed, result.Status);
|
|
Assert.Equal(24, result.TotalCommittedRows);
|
|
Assert.Equal(24, result.RowsCommittedThisRequest);
|
|
Assert.Equal(24, result.TotalExpectedRows);
|
|
Assert.False(result.Resumed);
|
|
Assert.Equal(
|
|
24,
|
|
await fixture.DbContext.TrainingItems.AsNoTracking().CountAsync());
|
|
}
|
|
|
|
private sealed class EndpointFixture : IAsyncDisposable
|
|
{
|
|
private readonly string _dataDirectory;
|
|
private readonly SqliteConnection _connection;
|
|
|
|
public EndpointFixture()
|
|
{
|
|
_dataDirectory = CreateTrainingData();
|
|
_connection = new SqliteConnection("Data Source=:memory:");
|
|
_connection.Open();
|
|
CreateDatabase(_connection);
|
|
|
|
DbContextOptions<RlhfGeneratorDbContext> dbOptions =
|
|
new DbContextOptionsBuilder<RlhfGeneratorDbContext>()
|
|
.UseSqlite(_connection)
|
|
.Options;
|
|
DbContext = new RlhfGeneratorDbContext(dbOptions);
|
|
Handler = new GatedLmStudioHandler();
|
|
Factory = new AtomicGenerationApiFactory(
|
|
dbOptions,
|
|
_dataDirectory,
|
|
Handler);
|
|
}
|
|
|
|
public RlhfGeneratorDbContext DbContext { get; }
|
|
|
|
public GatedLmStudioHandler Handler { get; }
|
|
|
|
public WebApplicationFactory<Program> Factory { get; }
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
await Factory.DisposeAsync();
|
|
await DbContext.DisposeAsync();
|
|
await _connection.DisposeAsync();
|
|
Directory.Delete(_dataDirectory, true);
|
|
}
|
|
|
|
private static void CreateDatabase(SqliteConnection connection)
|
|
{
|
|
using SqliteCommand command = connection.CreateCommand();
|
|
command.CommandText =
|
|
"""
|
|
CREATE TABLE Category (
|
|
Id INTEGER NOT NULL PRIMARY KEY,
|
|
Name TEXT NOT NULL
|
|
);
|
|
CREATE TABLE TrainingItem (
|
|
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
|
GenerationRunId TEXT NULL,
|
|
PlanOrdinal INTEGER NULL,
|
|
RecordKey TEXT NULL,
|
|
CategoryId INTEGER NOT NULL,
|
|
KeywordId INTEGER NULL,
|
|
IsPositive INTEGER NOT NULL,
|
|
Value TEXT NOT NULL,
|
|
FOREIGN KEY (GenerationRunId) REFERENCES GenerationRun (Id) ON DELETE RESTRICT,
|
|
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
|
|
);
|
|
CREATE TABLE GenerationRun (
|
|
Id TEXT NOT NULL PRIMARY KEY,
|
|
Status TEXT NOT NULL,
|
|
RandomSeed INTEGER NOT NULL,
|
|
InputFingerprint TEXT NOT NULL,
|
|
ConfigurationFingerprint TEXT NOT NULL,
|
|
CurrentCategoryId INTEGER NULL,
|
|
CurrentIsPositive INTEGER NULL,
|
|
LastCommittedPlanOrdinal INTEGER NOT NULL,
|
|
TotalExpectedRowCount INTEGER NOT NULL,
|
|
TotalCommittedRowCount INTEGER NOT NULL,
|
|
StartedAt TEXT NOT NULL,
|
|
LastCheckpointAt TEXT NOT NULL,
|
|
CompletedAt TEXT NULL,
|
|
FailureType TEXT NULL,
|
|
FailureMessage TEXT NULL,
|
|
FOREIGN KEY (CurrentCategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
|
|
);
|
|
CREATE INDEX IX_TrainingItem_CategoryId ON TrainingItem (CategoryId);
|
|
CREATE UNIQUE INDEX IX_TrainingItem_RecordKey
|
|
ON TrainingItem (RecordKey) WHERE RecordKey IS NOT NULL;
|
|
CREATE UNIQUE INDEX IX_TrainingItem_GenerationRunId_PlanOrdinal
|
|
ON TrainingItem (GenerationRunId, PlanOrdinal)
|
|
WHERE GenerationRunId IS NOT NULL AND PlanOrdinal IS NOT NULL;
|
|
""";
|
|
command.ExecuteNonQuery();
|
|
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
command.Parameters.Clear();
|
|
command.CommandText =
|
|
"INSERT INTO Category (Id, Name) VALUES ($id, $name);";
|
|
command.Parameters.AddWithValue("$id", category.Id);
|
|
command.Parameters.AddWithValue("$name", category.Name);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
|
|
private static string CreateTrainingData()
|
|
{
|
|
string dataDirectory = Path.Combine(
|
|
Path.GetTempPath(),
|
|
$"RLHF.Generator.EndpointTests-{Guid.NewGuid():N}");
|
|
string categoriesDirectory = Path.Combine(dataDirectory, "Categories");
|
|
string patternsDirectory = Path.Combine(dataDirectory, "Patterns");
|
|
Directory.CreateDirectory(categoriesDirectory);
|
|
Directory.CreateDirectory(patternsDirectory);
|
|
File.WriteAllLines(
|
|
Path.Combine(dataDirectory, "Subjects.txt"),
|
|
TrainingSubjects.All);
|
|
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
File.WriteAllText(
|
|
Path.Combine(categoriesDirectory, $"{category.Name}.txt"),
|
|
$"value-{category.Id}");
|
|
File.WriteAllText(
|
|
Path.Combine(patternsDirectory, $"{category.Name}.positive.txt"),
|
|
"[sub] confirms [x]");
|
|
File.WriteAllText(
|
|
Path.Combine(patternsDirectory, $"{category.Name}.negative.txt"),
|
|
"[sub] denies [x]");
|
|
}
|
|
|
|
return dataDirectory;
|
|
}
|
|
}
|
|
|
|
private sealed class AtomicGenerationApiFactory : WebApplicationFactory<Program>
|
|
{
|
|
private readonly DbContextOptions<RlhfGeneratorDbContext> _dbOptions;
|
|
private readonly string _dataDirectory;
|
|
private readonly GatedLmStudioHandler _handler;
|
|
|
|
public AtomicGenerationApiFactory(
|
|
DbContextOptions<RlhfGeneratorDbContext> dbOptions,
|
|
string dataDirectory,
|
|
GatedLmStudioHandler handler)
|
|
{
|
|
_dbOptions = dbOptions;
|
|
_dataDirectory = dataDirectory;
|
|
_handler = handler;
|
|
}
|
|
|
|
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
|
{
|
|
builder.ConfigureTestServices(services =>
|
|
{
|
|
services.RemoveAll<RlhfGeneratorDbContext>();
|
|
services.RemoveAll<DbContextOptions<RlhfGeneratorDbContext>>();
|
|
services.AddScoped(_ => new RlhfGeneratorDbContext(_dbOptions));
|
|
|
|
services.RemoveAll<TrainingDataReader>();
|
|
services.AddSingleton(new TrainingDataReader(_dataDirectory));
|
|
|
|
GenerationOptions generationOptions = new()
|
|
{
|
|
RandomSeed = 42,
|
|
MaxDegreeOfParallelism = 4,
|
|
SaveBatchSize = 10,
|
|
ProgressLogIntervalSeconds = 5
|
|
};
|
|
services.RemoveAll<GenerationOptions>();
|
|
services.AddSingleton(generationOptions);
|
|
|
|
LmStudioOptions lmStudioOptions = new()
|
|
{
|
|
ChatCompletionsUrl = "http://localhost/v1/chat/completions",
|
|
ModelName = "endpoint-test",
|
|
SystemPrompt = "endpoint-test",
|
|
Temperature = 0,
|
|
TopP = 1,
|
|
MaxTokens = 32,
|
|
Seed = 42,
|
|
TimeoutSeconds = 30
|
|
};
|
|
services.RemoveAll<LmStudioOptions>();
|
|
services.AddSingleton(lmStudioOptions);
|
|
services.RemoveAll<LmStudioClient>();
|
|
services.AddSingleton(
|
|
new LmStudioClient(new HttpClient(_handler), lmStudioOptions));
|
|
});
|
|
}
|
|
}
|
|
|
|
private sealed class GatedLmStudioHandler : HttpMessageHandler
|
|
{
|
|
private readonly TaskCompletionSource _requestStarted =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private readonly TaskCompletionSource _release =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public Task RequestStarted => _requestStarted.Task;
|
|
|
|
public void ReleaseResponses() => _release.TrySetResult();
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
_requestStarted.TrySetResult();
|
|
await _release.Task.WaitAsync(cancellationToken);
|
|
|
|
string requestBody = await request.Content!.ReadAsStringAsync(cancellationToken);
|
|
using JsonDocument requestJson = JsonDocument.Parse(requestBody);
|
|
string userContent = requestJson.RootElement
|
|
.GetProperty("messages")[1]
|
|
.GetProperty("content")
|
|
.GetString()!;
|
|
string sentence = userContent
|
|
.Split('\n')
|
|
.Single(line => line.StartsWith("Sentence: ", StringComparison.Ordinal))[10..];
|
|
string responseBody = JsonSerializer.Serialize(new
|
|
{
|
|
choices = new[]
|
|
{
|
|
new { message = new { content = sentence } }
|
|
}
|
|
});
|
|
return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
|
{
|
|
Content = new StringContent(responseBody)
|
|
};
|
|
}
|
|
}
|
|
} |