126 lines
4.5 KiB
C#
126 lines
4.5 KiB
C#
using System.Net;
|
|
using System.Reflection;
|
|
using System.Runtime.ExceptionServices;
|
|
using System.Text.Json;
|
|
using RLHF.Generator.Generation;
|
|
using RLHF.Generator.LmStudio;
|
|
using RLHF.Generator.TrainingData;
|
|
using Xunit;
|
|
|
|
namespace RLHF.Generator.Tests;
|
|
|
|
public sealed class CaseInsensitiveLmValidationTests
|
|
{
|
|
[Theory]
|
|
[InlineData("Member received a varicose vein diagnosis.")]
|
|
[InlineData("Member taught a class about Legg-Calvé-Perthes disease.")]
|
|
[InlineData("She reported an allergy to shellfish.")]
|
|
[InlineData("The member does not have a penicillin allergy.")]
|
|
[InlineData("allergic to honeybees")]
|
|
[InlineData("history of a heart attack")]
|
|
[InlineData("Helpful information about allergies")]
|
|
public void NaturalCorrections_PassStructuralValidation(string returnedValue)
|
|
{
|
|
ValidateResponse(returnedValue);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("")]
|
|
[InlineData(" ")]
|
|
[InlineData("Member has an allergy.\rFollow-up is required.")]
|
|
[InlineData("Member has an allergy.\nFollow-up is required.")]
|
|
[InlineData("Member has [a|an] peanut allergy.")]
|
|
[InlineData("Member has an allergy to [x].")]
|
|
[InlineData("\"Member has a peanut allergy.\"")]
|
|
[InlineData("'Member has a peanut allergy.'")]
|
|
[InlineData("```Member has a peanut allergy.```")]
|
|
[InlineData("```text\nMember has a peanut allergy.\n```")]
|
|
[InlineData("{\"text\":\"Member has a peanut allergy.\"}")]
|
|
[InlineData("[\"Member has a peanut allergy.\"]")]
|
|
[InlineData("Output: Member has a peanut allergy.")]
|
|
[InlineData("result: Member has a peanut allergy.")]
|
|
[InlineData("Completed text: Member has a peanut allergy.")]
|
|
[InlineData("CORRECTED SENTENCE: Member has a peanut allergy.")]
|
|
public void StructurallyInvalidOutput_IsRejected(string returnedValue)
|
|
{
|
|
Assert.Throws<InvalidDataException>(() => ValidateResponse(returnedValue));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("The result: Member has a peanut allergy.")]
|
|
[InlineData("Member said, \"I have a peanut allergy.\"")]
|
|
[InlineData("Member's allergy is documented.")]
|
|
[InlineData("Object {value} is discussed without square brackets.")]
|
|
public void OrdinarySentenceSyntax_IsNotTreatedAsAWrapper(string returnedValue)
|
|
{
|
|
ValidateResponse(returnedValue);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task LmStudioClient_TrimsOnlySurroundingWhitespace()
|
|
{
|
|
const string generatedText = " Member has a peanut allergy. ";
|
|
LmStudioOptions options = new()
|
|
{
|
|
ChatCompletionsUrl = "http://localhost/v1/chat/completions",
|
|
ModelName = "test-model",
|
|
SystemPrompt = "test prompt",
|
|
Temperature = 0,
|
|
TopP = 1,
|
|
MaxTokens = 32,
|
|
Seed = 42,
|
|
TimeoutSeconds = 30
|
|
};
|
|
using HttpClient httpClient = new(new FixedResponseHandler(generatedText));
|
|
LmStudioClient client = new(httpClient, options);
|
|
|
|
string result = await client.CompleteAsync(
|
|
"Member has a peanut allergy.",
|
|
tense: null,
|
|
isFragment: false);
|
|
|
|
Assert.Equal("Member has a peanut allergy.", result);
|
|
}
|
|
|
|
private static void ValidateResponse(string returnedValue)
|
|
{
|
|
MethodInfo validateMethod = typeof(TrainingGenerationService).GetMethod(
|
|
"ValidateLmStudioResponse",
|
|
BindingFlags.Static | BindingFlags.NonPublic)!;
|
|
|
|
InvokeUnwrapped(
|
|
validateMethod,
|
|
[returnedValue, TrainingCategories.All[0], true]);
|
|
}
|
|
|
|
private static object? InvokeUnwrapped(MethodInfo method, object?[] arguments)
|
|
{
|
|
try
|
|
{
|
|
return method.Invoke(null, arguments);
|
|
}
|
|
catch (TargetInvocationException exception) when (exception.InnerException is not null)
|
|
{
|
|
ExceptionDispatchInfo.Capture(exception.InnerException).Throw();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private sealed class FixedResponseHandler(string content) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
string responseBody = JsonSerializer.Serialize(new
|
|
{
|
|
choices = new[] { new { message = new { content } } }
|
|
});
|
|
HttpResponseMessage response = new(HttpStatusCode.OK)
|
|
{
|
|
Content = new StringContent(responseBody)
|
|
};
|
|
return Task.FromResult(response);
|
|
}
|
|
}
|
|
} |