PHI-Model-8B/PHI.Data.Generator/RLHF.Generator.Tests/CaseInsensitiveLmValidation...

126 lines
4.5 KiB
C#

using System.Net;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text.Json;
using RLHF.Generator.Generation;
using RLHF.Generator.LmStudio;
using RLHF.Generator.TrainingData;
using Xunit;
namespace RLHF.Generator.Tests;
public sealed class CaseInsensitiveLmValidationTests
{
[Theory]
[InlineData("Member received a varicose vein diagnosis.")]
[InlineData("Member taught a class about Legg-Calvé-Perthes disease.")]
[InlineData("She reported an allergy to shellfish.")]
[InlineData("The member does not have a penicillin allergy.")]
[InlineData("allergic to honeybees")]
[InlineData("history of a heart attack")]
[InlineData("Helpful information about allergies")]
public void NaturalCorrections_PassStructuralValidation(string returnedValue)
{
ValidateResponse(returnedValue);
}
[Theory]
[InlineData("")]
[InlineData(" ")]
[InlineData("Member has an allergy.\rFollow-up is required.")]
[InlineData("Member has an allergy.\nFollow-up is required.")]
[InlineData("Member has [a|an] peanut allergy.")]
[InlineData("Member has an allergy to [x].")]
[InlineData("\"Member has a peanut allergy.\"")]
[InlineData("'Member has a peanut allergy.'")]
[InlineData("```Member has a peanut allergy.```")]
[InlineData("```text\nMember has a peanut allergy.\n```")]
[InlineData("{\"text\":\"Member has a peanut allergy.\"}")]
[InlineData("[\"Member has a peanut allergy.\"]")]
[InlineData("Output: Member has a peanut allergy.")]
[InlineData("result: Member has a peanut allergy.")]
[InlineData("Completed text: Member has a peanut allergy.")]
[InlineData("CORRECTED SENTENCE: Member has a peanut allergy.")]
public void StructurallyInvalidOutput_IsRejected(string returnedValue)
{
Assert.Throws<InvalidDataException>(() => ValidateResponse(returnedValue));
}
[Theory]
[InlineData("The result: Member has a peanut allergy.")]
[InlineData("Member said, \"I have a peanut allergy.\"")]
[InlineData("Member's allergy is documented.")]
[InlineData("Object {value} is discussed without square brackets.")]
public void OrdinarySentenceSyntax_IsNotTreatedAsAWrapper(string returnedValue)
{
ValidateResponse(returnedValue);
}
[Fact]
public async Task LmStudioClient_TrimsOnlySurroundingWhitespace()
{
const string generatedText = " Member has a peanut allergy. ";
LmStudioOptions options = new()
{
ChatCompletionsUrl = "http://localhost/v1/chat/completions",
ModelName = "test-model",
SystemPrompt = "test prompt",
Temperature = 0,
TopP = 1,
MaxTokens = 32,
Seed = 42,
TimeoutSeconds = 30
};
using HttpClient httpClient = new(new FixedResponseHandler(generatedText));
LmStudioClient client = new(httpClient, options);
string result = await client.CompleteAsync(
"Member has a peanut allergy.",
tense: null,
isFragment: false);
Assert.Equal("Member has a peanut allergy.", result);
}
private static void ValidateResponse(string returnedValue)
{
MethodInfo validateMethod = typeof(TrainingGenerationService).GetMethod(
"ValidateLmStudioResponse",
BindingFlags.Static | BindingFlags.NonPublic)!;
InvokeUnwrapped(
validateMethod,
[returnedValue, TrainingCategories.All[0], true]);
}
private static object? InvokeUnwrapped(MethodInfo method, object?[] arguments)
{
try
{
return method.Invoke(null, arguments);
}
catch (TargetInvocationException exception) when (exception.InnerException is not null)
{
ExceptionDispatchInfo.Capture(exception.InnerException).Throw();
throw;
}
}
private sealed class FixedResponseHandler(string content) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
string responseBody = JsonSerializer.Serialize(new
{
choices = new[] { new { message = new { content } } }
});
HttpResponseMessage response = new(HttpStatusCode.OK)
{
Content = new StringContent(responseBody)
};
return Task.FromResult(response);
}
}
}