using System.Net; using System.Reflection; using System.Runtime.ExceptionServices; using System.Text.Json; using RLHF.Generator.Generation; using RLHF.Generator.LmStudio; using RLHF.Generator.TrainingData; using Xunit; namespace RLHF.Generator.Tests; public sealed class CaseInsensitiveLmValidationTests { [Theory] [InlineData("Member received a varicose vein diagnosis.")] [InlineData("Member taught a class about Legg-Calvé-Perthes disease.")] [InlineData("She reported an allergy to shellfish.")] [InlineData("The member does not have a penicillin allergy.")] [InlineData("allergic to honeybees")] [InlineData("history of a heart attack")] [InlineData("Helpful information about allergies")] public void NaturalCorrections_PassStructuralValidation(string returnedValue) { ValidateResponse(returnedValue); } [Theory] [InlineData("")] [InlineData(" ")] [InlineData("Member has an allergy.\rFollow-up is required.")] [InlineData("Member has an allergy.\nFollow-up is required.")] [InlineData("Member has [a|an] peanut allergy.")] [InlineData("Member has an allergy to [x].")] [InlineData("\"Member has a peanut allergy.\"")] [InlineData("'Member has a peanut allergy.'")] [InlineData("```Member has a peanut allergy.```")] [InlineData("```text\nMember has a peanut allergy.\n```")] [InlineData("{\"text\":\"Member has a peanut allergy.\"}")] [InlineData("[\"Member has a peanut allergy.\"]")] [InlineData("Output: Member has a peanut allergy.")] [InlineData("result: Member has a peanut allergy.")] [InlineData("Completed text: Member has a peanut allergy.")] [InlineData("CORRECTED SENTENCE: Member has a peanut allergy.")] public void StructurallyInvalidOutput_IsRejected(string returnedValue) { Assert.Throws(() => ValidateResponse(returnedValue)); } [Theory] [InlineData("The result: Member has a peanut allergy.")] [InlineData("Member said, \"I have a peanut allergy.\"")] [InlineData("Member's allergy is documented.")] [InlineData("Object {value} is discussed without square brackets.")] public void OrdinarySentenceSyntax_IsNotTreatedAsAWrapper(string returnedValue) { ValidateResponse(returnedValue); } [Fact] public async Task LmStudioClient_TrimsOnlySurroundingWhitespace() { const string generatedText = " Member has a peanut allergy. "; LmStudioOptions options = new() { ChatCompletionsUrl = "http://localhost/v1/chat/completions", ModelName = "test-model", SystemPrompt = "test prompt", Temperature = 0, TopP = 1, MaxTokens = 32, Seed = 42, TimeoutSeconds = 30 }; using HttpClient httpClient = new(new FixedResponseHandler(generatedText)); LmStudioClient client = new(httpClient, options); string result = await client.CompleteAsync( "Member has a peanut allergy.", tense: null, isFragment: false); Assert.Equal("Member has a peanut allergy.", result); } private static void ValidateResponse(string returnedValue) { MethodInfo validateMethod = typeof(TrainingGenerationService).GetMethod( "ValidateLmStudioResponse", BindingFlags.Static | BindingFlags.NonPublic)!; InvokeUnwrapped( validateMethod, [returnedValue, TrainingCategories.All[0], true]); } private static object? InvokeUnwrapped(MethodInfo method, object?[] arguments) { try { return method.Invoke(null, arguments); } catch (TargetInvocationException exception) when (exception.InnerException is not null) { ExceptionDispatchInfo.Capture(exception.InnerException).Throw(); throw; } } private sealed class FixedResponseHandler(string content) : HttpMessageHandler { protected override Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { string responseBody = JsonSerializer.Serialize(new { choices = new[] { new { message = new { content } } } }); HttpResponseMessage response = new(HttpStatusCode.OK) { Content = new StringContent(responseBody) }; return Task.FromResult(response); } } }