Models-Phi-8B/PHI.Data.Generator/RLHF.Generator.Tests/KeywordCatalogPhase1Tests.cs

324 lines
14 KiB
C#

using Microsoft.Data.SqlClient;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using RLHF.Generator.Keywords;
using RLHF.Generator.Persistence;
using RLHF.Generator.TrainingData;
using Xunit;
namespace RLHF.Generator.Tests;
public sealed class KeywordCatalogPhase1Tests
{
[Fact]
public async Task Model_ContainsNullableKeywordRelationshipAndRequiredIndexes()
{
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
await using RlhfGeneratorDbContext context = fixture.CreateContext();
Microsoft.EntityFrameworkCore.Metadata.IEntityType keyword =
context.Model.FindEntityType(typeof(Keyword))!;
Microsoft.EntityFrameworkCore.Metadata.IEntityType variant =
context.Model.FindEntityType(typeof(KeywordVariant))!;
Microsoft.EntityFrameworkCore.Metadata.IEntityType trainingItem =
context.Model.FindEntityType(typeof(TrainingItem))!;
Assert.Contains(
keyword.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(Keyword.CategoryId), nameof(Keyword.Value)]));
Assert.Contains(
variant.GetIndexes(),
index => index.IsUnique
&& index.Properties.Select(property => property.Name)
.SequenceEqual([nameof(KeywordVariant.ParentId), nameof(KeywordVariant.Value)]));
Assert.True(trainingItem.FindProperty(nameof(TrainingItem.KeywordId))!.IsNullable);
Assert.Contains(
trainingItem.GetIndexes(),
index => !index.IsUnique
&& index.Properties.Single().Name == nameof(TrainingItem.KeywordId));
Assert.Contains(
keyword.GetForeignKeys(),
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Category)
&& foreignKey.PrincipalToDependent?.Name == nameof(Category.Keywords)
&& foreignKey.DeleteBehavior == DeleteBehavior.Restrict);
Assert.Contains(
variant.GetForeignKeys(),
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Keyword)
&& foreignKey.DeleteBehavior == DeleteBehavior.Cascade);
Assert.Contains(
trainingItem.GetForeignKeys(),
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Keyword)
&& foreignKey.DeleteBehavior == DeleteBehavior.Restrict);
}
[Fact]
public async Task Load_IsAdditiveCaseInsensitiveAndIdempotent()
{
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
fixture.WriteCategoryValues(
TrainingCategories.MedicalDiagnosis,
"Café-au-lait spot|cafe au lait spot");
await using RlhfGeneratorDbContext context = fixture.CreateContext();
context.Keywords.Add(new Keyword
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "CAFÉ-AU-LAIT SPOT"
});
await context.SaveChangesAsync();
KeywordLoadService service = new(context, new TrainingDataReader(fixture.DataDirectory));
KeywordLoadResult first = await service.LoadAsync();
KeywordLoadResult second = await service.LoadAsync();
Assert.Equal(TrainingCategories.All.Count, first.SourceKeywordCount);
Assert.Equal(1, first.SourceVariantCount);
Assert.Equal(TrainingCategories.All.Count - 1, first.InsertedKeywordCount);
Assert.Equal(1, first.ExistingKeywordCount);
Assert.Equal(1, first.InsertedVariantCount);
Assert.Equal(0, second.InsertedKeywordCount);
Assert.Equal(0, second.InsertedVariantCount);
Assert.Equal(TrainingCategories.All.Count, await context.Keywords.CountAsync());
Keyword reused = await context.Keywords.SingleAsync(keyword =>
keyword.CategoryId == TrainingCategories.MedicalDiagnosis.Id);
KeywordVariant variant = Assert.Single(await context.KeywordVariants.ToListAsync());
Assert.Equal(reused.Id, variant.ParentId);
Assert.False(string.Equals(
reused.Value,
variant.Value,
StringComparison.OrdinalIgnoreCase));
}
[Fact]
public async Task Backfill_DryRunApplyAndRerun_HandleNormalizationBoundariesAndAmbiguity()
{
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
await using RlhfGeneratorDbContext context = fixture.CreateContext();
Keyword cafe = new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "café-au-lait spot",
Variants = [new KeywordVariant { Value = "cafe au lait spot" }]
};
Keyword pain = new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "pain"
};
Keyword chestPain = new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "chest pain"
};
Keyword sharedOne = new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "shared surface"
};
Keyword sharedTwo = new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
Value = "different canonical",
Variants = [new KeywordVariant { Value = "shared surface" }]
};
context.Keywords.AddRange(cafe, pain, chestPain, sharedOne, sharedTwo);
await context.SaveChangesAsync();
Guid generationRunId = Guid.NewGuid();
TrainingItem alreadyAssigned = new()
{
GenerationRunId = generationRunId,
PlanOrdinal = 42,
RecordKey = new string('a', 64),
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
KeywordId = pain.Id,
IsPositive = false,
Value = "Existing assigned metadata"
};
context.TrainingItems.AddRange(
CreateItem("The CAFE AU LAIT SPOT is unchanged."),
CreateItem("There is chest pain."),
CreateItem("There is chest pain and a café-au-lait spot."),
CreateItem("A shared surface is present."),
CreateItem("The painting is unchanged."),
new TrainingItem
{
CategoryId = TrainingCategories.MedicalAllergy.Id,
IsPositive = true,
Value = "Pain is mentioned in another category."
},
alreadyAssigned);
await context.SaveChangesAsync();
TrainingItemKeywordBackfillService service = new(context);
TrainingItemKeywordBackfillResult dryRun = await service.BackfillAsync(batchSize: 5);
Assert.Equal(7, dryRun.TotalTrainingItemCount);
Assert.Equal(1, dryRun.AlreadyAssignedCount);
Assert.Equal(6, dryRun.ProcessedCount);
Assert.Equal(2, dryRun.UniqueMatchCount);
Assert.Equal(2, dryRun.UnmatchedCount);
Assert.Equal(2, dryRun.AmbiguousCount);
Assert.Equal(0, dryRun.UpdatedCount);
Assert.Equal(1, await context.TrainingItems.CountAsync(item => item.KeywordId != null));
TrainingItemKeywordBackfillResult applied = await service.BackfillAsync(false, 5);
Assert.Equal(2, applied.UpdatedCount);
Assert.Equal(3, await context.TrainingItems.CountAsync(item => item.KeywordId != null));
TrainingItem preserved = await context.TrainingItems.SingleAsync(
item => item.Id == alreadyAssigned.Id);
Assert.Equal(pain.Id, preserved.KeywordId);
Assert.Equal(generationRunId, preserved.GenerationRunId);
Assert.Equal(42, preserved.PlanOrdinal);
Assert.Equal(new string('a', 64), preserved.RecordKey);
Assert.False(preserved.IsPositive);
Assert.Equal("Existing assigned metadata", preserved.Value);
TrainingItemKeywordBackfillResult rerun = await service.BackfillAsync(false, 5);
Assert.Equal(3, rerun.AlreadyAssignedCount);
Assert.Equal(4, rerun.ProcessedCount);
Assert.Equal(0, rerun.UpdatedCount);
}
[Fact]
public async Task Backfill_RejectsMissingCatalogAndInvalidBatchSize()
{
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
await using RlhfGeneratorDbContext context = fixture.CreateContext();
TrainingItemKeywordBackfillService service = new(context);
await Assert.ThrowsAsync<KeywordsNotLoadedException>(() => service.BackfillAsync());
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(() => service.BackfillAsync(batchSize: 0));
}
[Fact]
public void DesignTimeFactory_UsesOnlyGuardedSqlServerCatalog()
{
RlhfGeneratorDesignTimeDbContextFactory factory = new();
using RlhfGeneratorDbContext context = factory.CreateDbContext([]);
string connectionString = context.Database.GetConnectionString()!;
SqlConnectionStringBuilder connection = new(connectionString);
Assert.True(context.Database.IsSqlServer());
Assert.Equal(
RlhfGeneratorDesignTimeDbContextFactory.DesignTimeDatabaseName,
connection.InitialCatalog);
InvalidOperationException productionException = Assert.Throws<InvalidOperationException>(
() => RlhfGeneratorDesignTimeDbContextFactory.ValidateDatabaseName("RLHFGenerator"));
Assert.Contains(
"forbidden from using the production RLHFGenerator database",
productionException.Message,
StringComparison.Ordinal);
Assert.Throws<InvalidOperationException>(
() => RlhfGeneratorDesignTimeDbContextFactory.ValidateDatabaseName("anything-else"));
}
private static TrainingItem CreateItem(string value) => new()
{
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
IsPositive = true,
Value = value
};
}
internal sealed class KeywordFixture : IAsyncDisposable
{
private readonly string _rootDirectory;
private readonly string _connectionString;
private KeywordFixture(string rootDirectory)
{
_rootDirectory = rootDirectory;
_connectionString = $"Data Source={Path.Combine(rootDirectory, "keywords.db")};Pooling=False";
}
public string DataDirectory => Path.Combine(_rootDirectory, "Data");
public static async Task<KeywordFixture> CreateAsync()
{
string rootDirectory = Path.Combine(
Path.GetTempPath(),
$"RLHF.Generator.Keywords.Tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(Path.Combine(rootDirectory, "Data", "Categories"));
KeywordFixture fixture = new(rootDirectory);
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
fixture.WriteCategoryValues(category, $"value-{category.Id}");
}
await fixture.CreateDatabaseAsync();
return fixture;
}
public RlhfGeneratorDbContext CreateContext()
{
DbContextOptions<RlhfGeneratorDbContext> options =
new DbContextOptionsBuilder<RlhfGeneratorDbContext>()
.UseSqlite(_connectionString)
.Options;
return new RlhfGeneratorDbContext(options);
}
public void WriteCategoryValues(TrainingCategoryDefinition category, string content) =>
File.WriteAllText(
Path.Combine(DataDirectory, "Categories", $"{category.Name}.txt"),
content);
public async ValueTask DisposeAsync()
{
SqliteConnection.ClearAllPools();
await Task.Yield();
Directory.Delete(_rootDirectory, true);
}
private async Task CreateDatabaseAsync()
{
await using SqliteConnection connection = new(_connectionString);
await connection.OpenAsync();
await using SqliteCommand command = connection.CreateCommand();
command.CommandText =
"""
PRAGMA foreign_keys = ON;
CREATE TABLE Category (
Id INTEGER NOT NULL PRIMARY KEY,
Name TEXT NOT NULL
);
CREATE TABLE Keyword (
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
CategoryId INTEGER NOT NULL,
Value TEXT NOT NULL,
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
);
CREATE UNIQUE INDEX IX_Keyword_CategoryId_Value ON Keyword (CategoryId, Value);
CREATE TABLE KeywordVariant (
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
ParentId INTEGER NOT NULL,
Value TEXT NOT NULL,
FOREIGN KEY (ParentId) REFERENCES Keyword (Id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX IX_KeywordVariant_ParentId_Value
ON KeywordVariant (ParentId, Value);
CREATE TABLE TrainingItem (
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
GenerationRunId TEXT NULL,
PlanOrdinal INTEGER NULL,
RecordKey TEXT NULL,
CategoryId INTEGER NOT NULL,
KeywordId INTEGER NULL,
IsPositive INTEGER NOT NULL,
Value TEXT NOT NULL,
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT,
FOREIGN KEY (KeywordId) REFERENCES Keyword (Id) ON DELETE RESTRICT
);
CREATE INDEX IX_TrainingItem_KeywordId ON TrainingItem (KeywordId);
""";
await command.ExecuteNonQueryAsync();
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
command.Parameters.Clear();
command.CommandText = "INSERT INTO Category (Id, Name) VALUES ($id, $name);";
command.Parameters.AddWithValue("$id", category.Id);
command.Parameters.AddWithValue("$name", category.Name);
await command.ExecuteNonQueryAsync();
}
}
}