324 lines
14 KiB
C#
324 lines
14 KiB
C#
using Microsoft.Data.SqlClient;
|
|
using Microsoft.Data.Sqlite;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using RLHF.Generator.Keywords;
|
|
using RLHF.Generator.Persistence;
|
|
using RLHF.Generator.TrainingData;
|
|
using Xunit;
|
|
|
|
namespace RLHF.Generator.Tests;
|
|
|
|
public sealed class KeywordCatalogPhase1Tests
|
|
{
|
|
[Fact]
|
|
public async Task Model_ContainsNullableKeywordRelationshipAndRequiredIndexes()
|
|
{
|
|
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
|
|
await using RlhfGeneratorDbContext context = fixture.CreateContext();
|
|
Microsoft.EntityFrameworkCore.Metadata.IEntityType keyword =
|
|
context.Model.FindEntityType(typeof(Keyword))!;
|
|
Microsoft.EntityFrameworkCore.Metadata.IEntityType variant =
|
|
context.Model.FindEntityType(typeof(KeywordVariant))!;
|
|
Microsoft.EntityFrameworkCore.Metadata.IEntityType trainingItem =
|
|
context.Model.FindEntityType(typeof(TrainingItem))!;
|
|
|
|
Assert.Contains(
|
|
keyword.GetIndexes(),
|
|
index => index.IsUnique
|
|
&& index.Properties.Select(property => property.Name)
|
|
.SequenceEqual([nameof(Keyword.CategoryId), nameof(Keyword.Value)]));
|
|
Assert.Contains(
|
|
variant.GetIndexes(),
|
|
index => index.IsUnique
|
|
&& index.Properties.Select(property => property.Name)
|
|
.SequenceEqual([nameof(KeywordVariant.ParentId), nameof(KeywordVariant.Value)]));
|
|
Assert.True(trainingItem.FindProperty(nameof(TrainingItem.KeywordId))!.IsNullable);
|
|
Assert.Contains(
|
|
trainingItem.GetIndexes(),
|
|
index => !index.IsUnique
|
|
&& index.Properties.Single().Name == nameof(TrainingItem.KeywordId));
|
|
Assert.Contains(
|
|
keyword.GetForeignKeys(),
|
|
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Category)
|
|
&& foreignKey.PrincipalToDependent?.Name == nameof(Category.Keywords)
|
|
&& foreignKey.DeleteBehavior == DeleteBehavior.Restrict);
|
|
Assert.Contains(
|
|
variant.GetForeignKeys(),
|
|
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Keyword)
|
|
&& foreignKey.DeleteBehavior == DeleteBehavior.Cascade);
|
|
Assert.Contains(
|
|
trainingItem.GetForeignKeys(),
|
|
foreignKey => foreignKey.PrincipalEntityType.ClrType == typeof(Keyword)
|
|
&& foreignKey.DeleteBehavior == DeleteBehavior.Restrict);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Load_IsAdditiveCaseInsensitiveAndIdempotent()
|
|
{
|
|
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
|
|
fixture.WriteCategoryValues(
|
|
TrainingCategories.MedicalDiagnosis,
|
|
"Café-au-lait spot|cafe au lait spot");
|
|
await using RlhfGeneratorDbContext context = fixture.CreateContext();
|
|
context.Keywords.Add(new Keyword
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "CAFÉ-AU-LAIT SPOT"
|
|
});
|
|
await context.SaveChangesAsync();
|
|
|
|
KeywordLoadService service = new(context, new TrainingDataReader(fixture.DataDirectory));
|
|
KeywordLoadResult first = await service.LoadAsync();
|
|
KeywordLoadResult second = await service.LoadAsync();
|
|
|
|
Assert.Equal(TrainingCategories.All.Count, first.SourceKeywordCount);
|
|
Assert.Equal(1, first.SourceVariantCount);
|
|
Assert.Equal(TrainingCategories.All.Count - 1, first.InsertedKeywordCount);
|
|
Assert.Equal(1, first.ExistingKeywordCount);
|
|
Assert.Equal(1, first.InsertedVariantCount);
|
|
Assert.Equal(0, second.InsertedKeywordCount);
|
|
Assert.Equal(0, second.InsertedVariantCount);
|
|
Assert.Equal(TrainingCategories.All.Count, await context.Keywords.CountAsync());
|
|
Keyword reused = await context.Keywords.SingleAsync(keyword =>
|
|
keyword.CategoryId == TrainingCategories.MedicalDiagnosis.Id);
|
|
KeywordVariant variant = Assert.Single(await context.KeywordVariants.ToListAsync());
|
|
Assert.Equal(reused.Id, variant.ParentId);
|
|
Assert.False(string.Equals(
|
|
reused.Value,
|
|
variant.Value,
|
|
StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Backfill_DryRunApplyAndRerun_HandleNormalizationBoundariesAndAmbiguity()
|
|
{
|
|
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
|
|
await using RlhfGeneratorDbContext context = fixture.CreateContext();
|
|
Keyword cafe = new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "café-au-lait spot",
|
|
Variants = [new KeywordVariant { Value = "cafe au lait spot" }]
|
|
};
|
|
Keyword pain = new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "pain"
|
|
};
|
|
Keyword chestPain = new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "chest pain"
|
|
};
|
|
Keyword sharedOne = new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "shared surface"
|
|
};
|
|
Keyword sharedTwo = new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
Value = "different canonical",
|
|
Variants = [new KeywordVariant { Value = "shared surface" }]
|
|
};
|
|
context.Keywords.AddRange(cafe, pain, chestPain, sharedOne, sharedTwo);
|
|
await context.SaveChangesAsync();
|
|
Guid generationRunId = Guid.NewGuid();
|
|
TrainingItem alreadyAssigned = new()
|
|
{
|
|
GenerationRunId = generationRunId,
|
|
PlanOrdinal = 42,
|
|
RecordKey = new string('a', 64),
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
KeywordId = pain.Id,
|
|
IsPositive = false,
|
|
Value = "Existing assigned metadata"
|
|
};
|
|
context.TrainingItems.AddRange(
|
|
CreateItem("The CAFE AU LAIT SPOT is unchanged."),
|
|
CreateItem("There is chest pain."),
|
|
CreateItem("There is chest pain and a café-au-lait spot."),
|
|
CreateItem("A shared surface is present."),
|
|
CreateItem("The painting is unchanged."),
|
|
new TrainingItem
|
|
{
|
|
CategoryId = TrainingCategories.MedicalAllergy.Id,
|
|
IsPositive = true,
|
|
Value = "Pain is mentioned in another category."
|
|
},
|
|
alreadyAssigned);
|
|
await context.SaveChangesAsync();
|
|
|
|
TrainingItemKeywordBackfillService service = new(context);
|
|
TrainingItemKeywordBackfillResult dryRun = await service.BackfillAsync(batchSize: 5);
|
|
Assert.Equal(7, dryRun.TotalTrainingItemCount);
|
|
Assert.Equal(1, dryRun.AlreadyAssignedCount);
|
|
Assert.Equal(6, dryRun.ProcessedCount);
|
|
Assert.Equal(2, dryRun.UniqueMatchCount);
|
|
Assert.Equal(2, dryRun.UnmatchedCount);
|
|
Assert.Equal(2, dryRun.AmbiguousCount);
|
|
Assert.Equal(0, dryRun.UpdatedCount);
|
|
Assert.Equal(1, await context.TrainingItems.CountAsync(item => item.KeywordId != null));
|
|
|
|
TrainingItemKeywordBackfillResult applied = await service.BackfillAsync(false, 5);
|
|
Assert.Equal(2, applied.UpdatedCount);
|
|
Assert.Equal(3, await context.TrainingItems.CountAsync(item => item.KeywordId != null));
|
|
TrainingItem preserved = await context.TrainingItems.SingleAsync(
|
|
item => item.Id == alreadyAssigned.Id);
|
|
Assert.Equal(pain.Id, preserved.KeywordId);
|
|
Assert.Equal(generationRunId, preserved.GenerationRunId);
|
|
Assert.Equal(42, preserved.PlanOrdinal);
|
|
Assert.Equal(new string('a', 64), preserved.RecordKey);
|
|
Assert.False(preserved.IsPositive);
|
|
Assert.Equal("Existing assigned metadata", preserved.Value);
|
|
|
|
TrainingItemKeywordBackfillResult rerun = await service.BackfillAsync(false, 5);
|
|
Assert.Equal(3, rerun.AlreadyAssignedCount);
|
|
Assert.Equal(4, rerun.ProcessedCount);
|
|
Assert.Equal(0, rerun.UpdatedCount);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Backfill_RejectsMissingCatalogAndInvalidBatchSize()
|
|
{
|
|
await using KeywordFixture fixture = await KeywordFixture.CreateAsync();
|
|
await using RlhfGeneratorDbContext context = fixture.CreateContext();
|
|
TrainingItemKeywordBackfillService service = new(context);
|
|
|
|
await Assert.ThrowsAsync<KeywordsNotLoadedException>(() => service.BackfillAsync());
|
|
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(() => service.BackfillAsync(batchSize: 0));
|
|
}
|
|
|
|
[Fact]
|
|
public void DesignTimeFactory_UsesOnlyGuardedSqlServerCatalog()
|
|
{
|
|
RlhfGeneratorDesignTimeDbContextFactory factory = new();
|
|
using RlhfGeneratorDbContext context = factory.CreateDbContext([]);
|
|
string connectionString = context.Database.GetConnectionString()!;
|
|
SqlConnectionStringBuilder connection = new(connectionString);
|
|
|
|
Assert.True(context.Database.IsSqlServer());
|
|
Assert.Equal(
|
|
RlhfGeneratorDesignTimeDbContextFactory.DesignTimeDatabaseName,
|
|
connection.InitialCatalog);
|
|
InvalidOperationException productionException = Assert.Throws<InvalidOperationException>(
|
|
() => RlhfGeneratorDesignTimeDbContextFactory.ValidateDatabaseName("RLHFGenerator"));
|
|
Assert.Contains(
|
|
"forbidden from using the production RLHFGenerator database",
|
|
productionException.Message,
|
|
StringComparison.Ordinal);
|
|
Assert.Throws<InvalidOperationException>(
|
|
() => RlhfGeneratorDesignTimeDbContextFactory.ValidateDatabaseName("anything-else"));
|
|
}
|
|
|
|
private static TrainingItem CreateItem(string value) => new()
|
|
{
|
|
CategoryId = TrainingCategories.MedicalDiagnosis.Id,
|
|
IsPositive = true,
|
|
Value = value
|
|
};
|
|
}
|
|
|
|
internal sealed class KeywordFixture : IAsyncDisposable
|
|
{
|
|
private readonly string _rootDirectory;
|
|
private readonly string _connectionString;
|
|
|
|
private KeywordFixture(string rootDirectory)
|
|
{
|
|
_rootDirectory = rootDirectory;
|
|
_connectionString = $"Data Source={Path.Combine(rootDirectory, "keywords.db")};Pooling=False";
|
|
}
|
|
|
|
public string DataDirectory => Path.Combine(_rootDirectory, "Data");
|
|
|
|
public static async Task<KeywordFixture> CreateAsync()
|
|
{
|
|
string rootDirectory = Path.Combine(
|
|
Path.GetTempPath(),
|
|
$"RLHF.Generator.Keywords.Tests-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(Path.Combine(rootDirectory, "Data", "Categories"));
|
|
KeywordFixture fixture = new(rootDirectory);
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
fixture.WriteCategoryValues(category, $"value-{category.Id}");
|
|
}
|
|
|
|
await fixture.CreateDatabaseAsync();
|
|
return fixture;
|
|
}
|
|
|
|
public RlhfGeneratorDbContext CreateContext()
|
|
{
|
|
DbContextOptions<RlhfGeneratorDbContext> options =
|
|
new DbContextOptionsBuilder<RlhfGeneratorDbContext>()
|
|
.UseSqlite(_connectionString)
|
|
.Options;
|
|
return new RlhfGeneratorDbContext(options);
|
|
}
|
|
|
|
public void WriteCategoryValues(TrainingCategoryDefinition category, string content) =>
|
|
File.WriteAllText(
|
|
Path.Combine(DataDirectory, "Categories", $"{category.Name}.txt"),
|
|
content);
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
SqliteConnection.ClearAllPools();
|
|
await Task.Yield();
|
|
Directory.Delete(_rootDirectory, true);
|
|
}
|
|
|
|
private async Task CreateDatabaseAsync()
|
|
{
|
|
await using SqliteConnection connection = new(_connectionString);
|
|
await connection.OpenAsync();
|
|
await using SqliteCommand command = connection.CreateCommand();
|
|
command.CommandText =
|
|
"""
|
|
PRAGMA foreign_keys = ON;
|
|
CREATE TABLE Category (
|
|
Id INTEGER NOT NULL PRIMARY KEY,
|
|
Name TEXT NOT NULL
|
|
);
|
|
CREATE TABLE Keyword (
|
|
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
|
CategoryId INTEGER NOT NULL,
|
|
Value TEXT NOT NULL,
|
|
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT
|
|
);
|
|
CREATE UNIQUE INDEX IX_Keyword_CategoryId_Value ON Keyword (CategoryId, Value);
|
|
CREATE TABLE KeywordVariant (
|
|
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
|
ParentId INTEGER NOT NULL,
|
|
Value TEXT NOT NULL,
|
|
FOREIGN KEY (ParentId) REFERENCES Keyword (Id) ON DELETE CASCADE
|
|
);
|
|
CREATE UNIQUE INDEX IX_KeywordVariant_ParentId_Value
|
|
ON KeywordVariant (ParentId, Value);
|
|
CREATE TABLE TrainingItem (
|
|
Id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
|
GenerationRunId TEXT NULL,
|
|
PlanOrdinal INTEGER NULL,
|
|
RecordKey TEXT NULL,
|
|
CategoryId INTEGER NOT NULL,
|
|
KeywordId INTEGER NULL,
|
|
IsPositive INTEGER NOT NULL,
|
|
Value TEXT NOT NULL,
|
|
FOREIGN KEY (CategoryId) REFERENCES Category (Id) ON DELETE RESTRICT,
|
|
FOREIGN KEY (KeywordId) REFERENCES Keyword (Id) ON DELETE RESTRICT
|
|
);
|
|
CREATE INDEX IX_TrainingItem_KeywordId ON TrainingItem (KeywordId);
|
|
""";
|
|
await command.ExecuteNonQueryAsync();
|
|
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
command.Parameters.Clear();
|
|
command.CommandText = "INSERT INTO Category (Id, Name) VALUES ($id, $name);";
|
|
command.Parameters.AddWithValue("$id", category.Id);
|
|
command.Parameters.AddWithValue("$name", category.Name);
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
} |