Models-Phi-8B/PHI.Data.Generator/RLHF.Generator/Keywords/KeywordLoadService.cs

115 lines
4.0 KiB
C#

using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using RLHF.Generator.Persistence;
using RLHF.Generator.TrainingData;
namespace RLHF.Generator.Keywords;
public sealed record KeywordLoadResult(
int SourceKeywordCount,
int SourceVariantCount,
int ExistingKeywordCount,
int ExistingVariantCount,
int InsertedKeywordCount,
int InsertedVariantCount);
public sealed class KeywordLoadService
{
private readonly RlhfGeneratorDbContext _dbContext;
private readonly TrainingDataReader _trainingDataReader;
public KeywordLoadService(
RlhfGeneratorDbContext dbContext,
TrainingDataReader trainingDataReader)
{
_dbContext = dbContext;
_trainingDataReader = trainingDataReader;
}
public async Task<KeywordLoadResult> LoadAsync(
CancellationToken cancellationToken = default)
{
List<SourceKeyword> sourceKeywords = [];
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
IReadOnlyList<TrainingCategoryValue> values =
await _trainingDataReader.ReadCategoryValuesAsync(category, cancellationToken);
sourceKeywords.AddRange(values.Select(value => new SourceKeyword(
category.Id,
value.CanonicalForm,
value.AllowedForms
.Skip(1)
.Where(variant => !string.IsNullOrWhiteSpace(variant)
&& !string.Equals(
variant,
value.CanonicalForm,
StringComparison.OrdinalIgnoreCase))
.ToArray())));
}
List<Keyword> databaseKeywords = await _dbContext.Keywords
.Include(keyword => keyword.Variants)
.ToListAsync(cancellationToken);
int existingKeywordCount = 0;
int existingVariantCount = 0;
int insertedKeywordCount = 0;
int insertedVariantCount = 0;
await using IDbContextTransaction transaction =
await _dbContext.Database.BeginTransactionAsync(cancellationToken);
foreach (SourceKeyword sourceKeyword in sourceKeywords)
{
Keyword? keyword = databaseKeywords.FirstOrDefault(candidate =>
candidate.CategoryId == sourceKeyword.CategoryId
&& string.Equals(
candidate.Value,
sourceKeyword.Value,
StringComparison.OrdinalIgnoreCase));
if (keyword is null)
{
keyword = new Keyword
{
CategoryId = sourceKeyword.CategoryId,
Value = sourceKeyword.Value
};
databaseKeywords.Add(keyword);
_dbContext.Keywords.Add(keyword);
insertedKeywordCount++;
}
else
{
existingKeywordCount++;
}
foreach (string sourceVariant in sourceKeyword.Variants)
{
if (keyword.Variants.Any(variant => string.Equals(
variant.Value,
sourceVariant,
StringComparison.OrdinalIgnoreCase)))
{
existingVariantCount++;
continue;
}
keyword.Variants.Add(new KeywordVariant { Value = sourceVariant });
insertedVariantCount++;
}
}
await _dbContext.SaveChangesAsync(cancellationToken);
await transaction.CommitAsync(cancellationToken);
return new KeywordLoadResult(
sourceKeywords.Count,
sourceKeywords.Sum(keyword => keyword.Variants.Count),
existingKeywordCount,
existingVariantCount,
insertedKeywordCount,
insertedVariantCount);
}
private sealed record SourceKeyword(
int CategoryId,
string Value,
IReadOnlyList<string> Variants);
}