115 lines
4.0 KiB
C#
115 lines
4.0 KiB
C#
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.EntityFrameworkCore.Storage;
|
|
using RLHF.Generator.Persistence;
|
|
using RLHF.Generator.TrainingData;
|
|
|
|
namespace RLHF.Generator.Keywords;
|
|
|
|
public sealed record KeywordLoadResult(
|
|
int SourceKeywordCount,
|
|
int SourceVariantCount,
|
|
int ExistingKeywordCount,
|
|
int ExistingVariantCount,
|
|
int InsertedKeywordCount,
|
|
int InsertedVariantCount);
|
|
|
|
public sealed class KeywordLoadService
|
|
{
|
|
private readonly RlhfGeneratorDbContext _dbContext;
|
|
private readonly TrainingDataReader _trainingDataReader;
|
|
|
|
public KeywordLoadService(
|
|
RlhfGeneratorDbContext dbContext,
|
|
TrainingDataReader trainingDataReader)
|
|
{
|
|
_dbContext = dbContext;
|
|
_trainingDataReader = trainingDataReader;
|
|
}
|
|
|
|
public async Task<KeywordLoadResult> LoadAsync(
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
List<SourceKeyword> sourceKeywords = [];
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
IReadOnlyList<TrainingCategoryValue> values =
|
|
await _trainingDataReader.ReadCategoryValuesAsync(category, cancellationToken);
|
|
sourceKeywords.AddRange(values.Select(value => new SourceKeyword(
|
|
category.Id,
|
|
value.CanonicalForm,
|
|
value.AllowedForms
|
|
.Skip(1)
|
|
.Where(variant => !string.IsNullOrWhiteSpace(variant)
|
|
&& !string.Equals(
|
|
variant,
|
|
value.CanonicalForm,
|
|
StringComparison.OrdinalIgnoreCase))
|
|
.ToArray())));
|
|
}
|
|
|
|
List<Keyword> databaseKeywords = await _dbContext.Keywords
|
|
.Include(keyword => keyword.Variants)
|
|
.ToListAsync(cancellationToken);
|
|
int existingKeywordCount = 0;
|
|
int existingVariantCount = 0;
|
|
int insertedKeywordCount = 0;
|
|
int insertedVariantCount = 0;
|
|
|
|
await using IDbContextTransaction transaction =
|
|
await _dbContext.Database.BeginTransactionAsync(cancellationToken);
|
|
foreach (SourceKeyword sourceKeyword in sourceKeywords)
|
|
{
|
|
Keyword? keyword = databaseKeywords.FirstOrDefault(candidate =>
|
|
candidate.CategoryId == sourceKeyword.CategoryId
|
|
&& string.Equals(
|
|
candidate.Value,
|
|
sourceKeyword.Value,
|
|
StringComparison.OrdinalIgnoreCase));
|
|
if (keyword is null)
|
|
{
|
|
keyword = new Keyword
|
|
{
|
|
CategoryId = sourceKeyword.CategoryId,
|
|
Value = sourceKeyword.Value
|
|
};
|
|
databaseKeywords.Add(keyword);
|
|
_dbContext.Keywords.Add(keyword);
|
|
insertedKeywordCount++;
|
|
}
|
|
else
|
|
{
|
|
existingKeywordCount++;
|
|
}
|
|
|
|
foreach (string sourceVariant in sourceKeyword.Variants)
|
|
{
|
|
if (keyword.Variants.Any(variant => string.Equals(
|
|
variant.Value,
|
|
sourceVariant,
|
|
StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
existingVariantCount++;
|
|
continue;
|
|
}
|
|
|
|
keyword.Variants.Add(new KeywordVariant { Value = sourceVariant });
|
|
insertedVariantCount++;
|
|
}
|
|
}
|
|
|
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
|
await transaction.CommitAsync(cancellationToken);
|
|
return new KeywordLoadResult(
|
|
sourceKeywords.Count,
|
|
sourceKeywords.Sum(keyword => keyword.Variants.Count),
|
|
existingKeywordCount,
|
|
existingVariantCount,
|
|
insertedKeywordCount,
|
|
insertedVariantCount);
|
|
}
|
|
|
|
private sealed record SourceKeyword(
|
|
int CategoryId,
|
|
string Value,
|
|
IReadOnlyList<string> Variants);
|
|
} |