235 lines
7.9 KiB
C#
235 lines
7.9 KiB
C#
using System.Globalization;
|
|
using System.Text;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.EntityFrameworkCore.Storage;
|
|
using RLHF.Generator.Persistence;
|
|
|
|
namespace RLHF.Generator.Keywords;
|
|
|
|
public sealed record TrainingItemKeywordBackfillResult(
|
|
bool DryRun,
|
|
int TotalTrainingItemCount,
|
|
int AlreadyAssignedCount,
|
|
int ProcessedCount,
|
|
int UniqueMatchCount,
|
|
int UpdatedCount,
|
|
int UnmatchedCount,
|
|
int AmbiguousCount);
|
|
|
|
public sealed class KeywordsNotLoadedException : InvalidOperationException
|
|
{
|
|
public KeywordsNotLoadedException()
|
|
: base("No keywords are loaded. Run POST /api/keywords/load before backfilling.")
|
|
{
|
|
}
|
|
}
|
|
|
|
public sealed class TrainingItemKeywordBackfillService
|
|
{
|
|
public const int DefaultBatchSize = 500;
|
|
public const int MaximumBatchSize = 5000;
|
|
|
|
private readonly RlhfGeneratorDbContext _dbContext;
|
|
|
|
public TrainingItemKeywordBackfillService(RlhfGeneratorDbContext dbContext)
|
|
{
|
|
_dbContext = dbContext;
|
|
}
|
|
|
|
public async Task<TrainingItemKeywordBackfillResult> BackfillAsync(
|
|
bool dryRun = true,
|
|
int batchSize = DefaultBatchSize,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (batchSize is < 1 or > MaximumBatchSize)
|
|
{
|
|
throw new ArgumentOutOfRangeException(
|
|
nameof(batchSize),
|
|
$"Batch size must be between 1 and {MaximumBatchSize}.");
|
|
}
|
|
|
|
List<Keyword> keywords = await _dbContext.Keywords
|
|
.AsNoTracking()
|
|
.Include(keyword => keyword.Variants)
|
|
.ToListAsync(cancellationToken);
|
|
if (keywords.Count == 0)
|
|
{
|
|
throw new KeywordsNotLoadedException();
|
|
}
|
|
|
|
IReadOnlyDictionary<int, IReadOnlyList<Surface>> catalog = BuildCatalog(keywords);
|
|
int totalCount = await _dbContext.TrainingItems.CountAsync(cancellationToken);
|
|
int alreadyAssignedCount = await _dbContext.TrainingItems.CountAsync(
|
|
item => item.KeywordId != null,
|
|
cancellationToken);
|
|
int processedCount = 0;
|
|
int uniqueMatchCount = 0;
|
|
int updatedCount = 0;
|
|
int unmatchedCount = 0;
|
|
int ambiguousCount = 0;
|
|
int lastId = 0;
|
|
|
|
while (true)
|
|
{
|
|
IQueryable<TrainingItem> query = _dbContext.TrainingItems
|
|
.Where(item => item.KeywordId == null && item.Id > lastId)
|
|
.OrderBy(item => item.Id)
|
|
.Take(batchSize);
|
|
List<TrainingItem> batch = dryRun
|
|
? await query.AsNoTracking().ToListAsync(cancellationToken)
|
|
: await query.ToListAsync(cancellationToken);
|
|
if (batch.Count == 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
lastId = batch[^1].Id;
|
|
foreach (TrainingItem item in batch)
|
|
{
|
|
HashSet<int> matches = Match(catalog, item.CategoryId, item.Value);
|
|
processedCount++;
|
|
switch (matches.Count)
|
|
{
|
|
case 0:
|
|
unmatchedCount++;
|
|
break;
|
|
case 1:
|
|
uniqueMatchCount++;
|
|
if (!dryRun)
|
|
{
|
|
item.KeywordId = matches.Single();
|
|
updatedCount++;
|
|
}
|
|
break;
|
|
default:
|
|
ambiguousCount++;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!dryRun)
|
|
{
|
|
await using IDbContextTransaction transaction =
|
|
await _dbContext.Database.BeginTransactionAsync(cancellationToken);
|
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
|
await transaction.CommitAsync(cancellationToken);
|
|
_dbContext.ChangeTracker.Clear();
|
|
}
|
|
}
|
|
|
|
return new TrainingItemKeywordBackfillResult(
|
|
dryRun,
|
|
totalCount,
|
|
alreadyAssignedCount,
|
|
processedCount,
|
|
uniqueMatchCount,
|
|
updatedCount,
|
|
unmatchedCount,
|
|
ambiguousCount);
|
|
}
|
|
|
|
private static IReadOnlyDictionary<int, IReadOnlyList<Surface>> BuildCatalog(
|
|
IEnumerable<Keyword> keywords) => keywords
|
|
.GroupBy(keyword => keyword.CategoryId)
|
|
.ToDictionary(
|
|
category => category.Key,
|
|
category => (IReadOnlyList<Surface>)category
|
|
.SelectMany(keyword => keyword.Variants
|
|
.Select(variant => new Surface(Normalize(variant.Value), keyword.Id))
|
|
.Prepend(new Surface(Normalize(keyword.Value), keyword.Id)))
|
|
.Where(surface => surface.Value.Length > 0)
|
|
.OrderByDescending(surface => surface.Value.Length)
|
|
.ToArray());
|
|
|
|
private static HashSet<int> Match(
|
|
IReadOnlyDictionary<int, IReadOnlyList<Surface>> catalog,
|
|
int categoryId,
|
|
string value)
|
|
{
|
|
HashSet<int> keywordIds = [];
|
|
if (!catalog.TryGetValue(categoryId, out IReadOnlyList<Surface>? surfaces))
|
|
{
|
|
return keywordIds;
|
|
}
|
|
|
|
string normalizedValue = Normalize(value);
|
|
List<Span> occupiedSpans = [];
|
|
foreach (Surface surface in surfaces)
|
|
{
|
|
int searchIndex = 0;
|
|
while (searchIndex <= normalizedValue.Length - surface.Value.Length)
|
|
{
|
|
int matchIndex = normalizedValue.IndexOf(
|
|
surface.Value,
|
|
searchIndex,
|
|
StringComparison.OrdinalIgnoreCase);
|
|
if (matchIndex < 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
int matchEnd = matchIndex + surface.Value.Length;
|
|
bool hasBoundaries = !IsLetterOrDigitBefore(normalizedValue, matchIndex)
|
|
&& !IsLetterOrDigitAt(normalizedValue, matchEnd);
|
|
bool overlaps = occupiedSpans.Any(span =>
|
|
surface.Value.Length < span.Length
|
|
&& matchIndex < span.Start + span.Length
|
|
&& matchEnd > span.Start);
|
|
if (hasBoundaries && !overlaps)
|
|
{
|
|
occupiedSpans.Add(new Span(matchIndex, surface.Value.Length));
|
|
keywordIds.Add(surface.KeywordId);
|
|
}
|
|
|
|
searchIndex = matchIndex + Math.Max(1, surface.Value.Length);
|
|
}
|
|
}
|
|
|
|
return keywordIds;
|
|
}
|
|
|
|
private static string Normalize(string value)
|
|
{
|
|
string decomposed = value.Normalize(NormalizationForm.FormD);
|
|
StringBuilder normalized = new(decomposed.Length);
|
|
foreach (Rune rune in decomposed.EnumerateRunes())
|
|
{
|
|
UnicodeCategory category = Rune.GetUnicodeCategory(rune);
|
|
if (category is not UnicodeCategory.NonSpacingMark
|
|
and not UnicodeCategory.SpacingCombiningMark
|
|
and not UnicodeCategory.EnclosingMark)
|
|
{
|
|
normalized.Append(rune);
|
|
}
|
|
}
|
|
|
|
return normalized.ToString();
|
|
}
|
|
|
|
private static bool IsLetterOrDigitAt(string value, int index) =>
|
|
index >= 0
|
|
&& index < value.Length
|
|
&& Rune.IsLetterOrDigit(Rune.GetRuneAt(value, index));
|
|
|
|
private static bool IsLetterOrDigitBefore(string value, int index)
|
|
{
|
|
if (index <= 0 || index > value.Length)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
int runeStart = index - 1;
|
|
if (runeStart > 0
|
|
&& char.IsLowSurrogate(value[runeStart])
|
|
&& char.IsHighSurrogate(value[runeStart - 1]))
|
|
{
|
|
runeStart--;
|
|
}
|
|
|
|
return Rune.IsLetterOrDigit(Rune.GetRuneAt(value, runeStart));
|
|
}
|
|
|
|
private sealed record Surface(string Value, int KeywordId);
|
|
|
|
private sealed record Span(int Start, int Length);
|
|
} |