using System.Globalization; using System.Text; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; using RLHF.Generator.Persistence; namespace RLHF.Generator.Keywords; public sealed record TrainingItemKeywordBackfillResult( bool DryRun, int TotalTrainingItemCount, int AlreadyAssignedCount, int ProcessedCount, int UniqueMatchCount, int UpdatedCount, int UnmatchedCount, int AmbiguousCount); public sealed class KeywordsNotLoadedException : InvalidOperationException { public KeywordsNotLoadedException() : base("No keywords are loaded. Run POST /api/keywords/load before backfilling.") { } } public sealed class TrainingItemKeywordBackfillService { public const int DefaultBatchSize = 500; public const int MaximumBatchSize = 5000; private readonly RlhfGeneratorDbContext _dbContext; public TrainingItemKeywordBackfillService(RlhfGeneratorDbContext dbContext) { _dbContext = dbContext; } public async Task BackfillAsync( bool dryRun = true, int batchSize = DefaultBatchSize, CancellationToken cancellationToken = default) { if (batchSize is < 1 or > MaximumBatchSize) { throw new ArgumentOutOfRangeException( nameof(batchSize), $"Batch size must be between 1 and {MaximumBatchSize}."); } List keywords = await _dbContext.Keywords .AsNoTracking() .Include(keyword => keyword.Variants) .ToListAsync(cancellationToken); if (keywords.Count == 0) { throw new KeywordsNotLoadedException(); } IReadOnlyDictionary> catalog = BuildCatalog(keywords); int totalCount = await _dbContext.TrainingItems.CountAsync(cancellationToken); int alreadyAssignedCount = await _dbContext.TrainingItems.CountAsync( item => item.KeywordId != null, cancellationToken); int processedCount = 0; int uniqueMatchCount = 0; int updatedCount = 0; int unmatchedCount = 0; int ambiguousCount = 0; int lastId = 0; while (true) { IQueryable query = _dbContext.TrainingItems .Where(item => item.KeywordId == null && item.Id > lastId) .OrderBy(item => item.Id) .Take(batchSize); List batch = dryRun ? await query.AsNoTracking().ToListAsync(cancellationToken) : await query.ToListAsync(cancellationToken); if (batch.Count == 0) { break; } lastId = batch[^1].Id; foreach (TrainingItem item in batch) { HashSet matches = Match(catalog, item.CategoryId, item.Value); processedCount++; switch (matches.Count) { case 0: unmatchedCount++; break; case 1: uniqueMatchCount++; if (!dryRun) { item.KeywordId = matches.Single(); updatedCount++; } break; default: ambiguousCount++; break; } } if (!dryRun) { await using IDbContextTransaction transaction = await _dbContext.Database.BeginTransactionAsync(cancellationToken); await _dbContext.SaveChangesAsync(cancellationToken); await transaction.CommitAsync(cancellationToken); _dbContext.ChangeTracker.Clear(); } } return new TrainingItemKeywordBackfillResult( dryRun, totalCount, alreadyAssignedCount, processedCount, uniqueMatchCount, updatedCount, unmatchedCount, ambiguousCount); } private static IReadOnlyDictionary> BuildCatalog( IEnumerable keywords) => keywords .GroupBy(keyword => keyword.CategoryId) .ToDictionary( category => category.Key, category => (IReadOnlyList)category .SelectMany(keyword => keyword.Variants .Select(variant => new Surface(Normalize(variant.Value), keyword.Id)) .Prepend(new Surface(Normalize(keyword.Value), keyword.Id))) .Where(surface => surface.Value.Length > 0) .OrderByDescending(surface => surface.Value.Length) .ToArray()); private static HashSet Match( IReadOnlyDictionary> catalog, int categoryId, string value) { HashSet keywordIds = []; if (!catalog.TryGetValue(categoryId, out IReadOnlyList? surfaces)) { return keywordIds; } string normalizedValue = Normalize(value); List occupiedSpans = []; foreach (Surface surface in surfaces) { int searchIndex = 0; while (searchIndex <= normalizedValue.Length - surface.Value.Length) { int matchIndex = normalizedValue.IndexOf( surface.Value, searchIndex, StringComparison.OrdinalIgnoreCase); if (matchIndex < 0) { break; } int matchEnd = matchIndex + surface.Value.Length; bool hasBoundaries = !IsLetterOrDigitBefore(normalizedValue, matchIndex) && !IsLetterOrDigitAt(normalizedValue, matchEnd); bool overlaps = occupiedSpans.Any(span => surface.Value.Length < span.Length && matchIndex < span.Start + span.Length && matchEnd > span.Start); if (hasBoundaries && !overlaps) { occupiedSpans.Add(new Span(matchIndex, surface.Value.Length)); keywordIds.Add(surface.KeywordId); } searchIndex = matchIndex + Math.Max(1, surface.Value.Length); } } return keywordIds; } private static string Normalize(string value) { string decomposed = value.Normalize(NormalizationForm.FormD); StringBuilder normalized = new(decomposed.Length); foreach (Rune rune in decomposed.EnumerateRunes()) { UnicodeCategory category = Rune.GetUnicodeCategory(rune); if (category is not UnicodeCategory.NonSpacingMark and not UnicodeCategory.SpacingCombiningMark and not UnicodeCategory.EnclosingMark) { normalized.Append(rune); } } return normalized.ToString(); } private static bool IsLetterOrDigitAt(string value, int index) => index >= 0 && index < value.Length && Rune.IsLetterOrDigit(Rune.GetRuneAt(value, index)); private static bool IsLetterOrDigitBefore(string value, int index) { if (index <= 0 || index > value.Length) { return false; } int runeStart = index - 1; if (runeStart > 0 && char.IsLowSurrogate(value[runeStart]) && char.IsHighSurrogate(value[runeStart - 1])) { runeStart--; } return Rune.IsLetterOrDigit(Rune.GetRuneAt(value, runeStart)); } private sealed record Surface(string Value, int KeywordId); private sealed record Span(int Start, int Length); }