Models-Phi-8B/PHI.Data.Generator/RLHF.Generator/Keywords/TrainingItemKeywordBackfill...

235 lines
7.9 KiB
C#

using System.Globalization;
using System.Text;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using RLHF.Generator.Persistence;
namespace RLHF.Generator.Keywords;
public sealed record TrainingItemKeywordBackfillResult(
bool DryRun,
int TotalTrainingItemCount,
int AlreadyAssignedCount,
int ProcessedCount,
int UniqueMatchCount,
int UpdatedCount,
int UnmatchedCount,
int AmbiguousCount);
public sealed class KeywordsNotLoadedException : InvalidOperationException
{
public KeywordsNotLoadedException()
: base("No keywords are loaded. Run POST /api/keywords/load before backfilling.")
{
}
}
public sealed class TrainingItemKeywordBackfillService
{
public const int DefaultBatchSize = 500;
public const int MaximumBatchSize = 5000;
private readonly RlhfGeneratorDbContext _dbContext;
public TrainingItemKeywordBackfillService(RlhfGeneratorDbContext dbContext)
{
_dbContext = dbContext;
}
public async Task<TrainingItemKeywordBackfillResult> BackfillAsync(
bool dryRun = true,
int batchSize = DefaultBatchSize,
CancellationToken cancellationToken = default)
{
if (batchSize is < 1 or > MaximumBatchSize)
{
throw new ArgumentOutOfRangeException(
nameof(batchSize),
$"Batch size must be between 1 and {MaximumBatchSize}.");
}
List<Keyword> keywords = await _dbContext.Keywords
.AsNoTracking()
.Include(keyword => keyword.Variants)
.ToListAsync(cancellationToken);
if (keywords.Count == 0)
{
throw new KeywordsNotLoadedException();
}
IReadOnlyDictionary<int, IReadOnlyList<Surface>> catalog = BuildCatalog(keywords);
int totalCount = await _dbContext.TrainingItems.CountAsync(cancellationToken);
int alreadyAssignedCount = await _dbContext.TrainingItems.CountAsync(
item => item.KeywordId != null,
cancellationToken);
int processedCount = 0;
int uniqueMatchCount = 0;
int updatedCount = 0;
int unmatchedCount = 0;
int ambiguousCount = 0;
int lastId = 0;
while (true)
{
IQueryable<TrainingItem> query = _dbContext.TrainingItems
.Where(item => item.KeywordId == null && item.Id > lastId)
.OrderBy(item => item.Id)
.Take(batchSize);
List<TrainingItem> batch = dryRun
? await query.AsNoTracking().ToListAsync(cancellationToken)
: await query.ToListAsync(cancellationToken);
if (batch.Count == 0)
{
break;
}
lastId = batch[^1].Id;
foreach (TrainingItem item in batch)
{
HashSet<int> matches = Match(catalog, item.CategoryId, item.Value);
processedCount++;
switch (matches.Count)
{
case 0:
unmatchedCount++;
break;
case 1:
uniqueMatchCount++;
if (!dryRun)
{
item.KeywordId = matches.Single();
updatedCount++;
}
break;
default:
ambiguousCount++;
break;
}
}
if (!dryRun)
{
await using IDbContextTransaction transaction =
await _dbContext.Database.BeginTransactionAsync(cancellationToken);
await _dbContext.SaveChangesAsync(cancellationToken);
await transaction.CommitAsync(cancellationToken);
_dbContext.ChangeTracker.Clear();
}
}
return new TrainingItemKeywordBackfillResult(
dryRun,
totalCount,
alreadyAssignedCount,
processedCount,
uniqueMatchCount,
updatedCount,
unmatchedCount,
ambiguousCount);
}
private static IReadOnlyDictionary<int, IReadOnlyList<Surface>> BuildCatalog(
IEnumerable<Keyword> keywords) => keywords
.GroupBy(keyword => keyword.CategoryId)
.ToDictionary(
category => category.Key,
category => (IReadOnlyList<Surface>)category
.SelectMany(keyword => keyword.Variants
.Select(variant => new Surface(Normalize(variant.Value), keyword.Id))
.Prepend(new Surface(Normalize(keyword.Value), keyword.Id)))
.Where(surface => surface.Value.Length > 0)
.OrderByDescending(surface => surface.Value.Length)
.ToArray());
private static HashSet<int> Match(
IReadOnlyDictionary<int, IReadOnlyList<Surface>> catalog,
int categoryId,
string value)
{
HashSet<int> keywordIds = [];
if (!catalog.TryGetValue(categoryId, out IReadOnlyList<Surface>? surfaces))
{
return keywordIds;
}
string normalizedValue = Normalize(value);
List<Span> occupiedSpans = [];
foreach (Surface surface in surfaces)
{
int searchIndex = 0;
while (searchIndex <= normalizedValue.Length - surface.Value.Length)
{
int matchIndex = normalizedValue.IndexOf(
surface.Value,
searchIndex,
StringComparison.OrdinalIgnoreCase);
if (matchIndex < 0)
{
break;
}
int matchEnd = matchIndex + surface.Value.Length;
bool hasBoundaries = !IsLetterOrDigitBefore(normalizedValue, matchIndex)
&& !IsLetterOrDigitAt(normalizedValue, matchEnd);
bool overlaps = occupiedSpans.Any(span =>
surface.Value.Length < span.Length
&& matchIndex < span.Start + span.Length
&& matchEnd > span.Start);
if (hasBoundaries && !overlaps)
{
occupiedSpans.Add(new Span(matchIndex, surface.Value.Length));
keywordIds.Add(surface.KeywordId);
}
searchIndex = matchIndex + Math.Max(1, surface.Value.Length);
}
}
return keywordIds;
}
private static string Normalize(string value)
{
string decomposed = value.Normalize(NormalizationForm.FormD);
StringBuilder normalized = new(decomposed.Length);
foreach (Rune rune in decomposed.EnumerateRunes())
{
UnicodeCategory category = Rune.GetUnicodeCategory(rune);
if (category is not UnicodeCategory.NonSpacingMark
and not UnicodeCategory.SpacingCombiningMark
and not UnicodeCategory.EnclosingMark)
{
normalized.Append(rune);
}
}
return normalized.ToString();
}
private static bool IsLetterOrDigitAt(string value, int index) =>
index >= 0
&& index < value.Length
&& Rune.IsLetterOrDigit(Rune.GetRuneAt(value, index));
private static bool IsLetterOrDigitBefore(string value, int index)
{
if (index <= 0 || index > value.Length)
{
return false;
}
int runeStart = index - 1;
if (runeStart > 0
&& char.IsLowSurrogate(value[runeStart])
&& char.IsHighSurrogate(value[runeStart - 1]))
{
runeStart--;
}
return Rune.IsLetterOrDigit(Rune.GetRuneAt(value, runeStart));
}
private sealed record Surface(string Value, int KeywordId);
private sealed record Span(int Start, int Length);
}