416 lines
16 KiB
C#
416 lines
16 KiB
C#
using System.Collections.Concurrent;
|
|
using Microsoft.Extensions.Logging;
|
|
using RLHF.Generator.Generation;
|
|
using Xunit;
|
|
|
|
namespace RLHF.Generator.Tests;
|
|
|
|
public sealed class GenerationThroughputLoggingTests
|
|
{
|
|
[Fact]
|
|
public async Task PeriodicLog_ReportsRatesCommitsConcurrencyEtaAndStopsAfterSuccess()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
|
|
8,
|
|
requiresLmStudio: true);
|
|
ManualTimeProvider timeProvider = new();
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
CoordinatedLmStudioHandler handler = new(
|
|
allowBeforeGate: 4,
|
|
blockedRequestTarget: 4);
|
|
await using ServiceInstance instance = fixture.CreateService(
|
|
handler,
|
|
logger: logger,
|
|
timeProvider: timeProvider);
|
|
|
|
Task<GenerationRunResult> generationTask = instance.Service.GenerateAsync();
|
|
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
|
|
timeProvider.Advance(TimeSpan.FromSeconds(5));
|
|
TestLogEntry periodic = await logger.WaitForAsync(
|
|
entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
|
|
|
|
Assert.Contains("4/38", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("session 4", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("items/s current", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("items/s average", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("committed 0", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM active 4/4", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM peak 4", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("ETA ", periodic.Message, StringComparison.Ordinal);
|
|
|
|
handler.Release();
|
|
GenerationRunResult result = await generationTask;
|
|
Assert.Equal(38, result.TotalCommittedRows);
|
|
Assert.Contains(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal));
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
|
|
Assert.Contains("session processed 38", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("committed 38", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal);
|
|
|
|
int progressLogCount = logger.Entries.Count(entry =>
|
|
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
|
|
timeProvider.Advance(TimeSpan.FromMinutes(1));
|
|
await Task.Yield();
|
|
Assert.Equal(
|
|
progressLogCount,
|
|
logger.Entries.Count(entry =>
|
|
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal)));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DirectItems_NeverIncrementLmActivity()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1);
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
await using ServiceInstance instance = fixture.CreateService(logger: logger);
|
|
|
|
await instance.Service.GenerateAsync();
|
|
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
|
|
Assert.Contains("session processed 24", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM peak 0", final.Message, StringComparison.Ordinal);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FailedLmItems_AreNotProcessedAndReporterStops()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
|
|
8,
|
|
requiresLmStudio: true);
|
|
ManualTimeProvider timeProvider = new();
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
await using ServiceInstance instance = fixture.CreateService(
|
|
new InvalidOutputAfterResponsesHandler(0),
|
|
logger: logger,
|
|
timeProvider: timeProvider);
|
|
|
|
await Assert.ThrowsAsync<InvalidDataException>(
|
|
() => instance.Service.GenerateAsync());
|
|
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
|
|
Assert.Equal(LogLevel.Warning, final.Level);
|
|
Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("committed 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Matches("LM peak [1-4]", final.Message);
|
|
int logCount = logger.Entries.Count;
|
|
timeProvider.Advance(TimeSpan.FromMinutes(1));
|
|
await Task.Yield();
|
|
Assert.Equal(logCount, logger.Entries.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Cancellation_DecrementsLmActivityAndStopsReporter()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
|
|
8,
|
|
requiresLmStudio: true);
|
|
ManualTimeProvider timeProvider = new();
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
CoordinatedLmStudioHandler handler = new(
|
|
allowBeforeGate: 0,
|
|
blockedRequestTarget: 4);
|
|
await using ServiceInstance instance = fixture.CreateService(
|
|
handler,
|
|
logger: logger,
|
|
timeProvider: timeProvider);
|
|
using CancellationTokenSource cancellation = new();
|
|
|
|
Task<GenerationRunResult> generationTask = instance.Service.GenerateAsync(cancellation.Token);
|
|
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
|
|
await cancellation.CancelAsync();
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => generationTask);
|
|
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
|
|
Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal);
|
|
int logCount = logger.Entries.Count;
|
|
timeProvider.Advance(TimeSpan.FromMinutes(1));
|
|
await Task.Yield();
|
|
Assert.Equal(logCount, logger.Entries.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Resume_LogStartsSessionAtZeroAndRetainsCommittedProgress()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
|
|
20,
|
|
requiresLmStudio: true);
|
|
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(3);
|
|
await using (ServiceInstance failed = fixture.CreateService(interceptor: failure))
|
|
{
|
|
await Assert.ThrowsAsync<InvalidOperationException>(
|
|
() => failed.Service.GenerateAsync());
|
|
}
|
|
|
|
ManualTimeProvider timeProvider = new();
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
CoordinatedLmStudioHandler handler = new(
|
|
allowBeforeGate: 0,
|
|
blockedRequestTarget: 4);
|
|
await using ServiceInstance resumed = fixture.CreateService(
|
|
handler,
|
|
logger: logger,
|
|
timeProvider: timeProvider);
|
|
|
|
Task<GenerationRunResult> resumedTask = resumed.Service.GenerateAsync();
|
|
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
|
|
timeProvider.Advance(TimeSpan.FromSeconds(5));
|
|
TestLogEntry periodic = await logger.WaitForAsync(entry =>
|
|
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
|
|
Assert.Contains("10/62", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("session 0", periodic.Message, StringComparison.Ordinal);
|
|
Assert.Contains("committed 10", periodic.Message, StringComparison.Ordinal);
|
|
|
|
handler.Release();
|
|
await resumedTask;
|
|
|
|
TestLogEntry start = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal));
|
|
Assert.Contains("already committed 10", start.Message, StringComparison.Ordinal);
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
|
|
Assert.Contains("session processed 52", final.Message, StringComparison.Ordinal);
|
|
Assert.Contains("committed 62", final.Message, StringComparison.Ordinal);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FailedCommit_DoesNotAdvanceLoggedCommittedCount()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(20);
|
|
TestLogger<TrainingGenerationService> logger = new();
|
|
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(2);
|
|
await using ServiceInstance instance = fixture.CreateService(
|
|
interceptor: failure,
|
|
logger: logger);
|
|
|
|
await Assert.ThrowsAsync<InvalidOperationException>(
|
|
() => instance.Service.GenerateAsync());
|
|
|
|
TestLogEntry final = Assert.Single(logger.Entries, entry =>
|
|
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
|
|
Assert.Contains("committed 0", final.Message, StringComparison.Ordinal);
|
|
Assert.Equal(0, await fixture.CountRowsAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task NonPositiveProgressInterval_IsRejected()
|
|
{
|
|
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1);
|
|
GenerationOptions options = fixture.CreateGenerationOptions();
|
|
options.ProgressLogIntervalSeconds = 0;
|
|
|
|
InvalidOperationException exception = Assert.Throws<InvalidOperationException>(
|
|
() => fixture.CreateService(generationOptions: options));
|
|
|
|
Assert.Contains("ProgressLogIntervalSeconds", exception.Message, StringComparison.Ordinal);
|
|
}
|
|
}
|
|
|
|
internal sealed class CoordinatedLmStudioHandler : SuccessfulLmStudioHandler
|
|
{
|
|
private readonly int _allowBeforeGate;
|
|
private readonly int _blockedRequestTarget;
|
|
private readonly TaskCompletionSource _targetBlockedRequestsReached =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private readonly TaskCompletionSource _release =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
private int _requestCount;
|
|
private int _blockedRequestCount;
|
|
|
|
public CoordinatedLmStudioHandler(int allowBeforeGate, int blockedRequestTarget)
|
|
{
|
|
_allowBeforeGate = allowBeforeGate;
|
|
_blockedRequestTarget = blockedRequestTarget;
|
|
}
|
|
|
|
public Task TargetBlockedRequestsReached => _targetBlockedRequestsReached.Task;
|
|
|
|
public void Release() => _release.TrySetResult();
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (Interlocked.Increment(ref _requestCount) > _allowBeforeGate)
|
|
{
|
|
if (Interlocked.Increment(ref _blockedRequestCount) == _blockedRequestTarget)
|
|
{
|
|
_targetBlockedRequestsReached.TrySetResult();
|
|
}
|
|
|
|
await _release.Task.WaitAsync(cancellationToken);
|
|
}
|
|
|
|
string sentence = await ReadSentenceAsync(request, cancellationToken);
|
|
return CreateResponse(sentence);
|
|
}
|
|
}
|
|
|
|
internal sealed record TestLogEntry(LogLevel Level, string Message);
|
|
|
|
internal sealed class TestLogger<T> : ILogger<T>
|
|
{
|
|
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
|
|
private readonly SemaphoreSlim _logged = new(0);
|
|
|
|
public IReadOnlyCollection<TestLogEntry> Entries => _entries.ToArray();
|
|
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(
|
|
LogLevel logLevel,
|
|
EventId eventId,
|
|
TState state,
|
|
Exception? exception,
|
|
Func<TState, Exception?, string> formatter)
|
|
{
|
|
_entries.Enqueue(new TestLogEntry(logLevel, formatter(state, exception)));
|
|
_logged.Release();
|
|
}
|
|
|
|
public async Task<TestLogEntry> WaitForAsync(Func<TestLogEntry, bool> predicate)
|
|
{
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10));
|
|
while (true)
|
|
{
|
|
TestLogEntry? match = Entries.FirstOrDefault(predicate);
|
|
if (match is not null)
|
|
{
|
|
return match;
|
|
}
|
|
|
|
await _logged.WaitAsync(timeout.Token);
|
|
}
|
|
}
|
|
}
|
|
|
|
internal sealed class ManualTimeProvider : TimeProvider
|
|
{
|
|
private readonly object _sync = new();
|
|
private readonly List<ManualTimer> _timers = [];
|
|
private long _timestamp;
|
|
|
|
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
|
|
|
|
public override long GetTimestamp() => Volatile.Read(ref _timestamp);
|
|
|
|
public override ITimer CreateTimer(
|
|
TimerCallback callback,
|
|
object? state,
|
|
TimeSpan dueTime,
|
|
TimeSpan period)
|
|
{
|
|
ManualTimer timer = new(this, callback, state, dueTime, period);
|
|
lock (_sync)
|
|
{
|
|
_timers.Add(timer);
|
|
}
|
|
|
|
return timer;
|
|
}
|
|
|
|
public void Advance(TimeSpan duration)
|
|
{
|
|
List<(TimerCallback Callback, object? State)> callbacks = [];
|
|
lock (_sync)
|
|
{
|
|
_timestamp += duration.Ticks;
|
|
foreach (ManualTimer timer in _timers.ToArray())
|
|
{
|
|
timer.CollectCallbacks(_timestamp, callbacks);
|
|
}
|
|
}
|
|
|
|
foreach ((TimerCallback callback, object? state) in callbacks)
|
|
{
|
|
callback(state);
|
|
}
|
|
}
|
|
|
|
private void Remove(ManualTimer timer)
|
|
{
|
|
lock (_sync)
|
|
{
|
|
_timers.Remove(timer);
|
|
}
|
|
}
|
|
|
|
private sealed class ManualTimer : ITimer
|
|
{
|
|
private readonly ManualTimeProvider _provider;
|
|
private readonly TimerCallback _callback;
|
|
private readonly object? _state;
|
|
private long _nextTimestamp;
|
|
private long _periodTicks;
|
|
private bool _disposed;
|
|
|
|
public ManualTimer(
|
|
ManualTimeProvider provider,
|
|
TimerCallback callback,
|
|
object? state,
|
|
TimeSpan dueTime,
|
|
TimeSpan period)
|
|
{
|
|
_provider = provider;
|
|
_callback = callback;
|
|
_state = state;
|
|
Change(dueTime, period);
|
|
}
|
|
|
|
public bool Change(TimeSpan dueTime, TimeSpan period)
|
|
{
|
|
if (_disposed)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_nextTimestamp = dueTime == Timeout.InfiniteTimeSpan
|
|
? long.MaxValue
|
|
: _provider.GetTimestamp() + dueTime.Ticks;
|
|
_periodTicks = period == Timeout.InfiniteTimeSpan
|
|
? 0
|
|
: period.Ticks;
|
|
return true;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
_disposed = true;
|
|
_provider.Remove(this);
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
Dispose();
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
|
|
public void CollectCallbacks(
|
|
long timestamp,
|
|
List<(TimerCallback Callback, object? State)> callbacks)
|
|
{
|
|
while (!_disposed && timestamp >= _nextTimestamp)
|
|
{
|
|
callbacks.Add((_callback, _state));
|
|
if (_periodTicks <= 0)
|
|
{
|
|
_nextTimestamp = long.MaxValue;
|
|
break;
|
|
}
|
|
|
|
_nextTimestamp += _periodTicks;
|
|
}
|
|
}
|
|
}
|
|
} |