PHI-Model-8B/PHI.Data.Generator/RLHF.Generator.Tests/GenerationThroughputLogging...

416 lines
16 KiB
C#

using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using RLHF.Generator.Generation;
using Xunit;
namespace RLHF.Generator.Tests;
public sealed class GenerationThroughputLoggingTests
{
[Fact]
public async Task PeriodicLog_ReportsRatesCommitsConcurrencyEtaAndStopsAfterSuccess()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
8,
requiresLmStudio: true);
ManualTimeProvider timeProvider = new();
TestLogger<TrainingGenerationService> logger = new();
CoordinatedLmStudioHandler handler = new(
allowBeforeGate: 4,
blockedRequestTarget: 4);
await using ServiceInstance instance = fixture.CreateService(
handler,
logger: logger,
timeProvider: timeProvider);
Task<GenerationRunResult> generationTask = instance.Service.GenerateAsync();
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
timeProvider.Advance(TimeSpan.FromSeconds(5));
TestLogEntry periodic = await logger.WaitForAsync(
entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
Assert.Contains("4/38", periodic.Message, StringComparison.Ordinal);
Assert.Contains("session 4", periodic.Message, StringComparison.Ordinal);
Assert.Contains("items/s current", periodic.Message, StringComparison.Ordinal);
Assert.Contains("items/s average", periodic.Message, StringComparison.Ordinal);
Assert.Contains("committed 0", periodic.Message, StringComparison.Ordinal);
Assert.Contains("LM active 4/4", periodic.Message, StringComparison.Ordinal);
Assert.Contains("LM peak 4", periodic.Message, StringComparison.Ordinal);
Assert.Contains("ETA ", periodic.Message, StringComparison.Ordinal);
handler.Release();
GenerationRunResult result = await generationTask;
Assert.Equal(38, result.TotalCommittedRows);
Assert.Contains(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal));
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
Assert.Contains("session processed 38", final.Message, StringComparison.Ordinal);
Assert.Contains("committed 38", final.Message, StringComparison.Ordinal);
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal);
int progressLogCount = logger.Entries.Count(entry =>
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
timeProvider.Advance(TimeSpan.FromMinutes(1));
await Task.Yield();
Assert.Equal(
progressLogCount,
logger.Entries.Count(entry =>
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal)));
}
[Fact]
public async Task DirectItems_NeverIncrementLmActivity()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1);
TestLogger<TrainingGenerationService> logger = new();
await using ServiceInstance instance = fixture.CreateService(logger: logger);
await instance.Service.GenerateAsync();
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
Assert.Contains("session processed 24", final.Message, StringComparison.Ordinal);
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
Assert.Contains("LM peak 0", final.Message, StringComparison.Ordinal);
}
[Fact]
public async Task FailedLmItems_AreNotProcessedAndReporterStops()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
8,
requiresLmStudio: true);
ManualTimeProvider timeProvider = new();
TestLogger<TrainingGenerationService> logger = new();
await using ServiceInstance instance = fixture.CreateService(
new InvalidOutputAfterResponsesHandler(0),
logger: logger,
timeProvider: timeProvider);
await Assert.ThrowsAsync<InvalidDataException>(
() => instance.Service.GenerateAsync());
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
Assert.Equal(LogLevel.Warning, final.Level);
Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal);
Assert.Contains("committed 0", final.Message, StringComparison.Ordinal);
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
Assert.Matches("LM peak [1-4]", final.Message);
int logCount = logger.Entries.Count;
timeProvider.Advance(TimeSpan.FromMinutes(1));
await Task.Yield();
Assert.Equal(logCount, logger.Entries.Count);
}
[Fact]
public async Task Cancellation_DecrementsLmActivityAndStopsReporter()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
8,
requiresLmStudio: true);
ManualTimeProvider timeProvider = new();
TestLogger<TrainingGenerationService> logger = new();
CoordinatedLmStudioHandler handler = new(
allowBeforeGate: 0,
blockedRequestTarget: 4);
await using ServiceInstance instance = fixture.CreateService(
handler,
logger: logger,
timeProvider: timeProvider);
using CancellationTokenSource cancellation = new();
Task<GenerationRunResult> generationTask = instance.Service.GenerateAsync(cancellation.Token);
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
await cancellation.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => generationTask);
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal);
Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal);
Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal);
int logCount = logger.Entries.Count;
timeProvider.Advance(TimeSpan.FromMinutes(1));
await Task.Yield();
Assert.Equal(logCount, logger.Entries.Count);
}
[Fact]
public async Task Resume_LogStartsSessionAtZeroAndRetainsCommittedProgress()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(
20,
requiresLmStudio: true);
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(3);
await using (ServiceInstance failed = fixture.CreateService(interceptor: failure))
{
await Assert.ThrowsAsync<InvalidOperationException>(
() => failed.Service.GenerateAsync());
}
ManualTimeProvider timeProvider = new();
TestLogger<TrainingGenerationService> logger = new();
CoordinatedLmStudioHandler handler = new(
allowBeforeGate: 0,
blockedRequestTarget: 4);
await using ServiceInstance resumed = fixture.CreateService(
handler,
logger: logger,
timeProvider: timeProvider);
Task<GenerationRunResult> resumedTask = resumed.Service.GenerateAsync();
await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10));
timeProvider.Advance(TimeSpan.FromSeconds(5));
TestLogEntry periodic = await logger.WaitForAsync(entry =>
entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal));
Assert.Contains("10/62", periodic.Message, StringComparison.Ordinal);
Assert.Contains("session 0", periodic.Message, StringComparison.Ordinal);
Assert.Contains("committed 10", periodic.Message, StringComparison.Ordinal);
handler.Release();
await resumedTask;
TestLogEntry start = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal));
Assert.Contains("already committed 10", start.Message, StringComparison.Ordinal);
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal));
Assert.Contains("session processed 52", final.Message, StringComparison.Ordinal);
Assert.Contains("committed 62", final.Message, StringComparison.Ordinal);
}
[Fact]
public async Task FailedCommit_DoesNotAdvanceLoggedCommittedCount()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(20);
TestLogger<TrainingGenerationService> logger = new();
BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(2);
await using ServiceInstance instance = fixture.CreateService(
interceptor: failure,
logger: logger);
await Assert.ThrowsAsync<InvalidOperationException>(
() => instance.Service.GenerateAsync());
TestLogEntry final = Assert.Single(logger.Entries, entry =>
entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal));
Assert.Contains("committed 0", final.Message, StringComparison.Ordinal);
Assert.Equal(0, await fixture.CountRowsAsync());
}
[Fact]
public async Task NonPositiveProgressInterval_IsRejected()
{
await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1);
GenerationOptions options = fixture.CreateGenerationOptions();
options.ProgressLogIntervalSeconds = 0;
InvalidOperationException exception = Assert.Throws<InvalidOperationException>(
() => fixture.CreateService(generationOptions: options));
Assert.Contains("ProgressLogIntervalSeconds", exception.Message, StringComparison.Ordinal);
}
}
internal sealed class CoordinatedLmStudioHandler : SuccessfulLmStudioHandler
{
private readonly int _allowBeforeGate;
private readonly int _blockedRequestTarget;
private readonly TaskCompletionSource _targetBlockedRequestsReached =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _requestCount;
private int _blockedRequestCount;
public CoordinatedLmStudioHandler(int allowBeforeGate, int blockedRequestTarget)
{
_allowBeforeGate = allowBeforeGate;
_blockedRequestTarget = blockedRequestTarget;
}
public Task TargetBlockedRequestsReached => _targetBlockedRequestsReached.Task;
public void Release() => _release.TrySetResult();
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
if (Interlocked.Increment(ref _requestCount) > _allowBeforeGate)
{
if (Interlocked.Increment(ref _blockedRequestCount) == _blockedRequestTarget)
{
_targetBlockedRequestsReached.TrySetResult();
}
await _release.Task.WaitAsync(cancellationToken);
}
string sentence = await ReadSentenceAsync(request, cancellationToken);
return CreateResponse(sentence);
}
}
internal sealed record TestLogEntry(LogLevel Level, string Message);
internal sealed class TestLogger<T> : ILogger<T>
{
private readonly ConcurrentQueue<TestLogEntry> _entries = new();
private readonly SemaphoreSlim _logged = new(0);
public IReadOnlyCollection<TestLogEntry> Entries => _entries.ToArray();
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
_entries.Enqueue(new TestLogEntry(logLevel, formatter(state, exception)));
_logged.Release();
}
public async Task<TestLogEntry> WaitForAsync(Func<TestLogEntry, bool> predicate)
{
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10));
while (true)
{
TestLogEntry? match = Entries.FirstOrDefault(predicate);
if (match is not null)
{
return match;
}
await _logged.WaitAsync(timeout.Token);
}
}
}
internal sealed class ManualTimeProvider : TimeProvider
{
private readonly object _sync = new();
private readonly List<ManualTimer> _timers = [];
private long _timestamp;
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
public override long GetTimestamp() => Volatile.Read(ref _timestamp);
public override ITimer CreateTimer(
TimerCallback callback,
object? state,
TimeSpan dueTime,
TimeSpan period)
{
ManualTimer timer = new(this, callback, state, dueTime, period);
lock (_sync)
{
_timers.Add(timer);
}
return timer;
}
public void Advance(TimeSpan duration)
{
List<(TimerCallback Callback, object? State)> callbacks = [];
lock (_sync)
{
_timestamp += duration.Ticks;
foreach (ManualTimer timer in _timers.ToArray())
{
timer.CollectCallbacks(_timestamp, callbacks);
}
}
foreach ((TimerCallback callback, object? state) in callbacks)
{
callback(state);
}
}
private void Remove(ManualTimer timer)
{
lock (_sync)
{
_timers.Remove(timer);
}
}
private sealed class ManualTimer : ITimer
{
private readonly ManualTimeProvider _provider;
private readonly TimerCallback _callback;
private readonly object? _state;
private long _nextTimestamp;
private long _periodTicks;
private bool _disposed;
public ManualTimer(
ManualTimeProvider provider,
TimerCallback callback,
object? state,
TimeSpan dueTime,
TimeSpan period)
{
_provider = provider;
_callback = callback;
_state = state;
Change(dueTime, period);
}
public bool Change(TimeSpan dueTime, TimeSpan period)
{
if (_disposed)
{
return false;
}
_nextTimestamp = dueTime == Timeout.InfiniteTimeSpan
? long.MaxValue
: _provider.GetTimestamp() + dueTime.Ticks;
_periodTicks = period == Timeout.InfiniteTimeSpan
? 0
: period.Ticks;
return true;
}
public void Dispose()
{
_disposed = true;
_provider.Remove(this);
}
public ValueTask DisposeAsync()
{
Dispose();
return ValueTask.CompletedTask;
}
public void CollectCallbacks(
long timestamp,
List<(TimerCallback Callback, object? State)> callbacks)
{
while (!_disposed && timestamp >= _nextTimestamp)
{
callbacks.Add((_callback, _state));
if (_periodTicks <= 0)
{
_nextTimestamp = long.MaxValue;
break;
}
_nextTimestamp += _periodTicks;
}
}
}
}