using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using RLHF.Generator.Generation; using Xunit; namespace RLHF.Generator.Tests; public sealed class GenerationThroughputLoggingTests { [Fact] public async Task PeriodicLog_ReportsRatesCommitsConcurrencyEtaAndStopsAfterSuccess() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync( 8, requiresLmStudio: true); ManualTimeProvider timeProvider = new(); TestLogger logger = new(); CoordinatedLmStudioHandler handler = new( allowBeforeGate: 4, blockedRequestTarget: 4); await using ServiceInstance instance = fixture.CreateService( handler, logger: logger, timeProvider: timeProvider); Task generationTask = instance.Service.GenerateAsync(); await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10)); timeProvider.Advance(TimeSpan.FromSeconds(5)); TestLogEntry periodic = await logger.WaitForAsync( entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal)); Assert.Contains("4/38", periodic.Message, StringComparison.Ordinal); Assert.Contains("session 4", periodic.Message, StringComparison.Ordinal); Assert.Contains("items/s current", periodic.Message, StringComparison.Ordinal); Assert.Contains("items/s average", periodic.Message, StringComparison.Ordinal); Assert.Contains("committed 0", periodic.Message, StringComparison.Ordinal); Assert.Contains("LM active 4/4", periodic.Message, StringComparison.Ordinal); Assert.Contains("LM peak 4", periodic.Message, StringComparison.Ordinal); Assert.Contains("ETA ", periodic.Message, StringComparison.Ordinal); handler.Release(); GenerationRunResult result = await generationTask; Assert.Equal(38, result.TotalCommittedRows); Assert.Contains(logger.Entries, entry => entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal)); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal)); Assert.Contains("session processed 38", final.Message, StringComparison.Ordinal); Assert.Contains("committed 38", final.Message, StringComparison.Ordinal); Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal); Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal); int progressLogCount = logger.Entries.Count(entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal)); timeProvider.Advance(TimeSpan.FromMinutes(1)); await Task.Yield(); Assert.Equal( progressLogCount, logger.Entries.Count(entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal))); } [Fact] public async Task DirectItems_NeverIncrementLmActivity() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1); TestLogger logger = new(); await using ServiceInstance instance = fixture.CreateService(logger: logger); await instance.Service.GenerateAsync(); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal)); Assert.Contains("session processed 24", final.Message, StringComparison.Ordinal); Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal); Assert.Contains("LM peak 0", final.Message, StringComparison.Ordinal); } [Fact] public async Task FailedLmItems_AreNotProcessedAndReporterStops() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync( 8, requiresLmStudio: true); ManualTimeProvider timeProvider = new(); TestLogger logger = new(); await using ServiceInstance instance = fixture.CreateService( new InvalidOutputAfterResponsesHandler(0), logger: logger, timeProvider: timeProvider); await Assert.ThrowsAsync( () => instance.Service.GenerateAsync()); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal)); Assert.Equal(LogLevel.Warning, final.Level); Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal); Assert.Contains("committed 0", final.Message, StringComparison.Ordinal); Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal); Assert.Matches("LM peak [1-4]", final.Message); int logCount = logger.Entries.Count; timeProvider.Advance(TimeSpan.FromMinutes(1)); await Task.Yield(); Assert.Equal(logCount, logger.Entries.Count); } [Fact] public async Task Cancellation_DecrementsLmActivityAndStopsReporter() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync( 8, requiresLmStudio: true); ManualTimeProvider timeProvider = new(); TestLogger logger = new(); CoordinatedLmStudioHandler handler = new( allowBeforeGate: 0, blockedRequestTarget: 4); await using ServiceInstance instance = fixture.CreateService( handler, logger: logger, timeProvider: timeProvider); using CancellationTokenSource cancellation = new(); Task generationTask = instance.Service.GenerateAsync(cancellation.Token); await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10)); await cancellation.CancelAsync(); await Assert.ThrowsAnyAsync(() => generationTask); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal)); Assert.Contains("session processed 0", final.Message, StringComparison.Ordinal); Assert.Contains("LM active 0", final.Message, StringComparison.Ordinal); Assert.Contains("LM peak 4", final.Message, StringComparison.Ordinal); int logCount = logger.Entries.Count; timeProvider.Advance(TimeSpan.FromMinutes(1)); await Task.Yield(); Assert.Equal(logCount, logger.Entries.Count); } [Fact] public async Task Resume_LogStartsSessionAtZeroAndRetainsCommittedProgress() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync( 20, requiresLmStudio: true); BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(3); await using (ServiceInstance failed = fixture.CreateService(interceptor: failure)) { await Assert.ThrowsAsync( () => failed.Service.GenerateAsync()); } ManualTimeProvider timeProvider = new(); TestLogger logger = new(); CoordinatedLmStudioHandler handler = new( allowBeforeGate: 0, blockedRequestTarget: 4); await using ServiceInstance resumed = fixture.CreateService( handler, logger: logger, timeProvider: timeProvider); Task resumedTask = resumed.Service.GenerateAsync(); await handler.TargetBlockedRequestsReached.WaitAsync(TimeSpan.FromSeconds(10)); timeProvider.Advance(TimeSpan.FromSeconds(5)); TestLogEntry periodic = await logger.WaitForAsync(entry => entry.Message.StartsWith("Generation progress:", StringComparison.Ordinal)); Assert.Contains("10/62", periodic.Message, StringComparison.Ordinal); Assert.Contains("session 0", periodic.Message, StringComparison.Ordinal); Assert.Contains("committed 10", periodic.Message, StringComparison.Ordinal); handler.Release(); await resumedTask; TestLogEntry start = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution started:", StringComparison.Ordinal)); Assert.Contains("already committed 10", start.Message, StringComparison.Ordinal); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution completed:", StringComparison.Ordinal)); Assert.Contains("session processed 52", final.Message, StringComparison.Ordinal); Assert.Contains("committed 62", final.Message, StringComparison.Ordinal); } [Fact] public async Task FailedCommit_DoesNotAdvanceLoggedCommittedCount() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync(20); TestLogger logger = new(); BatchFaultInterceptor failure = BatchFaultInterceptor.BeforeSave(2); await using ServiceInstance instance = fixture.CreateService( interceptor: failure, logger: logger); await Assert.ThrowsAsync( () => instance.Service.GenerateAsync()); TestLogEntry final = Assert.Single(logger.Entries, entry => entry.Message.StartsWith("Generation execution interrupted:", StringComparison.Ordinal)); Assert.Contains("committed 0", final.Message, StringComparison.Ordinal); Assert.Equal(0, await fixture.CountRowsAsync()); } [Fact] public async Task NonPositiveProgressInterval_IsRejected() { await using GenerationFixture fixture = await GenerationFixture.CreateAsync(1); GenerationOptions options = fixture.CreateGenerationOptions(); options.ProgressLogIntervalSeconds = 0; InvalidOperationException exception = Assert.Throws( () => fixture.CreateService(generationOptions: options)); Assert.Contains("ProgressLogIntervalSeconds", exception.Message, StringComparison.Ordinal); } } internal sealed class CoordinatedLmStudioHandler : SuccessfulLmStudioHandler { private readonly int _allowBeforeGate; private readonly int _blockedRequestTarget; private readonly TaskCompletionSource _targetBlockedRequestsReached = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _requestCount; private int _blockedRequestCount; public CoordinatedLmStudioHandler(int allowBeforeGate, int blockedRequestTarget) { _allowBeforeGate = allowBeforeGate; _blockedRequestTarget = blockedRequestTarget; } public Task TargetBlockedRequestsReached => _targetBlockedRequestsReached.Task; public void Release() => _release.TrySetResult(); protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { if (Interlocked.Increment(ref _requestCount) > _allowBeforeGate) { if (Interlocked.Increment(ref _blockedRequestCount) == _blockedRequestTarget) { _targetBlockedRequestsReached.TrySetResult(); } await _release.Task.WaitAsync(cancellationToken); } string sentence = await ReadSentenceAsync(request, cancellationToken); return CreateResponse(sentence); } } internal sealed record TestLogEntry(LogLevel Level, string Message); internal sealed class TestLogger : ILogger { private readonly ConcurrentQueue _entries = new(); private readonly SemaphoreSlim _logged = new(0); public IReadOnlyCollection Entries => _entries.ToArray(); public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { _entries.Enqueue(new TestLogEntry(logLevel, formatter(state, exception))); _logged.Release(); } public async Task WaitForAsync(Func predicate) { using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); while (true) { TestLogEntry? match = Entries.FirstOrDefault(predicate); if (match is not null) { return match; } await _logged.WaitAsync(timeout.Token); } } } internal sealed class ManualTimeProvider : TimeProvider { private readonly object _sync = new(); private readonly List _timers = []; private long _timestamp; public override long TimestampFrequency => TimeSpan.TicksPerSecond; public override long GetTimestamp() => Volatile.Read(ref _timestamp); public override ITimer CreateTimer( TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) { ManualTimer timer = new(this, callback, state, dueTime, period); lock (_sync) { _timers.Add(timer); } return timer; } public void Advance(TimeSpan duration) { List<(TimerCallback Callback, object? State)> callbacks = []; lock (_sync) { _timestamp += duration.Ticks; foreach (ManualTimer timer in _timers.ToArray()) { timer.CollectCallbacks(_timestamp, callbacks); } } foreach ((TimerCallback callback, object? state) in callbacks) { callback(state); } } private void Remove(ManualTimer timer) { lock (_sync) { _timers.Remove(timer); } } private sealed class ManualTimer : ITimer { private readonly ManualTimeProvider _provider; private readonly TimerCallback _callback; private readonly object? _state; private long _nextTimestamp; private long _periodTicks; private bool _disposed; public ManualTimer( ManualTimeProvider provider, TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) { _provider = provider; _callback = callback; _state = state; Change(dueTime, period); } public bool Change(TimeSpan dueTime, TimeSpan period) { if (_disposed) { return false; } _nextTimestamp = dueTime == Timeout.InfiniteTimeSpan ? long.MaxValue : _provider.GetTimestamp() + dueTime.Ticks; _periodTicks = period == Timeout.InfiniteTimeSpan ? 0 : period.Ticks; return true; } public void Dispose() { _disposed = true; _provider.Remove(this); } public ValueTask DisposeAsync() { Dispose(); return ValueTask.CompletedTask; } public void CollectCallbacks( long timestamp, List<(TimerCallback Callback, object? State)> callbacks) { while (!_disposed && timestamp >= _nextTimestamp) { callbacks.Add((_callback, _state)); if (_periodTicks <= 0) { _nextTimestamp = long.MaxValue; break; } _nextTimestamp += _periodTicks; } } } }