PHI-Model-8B/PHI.Data.Generator/RLHF.Generator/TrainingData/TrainingDataReader.cs

225 lines
8.2 KiB
C#

using System.Collections.ObjectModel;
namespace RLHF.Generator.TrainingData;
public sealed class TrainingDataReader
{
public TrainingDataReader(string dataDirectory)
{
if (string.IsNullOrWhiteSpace(dataDirectory))
{
throw new ArgumentException("A data directory is required.", nameof(dataDirectory));
}
DataDirectory = Path.GetFullPath(dataDirectory);
}
public string DataDirectory { get; }
public async Task ValidateAsync(CancellationToken cancellationToken = default)
{
if (!Directory.Exists(DataDirectory))
{
throw new DirectoryNotFoundException($"Data directory not found: '{DataDirectory}'.");
}
string categoriesDirectory = Path.Combine(DataDirectory, "Categories");
if (!Directory.Exists(categoriesDirectory))
{
throw new DirectoryNotFoundException($"Categories directory not found: '{categoriesDirectory}'.");
}
string patternsDirectory = Path.Combine(DataDirectory, "Patterns");
if (!Directory.Exists(patternsDirectory))
{
throw new DirectoryNotFoundException($"Patterns directory not found: '{patternsDirectory}'.");
}
string subjectsPath = Path.Combine(DataDirectory, "Subjects.txt");
if (!File.Exists(subjectsPath))
{
throw new FileNotFoundException($"Subjects file not found: '{subjectsPath}'.", subjectsPath);
}
HashSet<string> expectedCategoryFiles = TrainingCategories.All
.Select(category => $"{category.Name}.txt")
.ToHashSet(StringComparer.Ordinal);
ValidateFileSet(categoriesDirectory, expectedCategoryFiles, "category");
HashSet<string> expectedPatternFiles = TrainingCategories.All
.SelectMany(category => new[]
{
$"{category.Name}.positive.txt",
$"{category.Name}.negative.txt"
})
.ToHashSet(StringComparer.Ordinal);
ValidateFileSet(patternsDirectory, expectedPatternFiles, "pattern");
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
{
await ReadCategoryValuesAsync(category, cancellationToken);
foreach (bool isPositive in new[] { true, false })
{
string patternPath = GetPatternPath(category, isPositive);
IReadOnlyList<string> patterns = await ReadActiveLinesAsync(patternPath, cancellationToken);
foreach (string pattern in patterns)
{
if (pattern.Contains("[s]", StringComparison.Ordinal))
{
throw new InvalidDataException(
$"Legacy placeholder '[s]' found in pattern file '{patternPath}'.");
}
}
}
}
IReadOnlyList<string> subjects = await ReadActiveLinesAsync(subjectsPath, cancellationToken);
if (!subjects.SequenceEqual(TrainingSubjects.All, StringComparer.Ordinal))
{
throw new InvalidDataException(
$"Active subjects in '{subjectsPath}' do not exactly match the subject catalog. " +
$"Found: '{string.Join("', '", subjects)}'.");
}
}
public async Task<IReadOnlyList<TrainingCategoryValue>> ReadCategoryValuesAsync(
TrainingCategoryDefinition category,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(category);
string path = Path.Combine(DataDirectory, "Categories", $"{category.Name}.txt");
if (!File.Exists(path))
{
throw new FileNotFoundException($"Required data file not found: '{path}'.", path);
}
List<TrainingCategoryValue> values = [];
Dictionary<string, int> formLineNumbers = new(StringComparer.OrdinalIgnoreCase);
using StreamReader reader = File.OpenText(path);
int lineNumber = 0;
while (await reader.ReadLineAsync(cancellationToken) is { } line)
{
lineNumber++;
string trimmedLine = line.Trim();
if (trimmedLine.Length == 0 || trimmedLine.StartsWith('#'))
{
continue;
}
string[] forms = trimmedLine
.Split('|')
.Select(form => form.Trim())
.ToArray();
if (forms.Any(form => form.Length == 0))
{
throw new InvalidDataException(
$"Category '{category.Name}' line {lineNumber} contains an empty surface form.");
}
HashSet<string> lineForms = new(StringComparer.OrdinalIgnoreCase);
foreach (string form in forms)
{
if (!lineForms.Add(form))
{
throw new InvalidDataException(
$"Category '{category.Name}' line {lineNumber} contains duplicate surface form '{form}'.");
}
if (formLineNumbers.TryGetValue(form, out int conflictingLineNumber))
{
throw new InvalidDataException(
$"Category '{category.Name}' surface form '{form}' conflicts between lines " +
$"{conflictingLineNumber} and {lineNumber}.");
}
}
foreach (string form in forms)
{
formLineNumbers.Add(form, lineNumber);
}
values.Add(new TrainingCategoryValue(forms[0], forms));
}
if (values.Count == 0)
{
throw new InvalidDataException($"Data file contains no active lines: '{path}'.");
}
return new ReadOnlyCollection<TrainingCategoryValue>(values);
}
public Task<IReadOnlyList<string>> ReadPatternsAsync(
TrainingCategoryDefinition category,
bool isPositive,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(category);
return ReadActiveLinesAsync(GetPatternPath(category, isPositive), cancellationToken);
}
private string GetPatternPath(TrainingCategoryDefinition category, bool isPositive)
{
string sentiment = isPositive ? "positive" : "negative";
return Path.Combine(DataDirectory, "Patterns", $"{category.Name}.{sentiment}.txt");
}
private static void ValidateFileSet(
string directory,
IReadOnlySet<string> expectedFiles,
string fileKind)
{
HashSet<string> actualFiles = Directory
.EnumerateFiles(directory, "*.txt", SearchOption.TopDirectoryOnly)
.Select(path => Path.GetFileName(path)!)
.ToHashSet(StringComparer.Ordinal);
string? missingFile = expectedFiles.FirstOrDefault(file => !actualFiles.Contains(file));
if (missingFile is not null)
{
string missingPath = Path.Combine(directory, missingFile);
throw new FileNotFoundException(
$"Expected {fileKind} file not found: '{missingPath}'.",
missingPath);
}
string? unexpectedFile = actualFiles.FirstOrDefault(file => !expectedFiles.Contains(file));
if (unexpectedFile is not null)
{
throw new InvalidDataException(
$"Unexpected {fileKind} file found: '{Path.Combine(directory, unexpectedFile)}'.");
}
}
private static async Task<IReadOnlyList<string>> ReadActiveLinesAsync(
string path,
CancellationToken cancellationToken)
{
if (!File.Exists(path))
{
throw new FileNotFoundException($"Required data file not found: '{path}'.", path);
}
List<string> activeLines = [];
using StreamReader reader = File.OpenText(path);
while (await reader.ReadLineAsync(cancellationToken) is { } line)
{
string trimmedLine = line.Trim();
if (trimmedLine.Length > 0 && !trimmedLine.StartsWith('#'))
{
activeLines.Add(trimmedLine);
}
}
if (activeLines.Count == 0)
{
throw new InvalidDataException($"Data file contains no active lines: '{path}'.");
}
return new ReadOnlyCollection<string>(activeLines);
}
}