225 lines
8.2 KiB
C#
225 lines
8.2 KiB
C#
using System.Collections.ObjectModel;
|
|
|
|
namespace RLHF.Generator.TrainingData;
|
|
|
|
public sealed class TrainingDataReader
|
|
{
|
|
public TrainingDataReader(string dataDirectory)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(dataDirectory))
|
|
{
|
|
throw new ArgumentException("A data directory is required.", nameof(dataDirectory));
|
|
}
|
|
|
|
DataDirectory = Path.GetFullPath(dataDirectory);
|
|
}
|
|
|
|
public string DataDirectory { get; }
|
|
|
|
public async Task ValidateAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
if (!Directory.Exists(DataDirectory))
|
|
{
|
|
throw new DirectoryNotFoundException($"Data directory not found: '{DataDirectory}'.");
|
|
}
|
|
|
|
string categoriesDirectory = Path.Combine(DataDirectory, "Categories");
|
|
if (!Directory.Exists(categoriesDirectory))
|
|
{
|
|
throw new DirectoryNotFoundException($"Categories directory not found: '{categoriesDirectory}'.");
|
|
}
|
|
|
|
string patternsDirectory = Path.Combine(DataDirectory, "Patterns");
|
|
if (!Directory.Exists(patternsDirectory))
|
|
{
|
|
throw new DirectoryNotFoundException($"Patterns directory not found: '{patternsDirectory}'.");
|
|
}
|
|
|
|
string subjectsPath = Path.Combine(DataDirectory, "Subjects.txt");
|
|
if (!File.Exists(subjectsPath))
|
|
{
|
|
throw new FileNotFoundException($"Subjects file not found: '{subjectsPath}'.", subjectsPath);
|
|
}
|
|
|
|
HashSet<string> expectedCategoryFiles = TrainingCategories.All
|
|
.Select(category => $"{category.Name}.txt")
|
|
.ToHashSet(StringComparer.Ordinal);
|
|
ValidateFileSet(categoriesDirectory, expectedCategoryFiles, "category");
|
|
|
|
HashSet<string> expectedPatternFiles = TrainingCategories.All
|
|
.SelectMany(category => new[]
|
|
{
|
|
$"{category.Name}.positive.txt",
|
|
$"{category.Name}.negative.txt"
|
|
})
|
|
.ToHashSet(StringComparer.Ordinal);
|
|
ValidateFileSet(patternsDirectory, expectedPatternFiles, "pattern");
|
|
|
|
foreach (TrainingCategoryDefinition category in TrainingCategories.All)
|
|
{
|
|
await ReadCategoryValuesAsync(category, cancellationToken);
|
|
|
|
foreach (bool isPositive in new[] { true, false })
|
|
{
|
|
string patternPath = GetPatternPath(category, isPositive);
|
|
IReadOnlyList<string> patterns = await ReadActiveLinesAsync(patternPath, cancellationToken);
|
|
foreach (string pattern in patterns)
|
|
{
|
|
if (pattern.Contains("[s]", StringComparison.Ordinal))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Legacy placeholder '[s]' found in pattern file '{patternPath}'.");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
IReadOnlyList<string> subjects = await ReadActiveLinesAsync(subjectsPath, cancellationToken);
|
|
if (!subjects.SequenceEqual(TrainingSubjects.All, StringComparer.Ordinal))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Active subjects in '{subjectsPath}' do not exactly match the subject catalog. " +
|
|
$"Found: '{string.Join("', '", subjects)}'.");
|
|
}
|
|
}
|
|
|
|
public async Task<IReadOnlyList<TrainingCategoryValue>> ReadCategoryValuesAsync(
|
|
TrainingCategoryDefinition category,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(category);
|
|
|
|
string path = Path.Combine(DataDirectory, "Categories", $"{category.Name}.txt");
|
|
if (!File.Exists(path))
|
|
{
|
|
throw new FileNotFoundException($"Required data file not found: '{path}'.", path);
|
|
}
|
|
|
|
List<TrainingCategoryValue> values = [];
|
|
Dictionary<string, int> formLineNumbers = new(StringComparer.OrdinalIgnoreCase);
|
|
using StreamReader reader = File.OpenText(path);
|
|
int lineNumber = 0;
|
|
while (await reader.ReadLineAsync(cancellationToken) is { } line)
|
|
{
|
|
lineNumber++;
|
|
string trimmedLine = line.Trim();
|
|
if (trimmedLine.Length == 0 || trimmedLine.StartsWith('#'))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string[] forms = trimmedLine
|
|
.Split('|')
|
|
.Select(form => form.Trim())
|
|
.ToArray();
|
|
if (forms.Any(form => form.Length == 0))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Category '{category.Name}' line {lineNumber} contains an empty surface form.");
|
|
}
|
|
|
|
HashSet<string> lineForms = new(StringComparer.OrdinalIgnoreCase);
|
|
foreach (string form in forms)
|
|
{
|
|
if (!lineForms.Add(form))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Category '{category.Name}' line {lineNumber} contains duplicate surface form '{form}'.");
|
|
}
|
|
|
|
if (formLineNumbers.TryGetValue(form, out int conflictingLineNumber))
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Category '{category.Name}' surface form '{form}' conflicts between lines " +
|
|
$"{conflictingLineNumber} and {lineNumber}.");
|
|
}
|
|
}
|
|
|
|
foreach (string form in forms)
|
|
{
|
|
formLineNumbers.Add(form, lineNumber);
|
|
}
|
|
|
|
values.Add(new TrainingCategoryValue(forms[0], forms));
|
|
}
|
|
|
|
if (values.Count == 0)
|
|
{
|
|
throw new InvalidDataException($"Data file contains no active lines: '{path}'.");
|
|
}
|
|
|
|
return new ReadOnlyCollection<TrainingCategoryValue>(values);
|
|
}
|
|
|
|
public Task<IReadOnlyList<string>> ReadPatternsAsync(
|
|
TrainingCategoryDefinition category,
|
|
bool isPositive,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(category);
|
|
|
|
return ReadActiveLinesAsync(GetPatternPath(category, isPositive), cancellationToken);
|
|
}
|
|
|
|
private string GetPatternPath(TrainingCategoryDefinition category, bool isPositive)
|
|
{
|
|
string sentiment = isPositive ? "positive" : "negative";
|
|
return Path.Combine(DataDirectory, "Patterns", $"{category.Name}.{sentiment}.txt");
|
|
}
|
|
|
|
private static void ValidateFileSet(
|
|
string directory,
|
|
IReadOnlySet<string> expectedFiles,
|
|
string fileKind)
|
|
{
|
|
HashSet<string> actualFiles = Directory
|
|
.EnumerateFiles(directory, "*.txt", SearchOption.TopDirectoryOnly)
|
|
.Select(path => Path.GetFileName(path)!)
|
|
.ToHashSet(StringComparer.Ordinal);
|
|
|
|
string? missingFile = expectedFiles.FirstOrDefault(file => !actualFiles.Contains(file));
|
|
if (missingFile is not null)
|
|
{
|
|
string missingPath = Path.Combine(directory, missingFile);
|
|
throw new FileNotFoundException(
|
|
$"Expected {fileKind} file not found: '{missingPath}'.",
|
|
missingPath);
|
|
}
|
|
|
|
string? unexpectedFile = actualFiles.FirstOrDefault(file => !expectedFiles.Contains(file));
|
|
if (unexpectedFile is not null)
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Unexpected {fileKind} file found: '{Path.Combine(directory, unexpectedFile)}'.");
|
|
}
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<string>> ReadActiveLinesAsync(
|
|
string path,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (!File.Exists(path))
|
|
{
|
|
throw new FileNotFoundException($"Required data file not found: '{path}'.", path);
|
|
}
|
|
|
|
List<string> activeLines = [];
|
|
using StreamReader reader = File.OpenText(path);
|
|
|
|
while (await reader.ReadLineAsync(cancellationToken) is { } line)
|
|
{
|
|
string trimmedLine = line.Trim();
|
|
if (trimmedLine.Length > 0 && !trimmedLine.StartsWith('#'))
|
|
{
|
|
activeLines.Add(trimmedLine);
|
|
}
|
|
}
|
|
|
|
if (activeLines.Count == 0)
|
|
{
|
|
throw new InvalidDataException($"Data file contains no active lines: '{path}'.");
|
|
}
|
|
|
|
return new ReadOnlyCollection<string>(activeLines);
|
|
}
|
|
} |