Models-Phi-8B/scaffolding/03-export-sql.ps1

81 lines
2.0 KiB
PowerShell

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$ProjectRoot = Split-Path -Parent $PSScriptRoot
$ConfigPath = Join-Path $ProjectRoot "config\pipeline.json"
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
$outputPath = Join-Path $ProjectRoot $config.paths.source_export
$builder = New-Object System.Data.SqlClient.SqlConnectionStringBuilder
$builder["Data Source"] = $config.sql.server
$builder["Initial Catalog"] = $config.sql.database
$builder["Integrated Security"] = $true
$builder["Encrypt"] = [bool]$config.sql.encrypt
$builder["TrustServerCertificate"] = [bool]$config.sql.trust_server_certificate
$builder["Application Name"] = "OpenBioLLM PHI Export"
$query = @"
SELECT
ti.Id,
c.Name AS CategoryName,
ti.IsPositive,
ti.Value
FROM dbo.TrainingItem AS ti
INNER JOIN dbo.Category AS c
ON c.Id = ti.CategoryId
ORDER BY ti.Id;
"@
$connection = New-Object System.Data.SqlClient.SqlConnection $builder.ConnectionString
$command = $connection.CreateCommand()
$command.CommandText = $query
$command.CommandTimeout = 120
$utf8 = New-Object System.Text.UTF8Encoding($false)
$writer = New-Object System.IO.StreamWriter($outputPath, $false, $utf8)
$total = 0
$positive = 0
$negative = 0
try {
$connection.Open()
$reader = $command.ExecuteReader()
while ($reader.Read()) {
$isPositive = [bool]$reader["IsPositive"]
$record = [ordered]@{
id = [int64]$reader["Id"]
category = [string]$reader["CategoryName"]
is_positive = $isPositive
value = [string]$reader["Value"]
}
$writer.WriteLine(($record | ConvertTo-Json -Compress))
$total++
if ($isPositive) {
$positive++
}
else {
$negative++
}
}
$reader.Close()
}
finally {
$writer.Dispose()
$connection.Dispose()
}
Write-Host ""
Write-Host "Export complete."
Write-Host "Output: $outputPath"
Write-Host "Total: $total"
Write-Host "Positive: $positive"
Write-Host "Negative: $negative"