diff --git a/docs/README.md b/docs/README.md index 419b6c3..3f6885a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,13 +45,27 @@ whether the input contains medical PHI and, for positives, its category. The editable draw.io documents are: -- [Level 1: System context](diagrams/01-system-context.drawio) -- [Level 2: Containers](diagrams/02-containers.drawio) -- [Level 3: Components](diagrams/03-generator-components.drawio) -- [Level 4: Code](diagrams/04-generator-code.drawio) +- Level 1: System context + ([draw.io](diagrams/01-system-context.drawio), + [Mermaid](diagrams/01-system-context.mmd), + [PDF](diagrams/01-system-context.pdf)) +- Level 2: Containers + ([draw.io](diagrams/02-containers.drawio), + [Mermaid](diagrams/02-containers.mmd), + [PDF](diagrams/02-containers.pdf)) +- Level 3: Components + ([draw.io](diagrams/03-generator-components.drawio), + [Mermaid](diagrams/03-generator-components.mmd), + [PDF](diagrams/03-generator-components.pdf)) +- Level 4: Code + ([draw.io](diagrams/04-generator-code.drawio), + [Mermaid](diagrams/04-generator-code.mmd), + [PDF](diagrams/04-generator-code.pdf)) Open the files with diagrams.net, the draw.io desktop application, or the VS -Code Draw.io Integration extension. +Code Draw.io Integration extension. The `.drawio` files are the authoritative +editable diagrams. Mermaid companions provide text-based, diff-friendly views; +PDF companions provide portable rendered views. ## Documentation conventions diff --git a/docs/diagrams/01-system-context.mmd b/docs/diagrams/01-system-context.mmd new file mode 100644 index 0000000..4082abf --- /dev/null +++ b/docs/diagrams/01-system-context.mmd @@ -0,0 +1,23 @@ +flowchart LR + operator["ML / Data Operator
Curates inputs, runs generation and training,
and reviews metrics"] + sources["NIH / WHO / CDC Sources
Owner-reported origins of
medical terminology catalogs"] + system["Medical PHI Model System
Generates contextual examples and fine-tunes
an 8B model to classify medical PHI
and emit category-based JSON"] + runtime["Model Runtime
Hosts the merged Hugging Face model
or a separately converted GGUF artifact"] + app["Application / Reviewer
Submits text and consumes
strict JSON classification results"] + presidio["Downstream Presidio Integration
Consumes compatible MEDICAL_* category/entity types
Implementation is outside this repository"] + + operator -->|Operates and reviews| system + sources -->|Curated terminology; provenance manifest needed| system + system -->|Produces model artifacts| runtime + system -.->|Intended category contract| presidio + app -->|Inference requests| runtime + runtime -->|Strict JSON decisions| app + + classDef person fill:#fff2cc,stroke:#d6b656,color:#000; + classDef system fill:#1168bd,stroke:#0b4884,color:#fff; + classDef external fill:#d5e8d4,stroke:#82b366,color:#000; + classDef source fill:#f8cecc,stroke:#b85450,color:#000; + class operator,app person; + class system system; + class runtime,presidio external; + class sources source; diff --git a/docs/diagrams/01-system-context.pdf b/docs/diagrams/01-system-context.pdf new file mode 100644 index 0000000..1116781 Binary files /dev/null and b/docs/diagrams/01-system-context.pdf differ diff --git a/docs/diagrams/02-containers.mmd b/docs/diagrams/02-containers.mmd new file mode 100644 index 0000000..1e04922 --- /dev/null +++ b/docs/diagrams/02-containers.mmd @@ -0,0 +1,31 @@ +flowchart LR + operator["Operator"] + lm["LM Studio
OpenAI-compatible API
qwen/qwen3.5-9b normalization"] + base["Base Model
Local Llama3-OpenBioLLM-8B"] + + subgraph repository["Medical PHI Model Repository"] + catalogs["Catalogs and Patterns
Text files
12 categories, variants, subjects,
positive/negative templates"] + api["Generator API
ASP.NET Core / .NET 10
Validates, plans, normalizes,
and persists ordered batches"] + sql[("Training Database
SQL Server
Categories, keywords,
generation runs, training items")] + jsonl[("Training Data
JSONL
Source, train, validation,
development")] + pipeline["Training Pipeline
Python 3.12 / PyTorch / TRL / PEFT
Validate, split, SFT, train,
evaluate, merge"] + artifacts[("Model Artifacts
LoRA adapters, metrics,
merged BF16 model, GGUF files")] + + catalogs -->|Reads| api + api -->|Persists and checkpoints| sql + sql -->|PowerShell export| jsonl + jsonl -->|Reads and rewrites| pipeline + pipeline -->|Writes| artifacts + end + + operator -->|Runs| api + operator -->|Operates and reviews| pipeline + api -->|Normalizes text| lm + base -->|Weights and tokenizer| pipeline + + classDef person fill:#fff2cc,stroke:#d6b656,color:#000; + classDef container fill:#438dd5,stroke:#0b4884,color:#fff; + classDef external fill:#999,stroke:#666,color:#fff; + class operator person; + class catalogs,api,sql,jsonl,pipeline,artifacts container; + class lm,base external; diff --git a/docs/diagrams/02-containers.pdf b/docs/diagrams/02-containers.pdf new file mode 100644 index 0000000..2696d76 Binary files /dev/null and b/docs/diagrams/02-containers.pdf differ diff --git a/docs/diagrams/03-generator-components.mmd b/docs/diagrams/03-generator-components.mmd new file mode 100644 index 0000000..1229025 --- /dev/null +++ b/docs/diagrams/03-generator-components.mmd @@ -0,0 +1,37 @@ +flowchart LR + data[("Catalog and pattern files")] + lm["LM Studio API"] + sql[("SQL Server")] + + subgraph generator["RLHF.Generator.Api + RLHF.Generator"] + routes["HTTP Endpoints
Program.cs
Validation, contracts,
status mapping"] + reader["TrainingDataReader
Validates and parses subjects,
categories, and patterns"] + generation["TrainingGenerationService
Plans, fingerprints, resumes,
parallelizes, orders, validates,
and checkpoints"] + lmclient["LmStudioClient
Local token resolution,
prompt construction,
HTTP completion"] + keyword["Keyword Services
Load canonical values/variants;
dry-run or apply unique backfill"] + reset["TrainingItemResetService
Clears generated rows
and run checkpoints"] + db["RlhfGeneratorDbContext
EF Core mappings, seeds, indexes,
relationships, and transactions"] + options["Options
GenerationOptions
LmStudioOptions
SentenceTenses"] + + routes -->|Dispatches| generation + routes -->|Dispatches| keyword + routes -->|Dispatches| reset + generation -->|Reads definitions| reader + generation -->|Normalizes| lmclient + generation -->|Persists batches| db + keyword -->|Uses| db + reset -->|Uses| db + options -->|Configures| generation + options -->|Configures| lmclient + end + + data -->|Reads| reader + lmclient -->|HTTP chat completion| lm + db -->|EF Core SQL| sql + + classDef component fill:#85bbf0,stroke:#0b4884,color:#000; + classDef primary fill:#1168bd,stroke:#0b4884,color:#fff; + classDef external fill:#e1d5e7,stroke:#9673a6,color:#000; + class routes,reader,lmclient,keyword,reset,db,options component; + class generation primary; + class data,lm,sql external; diff --git a/docs/diagrams/03-generator-components.pdf b/docs/diagrams/03-generator-components.pdf new file mode 100644 index 0000000..d3bac18 Binary files /dev/null and b/docs/diagrams/03-generator-components.pdf differ diff --git a/docs/diagrams/04-generator-code.mmd b/docs/diagrams/04-generator-code.mmd new file mode 100644 index 0000000..9547af3 --- /dev/null +++ b/docs/diagrams/04-generator-code.mmd @@ -0,0 +1,35 @@ +flowchart TB + reader["TrainingDataReader
+ ValidateAsync()
+ ReadCategoryValuesAsync()
+ ReadPatternsAsync()"] + definitions["Training definitions
TrainingCategories
TrainingSubjects
TrainingCategoryDefinition
TrainingCategoryValue"] + channel["Bounded Channel
Parallel workers write completed plan items;
ordered writer consumes"] + service["TrainingGenerationService
+ GenerateAsync()
- GenerateCategoryAsync()
- CreatePlan()
- GeneratePassAsync()
- PersistOrderedAsync()
- FindResumableRunAsync()
- ValidateGeneratedValue()"] + lmclient["LmStudioClient
+ CompleteAsync()
- ResolveFragmentTokens()
- ResolveArticles()
- SendCompletionAsync()"] + config["Configuration
GenerationOptions
LmStudioOptions
SentenceTenses"] + context["RlhfGeneratorDbContext
DbSet Category
DbSet TrainingItem
DbSet GenerationRun
DbSet Keyword
DbSet KeywordVariant"] + run["GenerationRun
Status, seed, fingerprints,
checkpoint and counts,
failure/cancellation data"] + item["TrainingItem
RecordKey, PlanOrdinal,
Category, Keyword,
IsPositive, Value"] + + service -->|Validates and reads inputs| reader + reader -->|Returns| definitions + service -->|Plans with| definitions + service -->|Coordinates producers and consumer| channel + service -->|Calls concurrently| lmclient + config -->|Binds| service + config -->|Binds| lmclient + service -->|Queries and transacts| context + context -->|Tracks| run + context -->|Tracks| item + run -->|Owns ordered items| item + + classDef core fill:#dae8fc,stroke:#6c8ebf,color:#000; + classDef input fill:#d5e8d4,stroke:#82b366,color:#000; + classDef integration fill:#e1d5e7,stroke:#9673a6,color:#000; + classDef persistence fill:#f8cecc,stroke:#b85450,color:#000; + classDef config fill:#fff2cc,stroke:#d6b656,color:#000; + classDef coordination fill:#ffe6cc,stroke:#d79b00,color:#000; + class service core; + class reader,definitions input; + class lmclient integration; + class context,run,item persistence; + class config config; + class channel coordination; diff --git a/docs/diagrams/04-generator-code.pdf b/docs/diagrams/04-generator-code.pdf new file mode 100644 index 0000000..9a05bd8 Binary files /dev/null and b/docs/diagrams/04-generator-code.pdf differ