added mmd and pdf
This commit is contained in:
parent
42ee4c2605
commit
55065d88ff
|
|
@ -45,13 +45,27 @@ whether the input contains medical PHI and, for positives, its category.
|
||||||
|
|
||||||
The editable draw.io documents are:
|
The editable draw.io documents are:
|
||||||
|
|
||||||
- [Level 1: System context](diagrams/01-system-context.drawio)
|
- Level 1: System context
|
||||||
- [Level 2: Containers](diagrams/02-containers.drawio)
|
([draw.io](diagrams/01-system-context.drawio),
|
||||||
- [Level 3: Components](diagrams/03-generator-components.drawio)
|
[Mermaid](diagrams/01-system-context.mmd),
|
||||||
- [Level 4: Code](diagrams/04-generator-code.drawio)
|
[PDF](diagrams/01-system-context.pdf))
|
||||||
|
- Level 2: Containers
|
||||||
|
([draw.io](diagrams/02-containers.drawio),
|
||||||
|
[Mermaid](diagrams/02-containers.mmd),
|
||||||
|
[PDF](diagrams/02-containers.pdf))
|
||||||
|
- Level 3: Components
|
||||||
|
([draw.io](diagrams/03-generator-components.drawio),
|
||||||
|
[Mermaid](diagrams/03-generator-components.mmd),
|
||||||
|
[PDF](diagrams/03-generator-components.pdf))
|
||||||
|
- Level 4: Code
|
||||||
|
([draw.io](diagrams/04-generator-code.drawio),
|
||||||
|
[Mermaid](diagrams/04-generator-code.mmd),
|
||||||
|
[PDF](diagrams/04-generator-code.pdf))
|
||||||
|
|
||||||
Open the files with diagrams.net, the draw.io desktop application, or the VS
|
Open the files with diagrams.net, the draw.io desktop application, or the VS
|
||||||
Code Draw.io Integration extension.
|
Code Draw.io Integration extension. The `.drawio` files are the authoritative
|
||||||
|
editable diagrams. Mermaid companions provide text-based, diff-friendly views;
|
||||||
|
PDF companions provide portable rendered views.
|
||||||
|
|
||||||
## Documentation conventions
|
## Documentation conventions
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
flowchart LR
|
||||||
|
operator["ML / Data Operator<br/>Curates inputs, runs generation and training,<br/>and reviews metrics"]
|
||||||
|
sources["NIH / WHO / CDC Sources<br/>Owner-reported origins of<br/>medical terminology catalogs"]
|
||||||
|
system["Medical PHI Model System<br/>Generates contextual examples and fine-tunes<br/>an 8B model to classify medical PHI<br/>and emit category-based JSON"]
|
||||||
|
runtime["Model Runtime<br/>Hosts the merged Hugging Face model<br/>or a separately converted GGUF artifact"]
|
||||||
|
app["Application / Reviewer<br/>Submits text and consumes<br/>strict JSON classification results"]
|
||||||
|
presidio["Downstream Presidio Integration<br/>Consumes compatible MEDICAL_* category/entity types<br/>Implementation is outside this repository"]
|
||||||
|
|
||||||
|
operator -->|Operates and reviews| system
|
||||||
|
sources -->|Curated terminology; provenance manifest needed| system
|
||||||
|
system -->|Produces model artifacts| runtime
|
||||||
|
system -.->|Intended category contract| presidio
|
||||||
|
app -->|Inference requests| runtime
|
||||||
|
runtime -->|Strict JSON decisions| app
|
||||||
|
|
||||||
|
classDef person fill:#fff2cc,stroke:#d6b656,color:#000;
|
||||||
|
classDef system fill:#1168bd,stroke:#0b4884,color:#fff;
|
||||||
|
classDef external fill:#d5e8d4,stroke:#82b366,color:#000;
|
||||||
|
classDef source fill:#f8cecc,stroke:#b85450,color:#000;
|
||||||
|
class operator,app person;
|
||||||
|
class system system;
|
||||||
|
class runtime,presidio external;
|
||||||
|
class sources source;
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,31 @@
|
||||||
|
flowchart LR
|
||||||
|
operator["Operator"]
|
||||||
|
lm["LM Studio<br/>OpenAI-compatible API<br/>qwen/qwen3.5-9b normalization"]
|
||||||
|
base["Base Model<br/>Local Llama3-OpenBioLLM-8B"]
|
||||||
|
|
||||||
|
subgraph repository["Medical PHI Model Repository"]
|
||||||
|
catalogs["Catalogs and Patterns<br/>Text files<br/>12 categories, variants, subjects,<br/>positive/negative templates"]
|
||||||
|
api["Generator API<br/>ASP.NET Core / .NET 10<br/>Validates, plans, normalizes,<br/>and persists ordered batches"]
|
||||||
|
sql[("Training Database<br/>SQL Server<br/>Categories, keywords,<br/>generation runs, training items")]
|
||||||
|
jsonl[("Training Data<br/>JSONL<br/>Source, train, validation,<br/>development")]
|
||||||
|
pipeline["Training Pipeline<br/>Python 3.12 / PyTorch / TRL / PEFT<br/>Validate, split, SFT, train,<br/>evaluate, merge"]
|
||||||
|
artifacts[("Model Artifacts<br/>LoRA adapters, metrics,<br/>merged BF16 model, GGUF files")]
|
||||||
|
|
||||||
|
catalogs -->|Reads| api
|
||||||
|
api -->|Persists and checkpoints| sql
|
||||||
|
sql -->|PowerShell export| jsonl
|
||||||
|
jsonl -->|Reads and rewrites| pipeline
|
||||||
|
pipeline -->|Writes| artifacts
|
||||||
|
end
|
||||||
|
|
||||||
|
operator -->|Runs| api
|
||||||
|
operator -->|Operates and reviews| pipeline
|
||||||
|
api -->|Normalizes text| lm
|
||||||
|
base -->|Weights and tokenizer| pipeline
|
||||||
|
|
||||||
|
classDef person fill:#fff2cc,stroke:#d6b656,color:#000;
|
||||||
|
classDef container fill:#438dd5,stroke:#0b4884,color:#fff;
|
||||||
|
classDef external fill:#999,stroke:#666,color:#fff;
|
||||||
|
class operator person;
|
||||||
|
class catalogs,api,sql,jsonl,pipeline,artifacts container;
|
||||||
|
class lm,base external;
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,37 @@
|
||||||
|
flowchart LR
|
||||||
|
data[("Catalog and pattern files")]
|
||||||
|
lm["LM Studio API"]
|
||||||
|
sql[("SQL Server")]
|
||||||
|
|
||||||
|
subgraph generator["RLHF.Generator.Api + RLHF.Generator"]
|
||||||
|
routes["HTTP Endpoints<br/>Program.cs<br/>Validation, contracts,<br/>status mapping"]
|
||||||
|
reader["TrainingDataReader<br/>Validates and parses subjects,<br/>categories, and patterns"]
|
||||||
|
generation["TrainingGenerationService<br/>Plans, fingerprints, resumes,<br/>parallelizes, orders, validates,<br/>and checkpoints"]
|
||||||
|
lmclient["LmStudioClient<br/>Local token resolution,<br/>prompt construction,<br/>HTTP completion"]
|
||||||
|
keyword["Keyword Services<br/>Load canonical values/variants;<br/>dry-run or apply unique backfill"]
|
||||||
|
reset["TrainingItemResetService<br/>Clears generated rows<br/>and run checkpoints"]
|
||||||
|
db["RlhfGeneratorDbContext<br/>EF Core mappings, seeds, indexes,<br/>relationships, and transactions"]
|
||||||
|
options["Options<br/>GenerationOptions<br/>LmStudioOptions<br/>SentenceTenses"]
|
||||||
|
|
||||||
|
routes -->|Dispatches| generation
|
||||||
|
routes -->|Dispatches| keyword
|
||||||
|
routes -->|Dispatches| reset
|
||||||
|
generation -->|Reads definitions| reader
|
||||||
|
generation -->|Normalizes| lmclient
|
||||||
|
generation -->|Persists batches| db
|
||||||
|
keyword -->|Uses| db
|
||||||
|
reset -->|Uses| db
|
||||||
|
options -->|Configures| generation
|
||||||
|
options -->|Configures| lmclient
|
||||||
|
end
|
||||||
|
|
||||||
|
data -->|Reads| reader
|
||||||
|
lmclient -->|HTTP chat completion| lm
|
||||||
|
db -->|EF Core SQL| sql
|
||||||
|
|
||||||
|
classDef component fill:#85bbf0,stroke:#0b4884,color:#000;
|
||||||
|
classDef primary fill:#1168bd,stroke:#0b4884,color:#fff;
|
||||||
|
classDef external fill:#e1d5e7,stroke:#9673a6,color:#000;
|
||||||
|
class routes,reader,lmclient,keyword,reset,db,options component;
|
||||||
|
class generation primary;
|
||||||
|
class data,lm,sql external;
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,35 @@
|
||||||
|
flowchart TB
|
||||||
|
reader["TrainingDataReader<br/>+ ValidateAsync()<br/>+ ReadCategoryValuesAsync()<br/>+ ReadPatternsAsync()"]
|
||||||
|
definitions["Training definitions<br/>TrainingCategories<br/>TrainingSubjects<br/>TrainingCategoryDefinition<br/>TrainingCategoryValue"]
|
||||||
|
channel["Bounded Channel<br/>Parallel workers write completed plan items;<br/>ordered writer consumes"]
|
||||||
|
service["TrainingGenerationService<br/>+ GenerateAsync()<br/>- GenerateCategoryAsync()<br/>- CreatePlan()<br/>- GeneratePassAsync()<br/>- PersistOrderedAsync()<br/>- FindResumableRunAsync()<br/>- ValidateGeneratedValue()"]
|
||||||
|
lmclient["LmStudioClient<br/>+ CompleteAsync()<br/>- ResolveFragmentTokens()<br/>- ResolveArticles()<br/>- SendCompletionAsync()"]
|
||||||
|
config["Configuration<br/>GenerationOptions<br/>LmStudioOptions<br/>SentenceTenses"]
|
||||||
|
context["RlhfGeneratorDbContext<br/>DbSet Category<br/>DbSet TrainingItem<br/>DbSet GenerationRun<br/>DbSet Keyword<br/>DbSet KeywordVariant"]
|
||||||
|
run["GenerationRun<br/>Status, seed, fingerprints,<br/>checkpoint and counts,<br/>failure/cancellation data"]
|
||||||
|
item["TrainingItem<br/>RecordKey, PlanOrdinal,<br/>Category, Keyword,<br/>IsPositive, Value"]
|
||||||
|
|
||||||
|
service -->|Validates and reads inputs| reader
|
||||||
|
reader -->|Returns| definitions
|
||||||
|
service -->|Plans with| definitions
|
||||||
|
service -->|Coordinates producers and consumer| channel
|
||||||
|
service -->|Calls concurrently| lmclient
|
||||||
|
config -->|Binds| service
|
||||||
|
config -->|Binds| lmclient
|
||||||
|
service -->|Queries and transacts| context
|
||||||
|
context -->|Tracks| run
|
||||||
|
context -->|Tracks| item
|
||||||
|
run -->|Owns ordered items| item
|
||||||
|
|
||||||
|
classDef core fill:#dae8fc,stroke:#6c8ebf,color:#000;
|
||||||
|
classDef input fill:#d5e8d4,stroke:#82b366,color:#000;
|
||||||
|
classDef integration fill:#e1d5e7,stroke:#9673a6,color:#000;
|
||||||
|
classDef persistence fill:#f8cecc,stroke:#b85450,color:#000;
|
||||||
|
classDef config fill:#fff2cc,stroke:#d6b656,color:#000;
|
||||||
|
classDef coordination fill:#ffe6cc,stroke:#d79b00,color:#000;
|
||||||
|
class service core;
|
||||||
|
class reader,definitions input;
|
||||||
|
class lmclient integration;
|
||||||
|
class context,run,item persistence;
|
||||||
|
class config config;
|
||||||
|
class channel coordination;
|
||||||
Binary file not shown.
Loading…
Reference in New Issue