106 lines
2.8 KiB
Python
106 lines
2.8 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
from transformers import (
|
|
AutoModelForCausalLM,
|
|
AutoTokenizer,
|
|
BitsAndBytesConfig,
|
|
)
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
model_path = config["model"]["base_model"]
|
|
max_length = int(config["model"]["max_sequence_length"])
|
|
train_path = PROJECT_ROOT / config["paths"]["train"]
|
|
|
|
with train_path.open("r", encoding="utf-8") as file:
|
|
sample = json.loads(file.readline())
|
|
|
|
print("Loading tokenizer...")
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
model_path,
|
|
use_fast=True,
|
|
)
|
|
|
|
if not tokenizer.chat_template:
|
|
template_path = PROJECT_ROOT / "config" / "chat_template.jinja"
|
|
tokenizer.chat_template = template_path.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
print(f"Using project chat template: {template_path}")
|
|
|
|
if tokenizer.pad_token_id is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
|
|
rendered = tokenizer.apply_chat_template(
|
|
sample["messages"],
|
|
tokenize=False,
|
|
add_generation_prompt=False,
|
|
)
|
|
|
|
encoded = tokenizer(
|
|
rendered,
|
|
return_tensors="pt",
|
|
truncation=True,
|
|
max_length=max_length,
|
|
)
|
|
|
|
print(f"Sample ID: {sample['id']}")
|
|
print(f"Sample tokens: {encoded['input_ids'].shape[1]}")
|
|
print("Loading model in 4-bit NF4...")
|
|
|
|
quantization_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_use_double_quant=True,
|
|
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
)
|
|
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=quantization_config,
|
|
dtype=torch.bfloat16,
|
|
device_map="auto",
|
|
low_cpu_mem_usage=True,
|
|
)
|
|
|
|
model.config.use_cache = False
|
|
model.eval()
|
|
|
|
device = next(model.parameters()).device
|
|
encoded = {
|
|
name: tensor.to(device)
|
|
for name, tensor in encoded.items()
|
|
}
|
|
|
|
print("Running one forward pass...")
|
|
|
|
with torch.no_grad():
|
|
output = model(**encoded)
|
|
|
|
if not torch.isfinite(output.logits).all():
|
|
raise RuntimeError("Model returned non-finite logits.")
|
|
|
|
allocated = torch.cuda.memory_allocated() / 1024**3
|
|
reserved = torch.cuda.memory_reserved() / 1024**3
|
|
|
|
print()
|
|
print("Model smoke test passed.")
|
|
print(f"Model class: {type(model).__name__}")
|
|
print(f"Device: {device}")
|
|
print(f"Logits shape: {tuple(output.logits.shape)}")
|
|
print(f"GPU allocated: {allocated:.2f} GiB")
|
|
print(f"GPU reserved: {reserved:.2f} GiB")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|