Models-Phi-8B/scripts/model_smoke_test.py

106 lines
2.8 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
import torch
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
BitsAndBytesConfig,
)
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
model_path = config["model"]["base_model"]
max_length = int(config["model"]["max_sequence_length"])
train_path = PROJECT_ROOT / config["paths"]["train"]
with train_path.open("r", encoding="utf-8") as file:
sample = json.loads(file.readline())
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
model_path,
use_fast=True,
)
if not tokenizer.chat_template:
template_path = PROJECT_ROOT / "config" / "chat_template.jinja"
tokenizer.chat_template = template_path.read_text(
encoding="utf-8"
)
print(f"Using project chat template: {template_path}")
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
rendered = tokenizer.apply_chat_template(
sample["messages"],
tokenize=False,
add_generation_prompt=False,
)
encoded = tokenizer(
rendered,
return_tensors="pt",
truncation=True,
max_length=max_length,
)
print(f"Sample ID: {sample['id']}")
print(f"Sample tokens: {encoded['input_ids'].shape[1]}")
print("Loading model in 4-bit NF4...")
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16,
)
model = AutoModelForCausalLM.from_pretrained(
model_path,
quantization_config=quantization_config,
dtype=torch.bfloat16,
device_map="auto",
low_cpu_mem_usage=True,
)
model.config.use_cache = False
model.eval()
device = next(model.parameters()).device
encoded = {
name: tensor.to(device)
for name, tensor in encoded.items()
}
print("Running one forward pass...")
with torch.no_grad():
output = model(**encoded)
if not torch.isfinite(output.logits).all():
raise RuntimeError("Model returned non-finite logits.")
allocated = torch.cuda.memory_allocated() / 1024**3
reserved = torch.cuda.memory_reserved() / 1024**3
print()
print("Model smoke test passed.")
print(f"Model class: {type(model).__name__}")
print(f"Device: {device}")
print(f"Logits shape: {tuple(output.logits.shape)}")
print(f"GPU allocated: {allocated:.2f} GiB")
print(f"GPU reserved: {reserved:.2f} GiB")
if __name__ == "__main__":
main()