from __future__ import annotations import json from pathlib import Path import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, ) PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) model_path = config["model"]["base_model"] max_length = int(config["model"]["max_sequence_length"]) train_path = PROJECT_ROOT / config["paths"]["train"] with train_path.open("r", encoding="utf-8") as file: sample = json.loads(file.readline()) print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained( model_path, use_fast=True, ) if not tokenizer.chat_template: template_path = PROJECT_ROOT / "config" / "chat_template.jinja" tokenizer.chat_template = template_path.read_text( encoding="utf-8" ) print(f"Using project chat template: {template_path}") if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token rendered = tokenizer.apply_chat_template( sample["messages"], tokenize=False, add_generation_prompt=False, ) encoded = tokenizer( rendered, return_tensors="pt", truncation=True, max_length=max_length, ) print(f"Sample ID: {sample['id']}") print(f"Sample tokens: {encoded['input_ids'].shape[1]}") print("Loading model in 4-bit NF4...") quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, ) model = AutoModelForCausalLM.from_pretrained( model_path, quantization_config=quantization_config, dtype=torch.bfloat16, device_map="auto", low_cpu_mem_usage=True, ) model.config.use_cache = False model.eval() device = next(model.parameters()).device encoded = { name: tensor.to(device) for name, tensor in encoded.items() } print("Running one forward pass...") with torch.no_grad(): output = model(**encoded) if not torch.isfinite(output.logits).all(): raise RuntimeError("Model returned non-finite logits.") allocated = torch.cuda.memory_allocated() / 1024**3 reserved = torch.cuda.memory_reserved() / 1024**3 print() print("Model smoke test passed.") print(f"Model class: {type(model).__name__}") print(f"Device: {device}") print(f"Logits shape: {tuple(output.logits.shape)}") print(f"GPU allocated: {allocated:.2f} GiB") print(f"GPU reserved: {reserved:.2f} GiB") if __name__ == "__main__": main()