from __future__ import annotations import json from pathlib import Path import torch from peft import PeftModel from transformers import ( AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, StoppingCriteriaList, ) from json_stopping import JsonObjectStoppingCriteria PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja" SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt" def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) model_path = config["model"]["base_model"] adapter_path = ( PROJECT_ROOT / config["training"]["full"]["output_dir"] ) development_path = ( PROJECT_ROOT / config["paths"]["development"] ) system_prompt = SYSTEM_PROMPT_PATH.read_text( encoding="utf-8" ).strip() records = [] with development_path.open("r", encoding="utf-8") as file: for line in file: records.append(json.loads(line)) if len(records) == 5: break print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained( model_path, use_fast=True, ) tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text( encoding="utf-8" ) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token tokenizer.padding_side = "left" print("Loading model and adapter...") base_model = AutoModelForCausalLM.from_pretrained( model_path, quantization_config=BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, ), dtype=torch.bfloat16, device_map={"": 0}, low_cpu_mem_usage=True, ) model = PeftModel.from_pretrained( base_model, adapter_path, ) model.eval() model.config.use_cache = True prompts = [ tokenizer.apply_chat_template( [ { "role": "system", "content": system_prompt, }, { "role": "user", "content": record["messages"][1]["content"], }, ], tokenize=False, add_generation_prompt=True, ) for record in records ] encoded = tokenizer( prompts, return_tensors="pt", padding=True, truncation=True, max_length=int(config["model"]["max_sequence_length"]), ) encoded = { name: tensor.to(model.device) for name, tensor in encoded.items() } prompt_length = encoded["input_ids"].shape[1] stopping_criteria = StoppingCriteriaList( [ JsonObjectStoppingCriteria( tokenizer, prompt_length, ) ] ) with torch.inference_mode(): generated = model.generate( **encoded, do_sample=False, max_new_tokens=256, stopping_criteria=stopping_criteria, pad_token_id=tokenizer.pad_token_id, use_cache=True, ) generated_tokens = generated[:, prompt_length:] outputs = tokenizer.batch_decode( generated_tokens, skip_special_tokens=True, clean_up_tokenization_spaces=False, ) for index, (record, output) in enumerate( zip(records, outputs), start=1, ): output = output.strip() parsed = json.loads(output) print() print(f"=== RECORD {index} ===") print(f"Input: {record['messages'][1]['content']}") print(f"Tokens: {len(generated_tokens[index - 1])}") print(f"Output: {output}") print(f"Valid: {isinstance(parsed, dict)}") print() print("JSON stopping verification passed.") if __name__ == "__main__": main()