PHI-Model-8B/scripts/verify_generation_stop.py

169 lines
4.0 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
import torch
from peft import PeftModel
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
BitsAndBytesConfig,
StoppingCriteriaList,
)
from json_stopping import JsonObjectStoppingCriteria
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja"
SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt"
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
model_path = config["model"]["base_model"]
adapter_path = (
PROJECT_ROOT
/ config["training"]["full"]["output_dir"]
)
development_path = (
PROJECT_ROOT
/ config["paths"]["development"]
)
system_prompt = SYSTEM_PROMPT_PATH.read_text(
encoding="utf-8"
).strip()
records = []
with development_path.open("r", encoding="utf-8") as file:
for line in file:
records.append(json.loads(line))
if len(records) == 5:
break
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
model_path,
use_fast=True,
)
tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text(
encoding="utf-8"
)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"
print("Loading model and adapter...")
base_model = AutoModelForCausalLM.from_pretrained(
model_path,
quantization_config=BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16,
),
dtype=torch.bfloat16,
device_map={"": 0},
low_cpu_mem_usage=True,
)
model = PeftModel.from_pretrained(
base_model,
adapter_path,
)
model.eval()
model.config.use_cache = True
prompts = [
tokenizer.apply_chat_template(
[
{
"role": "system",
"content": system_prompt,
},
{
"role": "user",
"content": record["messages"][1]["content"],
},
],
tokenize=False,
add_generation_prompt=True,
)
for record in records
]
encoded = tokenizer(
prompts,
return_tensors="pt",
padding=True,
truncation=True,
max_length=int(config["model"]["max_sequence_length"]),
)
encoded = {
name: tensor.to(model.device)
for name, tensor in encoded.items()
}
prompt_length = encoded["input_ids"].shape[1]
stopping_criteria = StoppingCriteriaList(
[
JsonObjectStoppingCriteria(
tokenizer,
prompt_length,
)
]
)
with torch.inference_mode():
generated = model.generate(
**encoded,
do_sample=False,
max_new_tokens=256,
stopping_criteria=stopping_criteria,
pad_token_id=tokenizer.pad_token_id,
use_cache=True,
)
generated_tokens = generated[:, prompt_length:]
outputs = tokenizer.batch_decode(
generated_tokens,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
)
for index, (record, output) in enumerate(
zip(records, outputs),
start=1,
):
output = output.strip()
parsed = json.loads(output)
print()
print(f"=== RECORD {index} ===")
print(f"Input: {record['messages'][1]['content']}")
print(f"Tokens: {len(generated_tokens[index - 1])}")
print(f"Output: {output}")
print(f"Valid: {isinstance(parsed, dict)}")
print()
print("JSON stopping verification passed.")
if __name__ == "__main__":
main()