169 lines
4.0 KiB
Python
169 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
from peft import PeftModel
|
|
from transformers import (
|
|
AutoModelForCausalLM,
|
|
AutoTokenizer,
|
|
BitsAndBytesConfig,
|
|
StoppingCriteriaList,
|
|
)
|
|
|
|
from json_stopping import JsonObjectStoppingCriteria
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja"
|
|
SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt"
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
|
|
model_path = config["model"]["base_model"]
|
|
adapter_path = (
|
|
PROJECT_ROOT
|
|
/ config["training"]["full"]["output_dir"]
|
|
)
|
|
development_path = (
|
|
PROJECT_ROOT
|
|
/ config["paths"]["development"]
|
|
)
|
|
|
|
system_prompt = SYSTEM_PROMPT_PATH.read_text(
|
|
encoding="utf-8"
|
|
).strip()
|
|
|
|
records = []
|
|
|
|
with development_path.open("r", encoding="utf-8") as file:
|
|
for line in file:
|
|
records.append(json.loads(line))
|
|
|
|
if len(records) == 5:
|
|
break
|
|
|
|
print("Loading tokenizer...")
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
model_path,
|
|
use_fast=True,
|
|
)
|
|
|
|
tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
if tokenizer.pad_token_id is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
|
|
tokenizer.padding_side = "left"
|
|
|
|
print("Loading model and adapter...")
|
|
|
|
base_model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_use_double_quant=True,
|
|
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
),
|
|
dtype=torch.bfloat16,
|
|
device_map={"": 0},
|
|
low_cpu_mem_usage=True,
|
|
)
|
|
|
|
model = PeftModel.from_pretrained(
|
|
base_model,
|
|
adapter_path,
|
|
)
|
|
|
|
model.eval()
|
|
model.config.use_cache = True
|
|
|
|
prompts = [
|
|
tokenizer.apply_chat_template(
|
|
[
|
|
{
|
|
"role": "system",
|
|
"content": system_prompt,
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": record["messages"][1]["content"],
|
|
},
|
|
],
|
|
tokenize=False,
|
|
add_generation_prompt=True,
|
|
)
|
|
for record in records
|
|
]
|
|
|
|
encoded = tokenizer(
|
|
prompts,
|
|
return_tensors="pt",
|
|
padding=True,
|
|
truncation=True,
|
|
max_length=int(config["model"]["max_sequence_length"]),
|
|
)
|
|
|
|
encoded = {
|
|
name: tensor.to(model.device)
|
|
for name, tensor in encoded.items()
|
|
}
|
|
|
|
prompt_length = encoded["input_ids"].shape[1]
|
|
|
|
stopping_criteria = StoppingCriteriaList(
|
|
[
|
|
JsonObjectStoppingCriteria(
|
|
tokenizer,
|
|
prompt_length,
|
|
)
|
|
]
|
|
)
|
|
|
|
with torch.inference_mode():
|
|
generated = model.generate(
|
|
**encoded,
|
|
do_sample=False,
|
|
max_new_tokens=256,
|
|
stopping_criteria=stopping_criteria,
|
|
pad_token_id=tokenizer.pad_token_id,
|
|
use_cache=True,
|
|
)
|
|
|
|
generated_tokens = generated[:, prompt_length:]
|
|
|
|
outputs = tokenizer.batch_decode(
|
|
generated_tokens,
|
|
skip_special_tokens=True,
|
|
clean_up_tokenization_spaces=False,
|
|
)
|
|
|
|
for index, (record, output) in enumerate(
|
|
zip(records, outputs),
|
|
start=1,
|
|
):
|
|
output = output.strip()
|
|
parsed = json.loads(output)
|
|
|
|
print()
|
|
print(f"=== RECORD {index} ===")
|
|
print(f"Input: {record['messages'][1]['content']}")
|
|
print(f"Tokens: {len(generated_tokens[index - 1])}")
|
|
print(f"Output: {output}")
|
|
print(f"Valid: {isinstance(parsed, dict)}")
|
|
|
|
print()
|
|
print("JSON stopping verification passed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|