131 lines
3.2 KiB
Python
131 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
from peft import PeftModel
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja"
|
|
SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt"
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
|
|
base_model_path = Path(config["model"]["base_model"])
|
|
adapter_path = (
|
|
PROJECT_ROOT
|
|
/ config["training"]["full"]["output_dir"]
|
|
)
|
|
output_path = PROJECT_ROOT / config["merge"]["output_dir"]
|
|
temporary_path = output_path.with_name(
|
|
output_path.name + ".tmp"
|
|
)
|
|
|
|
if temporary_path.exists():
|
|
shutil.rmtree(temporary_path)
|
|
|
|
temporary_path.mkdir(parents=True)
|
|
|
|
print(f"Base model: {base_model_path}")
|
|
print(f"Adapter: {adapter_path}")
|
|
print(f"Output: {output_path}")
|
|
print()
|
|
print("Loading tokenizer...")
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
base_model_path,
|
|
use_fast=True,
|
|
)
|
|
|
|
tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
if tokenizer.pad_token_id is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
|
|
print("Loading unquantized base model in BF16 on CPU...")
|
|
|
|
base_model = AutoModelForCausalLM.from_pretrained(
|
|
base_model_path,
|
|
dtype=torch.bfloat16,
|
|
device_map={"": "cpu"},
|
|
low_cpu_mem_usage=True,
|
|
)
|
|
|
|
print("Loading LoRA adapter...")
|
|
|
|
peft_model = PeftModel.from_pretrained(
|
|
base_model,
|
|
adapter_path,
|
|
)
|
|
|
|
print("Merging adapter into base model...")
|
|
|
|
merged_model = peft_model.merge_and_unload()
|
|
|
|
merged_model.config.use_cache = True
|
|
merged_model.config.pad_token_id = tokenizer.pad_token_id
|
|
|
|
if merged_model.generation_config is not None:
|
|
merged_model.generation_config.pad_token_id = (
|
|
tokenizer.pad_token_id
|
|
)
|
|
|
|
print("Saving standalone merged model...")
|
|
|
|
merged_model.save_pretrained(
|
|
temporary_path,
|
|
safe_serialization=True,
|
|
max_shard_size="5GB",
|
|
)
|
|
|
|
tokenizer.save_pretrained(temporary_path)
|
|
|
|
shutil.copy2(
|
|
SYSTEM_PROMPT_PATH,
|
|
temporary_path / "system_prompt.txt",
|
|
)
|
|
|
|
weight_files = list(
|
|
temporary_path.glob("*.safetensors")
|
|
)
|
|
|
|
if not weight_files:
|
|
raise RuntimeError(
|
|
"No merged model weight files were saved."
|
|
)
|
|
|
|
if not (temporary_path / "config.json").exists():
|
|
raise RuntimeError(
|
|
"Merged model config.json was not saved."
|
|
)
|
|
|
|
if output_path.exists():
|
|
shutil.rmtree(output_path)
|
|
|
|
temporary_path.rename(output_path)
|
|
|
|
total_bytes = sum(
|
|
file_path.stat().st_size
|
|
for file_path in output_path.rglob("*")
|
|
if file_path.is_file()
|
|
)
|
|
|
|
print()
|
|
print("Standalone model merge passed.")
|
|
print(f"Output: {output_path}")
|
|
print(f"Size: {total_bytes / 1024**3:.2f} GiB")
|
|
print(f"Weight files: {len(weight_files)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|