from __future__ import annotations import json import shutil from pathlib import Path import torch from peft import PeftModel from transformers import AutoModelForCausalLM, AutoTokenizer PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja" SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt" def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) base_model_path = Path(config["model"]["base_model"]) adapter_path = ( PROJECT_ROOT / config["training"]["full"]["output_dir"] ) output_path = PROJECT_ROOT / config["merge"]["output_dir"] temporary_path = output_path.with_name( output_path.name + ".tmp" ) if temporary_path.exists(): shutil.rmtree(temporary_path) temporary_path.mkdir(parents=True) print(f"Base model: {base_model_path}") print(f"Adapter: {adapter_path}") print(f"Output: {output_path}") print() print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained( base_model_path, use_fast=True, ) tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text( encoding="utf-8" ) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token print("Loading unquantized base model in BF16 on CPU...") base_model = AutoModelForCausalLM.from_pretrained( base_model_path, dtype=torch.bfloat16, device_map={"": "cpu"}, low_cpu_mem_usage=True, ) print("Loading LoRA adapter...") peft_model = PeftModel.from_pretrained( base_model, adapter_path, ) print("Merging adapter into base model...") merged_model = peft_model.merge_and_unload() merged_model.config.use_cache = True merged_model.config.pad_token_id = tokenizer.pad_token_id if merged_model.generation_config is not None: merged_model.generation_config.pad_token_id = ( tokenizer.pad_token_id ) print("Saving standalone merged model...") merged_model.save_pretrained( temporary_path, safe_serialization=True, max_shard_size="5GB", ) tokenizer.save_pretrained(temporary_path) shutil.copy2( SYSTEM_PROMPT_PATH, temporary_path / "system_prompt.txt", ) weight_files = list( temporary_path.glob("*.safetensors") ) if not weight_files: raise RuntimeError( "No merged model weight files were saved." ) if not (temporary_path / "config.json").exists(): raise RuntimeError( "Merged model config.json was not saved." ) if output_path.exists(): shutil.rmtree(output_path) temporary_path.rename(output_path) total_bytes = sum( file_path.stat().st_size for file_path in output_path.rglob("*") if file_path.is_file() ) print() print("Standalone model merge passed.") print(f"Output: {output_path}") print(f"Size: {total_bytes / 1024**3:.2f} GiB") print(f"Weight files: {len(weight_files)}") if __name__ == "__main__": main()