import os import os import pathlib import random import string import tempfile from typing import Iterable, List import gradio as gr import huggingface_hub import spaces import torch import yaml from log_runner import Log, LogsViewRunner, format_logs from huggingface_hub import ModelCard from mergekit.config import MergeConfiguration from model_card import generate_model_card MARKDOWN_HEADER = """ # Model Merger Interactive model merging powered by [MergeKit](https://github.com/arcee-ai/mergekit). Configure a merge using the **Quick Builder** or paste/edit a YAML config in **Advanced** mode, then click **Merge & Upload** to run the merge and publish the result to your Hugging Face account. """ MARKDOWN_DOCS = """ --- ## Merge Methods | Method | `merge_method` | Multi-Model | Uses base model | |--------|----------------|-------------|-----------------| | Linear | `linear` | Yes | No | | SLERP | `slerp` | No (2 models) | Yes | | Task Arithmetic | `task_arithmetic` | Yes | Yes | | TIES | `ties` | Yes | Yes | | DARE-TIES | `dare_ties` | Yes | Yes | | DARE Linear | `dare_linear` | Yes | Yes | See the [MergeKit docs](https://github.com/arcee-ai/mergekit) for full configuration options. ## Citation ```bibtex @article{goddard2024arcee, title={Arcee's MergeKit: A Toolkit for Merging Large Language Models}, author={Goddard, Charles and Siriwardhana, Shamane and Ehghaghi, Malikeh and Meyers, Luke and Karpukhin, Vlad and Benedict, Brian and McQuade, Mark and Solawetz, Jacob}, journal={arXiv preprint arXiv:2403.13257}, year={2024} } ``` """ DEFAULT_SLERP_YAML = pathlib.Path("examples/slerp.yaml").read_text(encoding="utf-8") EXAMPLES = [[str(f)] for f in sorted(pathlib.Path("examples").glob("*.yaml"))] def build_slerp_yaml( model_a: str, model_b: str, base_model: str, num_layers: int, t_default: float, dtype: str, ) -> str: """Build a SLERP merge YAML from form inputs.""" layer_range = [0, num_layers] attn_values = [0, 0.5, 0.3, 0.7, 1] mlp_values = [1, 0.5, 0.7, 0.3, 0] config = { "slices": [ { "sources": [ {"model": model_a.strip(), "layer_range": layer_range}, {"model": model_b.strip(), "layer_range": layer_range}, ] } ], "merge_method": "slerp", "base_model": base_model.strip() or model_a.strip(), "parameters": { "t": [ {"filter": "self_attn", "value": attn_values}, {"filter": "mlp", "value": mlp_values}, {"value": t_default}, ] }, "dtype": dtype, } return yaml.dump(config, default_flow_style=False, sort_keys=False) def update_yaml_from_builder( model_a, model_b, base_model, num_layers, t_default, dtype ) -> str: return build_slerp_yaml(model_a, model_b, base_model, num_layers, t_default, dtype) def load_example(path: str) -> str: return pathlib.Path(path).read_text(encoding="utf-8") def prefetch_models( runner: LogsViewRunner, merge_config: MergeConfiguration, hf_home: str, lora_merge_cache: str, ): for model in merge_config.referenced_models(): yield runner.log(f"Downloading {model}...") merged = model.merged(cache_dir=lora_merge_cache, trust_remote_code=False) local_path = merged.local_path(cache_dir=hf_home) yield runner.log(f" -> {local_path}") def run_merge_cpu( runner: LogsViewRunner, cli: str, merged_path: str, tmpdirname: str ): tmp_env = os.environ.copy() tmp_env["HF_HOME"] = f"{tmpdirname}/.cache" full_cli = ( cli + f" --lora-merge-cache {tmpdirname}/.lora_cache" + f" --transformers-cache {tmpdirname}/.cache" ) yield from runner.run_command(full_cli.split(), cwd=merged_path, env=tmp_env) yield ("done", runner.exit_code) @spaces.GPU(duration=60 * 10) def run_merge_gpu( runner: LogsViewRunner, cli: str, merged_path: str, tmpdirname: str ): yield from run_merge_cpu( runner, cli + " --cuda --low-cpu-memory --read-to-gpu", merged_path, tmpdirname, ) def run_merge( runner: LogsViewRunner, cli: str, merged_path: str, tmpdirname: str, use_gpu: bool, ): if use_gpu: yield from run_merge_gpu(runner, cli, merged_path, tmpdirname) else: yield from run_merge_cpu(runner, cli, merged_path, tmpdirname) def merge_and_upload( yaml_config: str, model_name: str, hf_token: str, username: str, license: str, runtime: str, trust_remote_code: bool, private: bool, ) -> Iterable[List[Log]]: runner = LogsViewRunner() if not yaml_config.strip(): yield runner.log("Please provide a merge configuration.", level="ERROR") return if not hf_token.strip(): yield runner.log( "A Hugging Face write token is required. Create one at https://huggingface.co/settings/tokens", level="ERROR", ) return if not model_name.strip(): yield runner.log("Please provide a name for the merged model.", level="ERROR") return if not username.strip(): yield runner.log("Please provide your Hugging Face username.", level="ERROR") return try: merge_config = MergeConfiguration.model_validate(yaml.safe_load(yaml_config)) except Exception as exc: yield runner.log(f"Invalid YAML configuration: {exc}", level="ERROR") return api = huggingface_hub.HfApi(token=hf_token) repo_id = f"{username.strip()}/{model_name.strip()}" has_gpu = torch.cuda.is_available() and runtime == "GPU" cli = "mergekit-yaml config.yaml merge --copy-tokenizer --allow-crimes -v" if not has_gpu: cli += " --out-shard-size 1B --lazy-unpickle" if trust_remote_code: cli += " --trust-remote-code" yield runner.log(f"Runtime: {'GPU' if has_gpu else 'CPU'}") yield runner.log(f"CLI: {cli}") with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdirname: tmpdir = pathlib.Path(tmpdirname) merged_path = tmpdir / "merged" merged_path.mkdir(parents=True, exist_ok=True) (merged_path / "config.yaml").write_text(yaml_config, encoding="utf-8") try: yield runner.log(f"Creating repository {repo_id}...") repo_url = api.create_repo( repo_id, exist_ok=True, private=private, repo_type="model" ) yield runner.log(f"Repository ready: {repo_url}") except Exception as exc: yield runner.log(f"Failed to create repository: {exc}", level="ERROR") return yield runner.log("Prefetching source models...") yield from prefetch_models( runner, merge_config, hf_home=f"{tmpdirname}/.cache", lora_merge_cache=f"{tmpdirname}/.lora_cache", ) yield runner.log("Starting merge...") exit_code = None try: for event in run_merge( runner, cli, merged_path, tmpdirname, use_gpu=has_gpu ): if isinstance(event, tuple) and event[0] == "done": exit_code = event[1] continue yield event except Exception as exc: yield runner.log(f"Merge error: {exc}", level="ERROR") yield runner.log("Cleaning up empty repository...", level="ERROR") try: api.delete_repo(repo_id) except Exception: pass return if exit_code != 0: yield runner.log(f"Merge failed with exit code {exit_code}", level="ERROR") try: api.delete_repo(repo_id) except Exception: pass return yield runner.log("Merge complete. Generating model card...") card_content = generate_model_card( model_name=model_name.strip(), yaml_config=yaml_config, username=username.strip(), license=license, ) card = ModelCard(card_content) merge_output = merged_path / "merge" card.save(str(merge_output / "README.md")) yield runner.log(f"Uploading to {repo_id}...") yield from runner.run_python( api.upload_folder, repo_id=repo_id, folder_path=str(merge_output), ) yield runner.log(f"Success! Model uploaded to https://huggingface.co/{repo_id}") def merge_and_upload_text(*args) -> Iterable[str]: accumulated: list[str] = [] for batch in merge_and_upload(*args): accumulated.append(format_logs(batch)) yield "".join(accumulated) merge_and_upload.zerogpu = True run_merge.zerogpu = True with gr.Blocks(theme=gr.themes.Soft()) as demo: gr.Markdown(MARKDOWN_HEADER) with gr.Tabs(): with gr.Tab("Quick Builder (SLERP)"): gr.Markdown( "Build a SLERP merge between two models. The generated YAML is editable before merging." ) with gr.Row(): model_a = gr.Textbox( label="Model A", value="OpenPipe/mistral-ft-optimized-1218", placeholder="e.g. OpenPipe/mistral-ft-optimized-1218", ) model_b = gr.Textbox( label="Model B", value="mlabonne/NeuralHermes-2.5-Mistral-7B", placeholder="e.g. mlabonne/NeuralHermes-2.5-Mistral-7B", ) with gr.Row(): base_model = gr.Textbox( label="Base model", value="OpenPipe/mistral-ft-optimized-1218", info="Usually Model A for SLERP merges", ) num_layers = gr.Slider( label="Number of layers", minimum=1, maximum=80, step=1, value=32, ) with gr.Row(): t_default = gr.Slider( label="Default interpolation (t)", minimum=0.0, maximum=1.0, step=0.05, value=0.5, ) dtype = gr.Dropdown( label="dtype", choices=["bfloat16", "float16", "float32"], value="bfloat16", ) build_btn = gr.Button("Generate YAML", variant="secondary") with gr.Tab("Advanced (YAML)"): gr.Markdown("Paste or edit a full MergeKit YAML configuration.") yaml_config = gr.Code( language="yaml", lines=18, label="Merge configuration (config.yaml)", value=DEFAULT_SLERP_YAML, ) example_path = gr.Textbox(visible=False) gr.Examples( examples=EXAMPLES, fn=load_example, run_on_click=True, cache_examples=False, label="Example configurations", inputs=[example_path], outputs=[yaml_config], ) gr.Markdown("### Upload settings") with gr.Row(): model_name = gr.Textbox( label="Merged model name", placeholder="NeuralPipe-7B-slerp", value="NeuralPipe-7B-slerp", ) username = gr.Textbox( label="Hugging Face username", placeholder="your-username", value="salihfurkaan", ) with gr.Row(): hf_token = gr.Textbox( label="Hugging Face write token", type="password", info="Needs write access. https://huggingface.co/settings/tokens", ) license_choice = gr.Dropdown( label="License", choices=["apache-2.0", "cc-by-nc-4.0", "mit", "openrail"], value="apache-2.0", allow_custom_value=True, ) with gr.Row(): runtime = gr.Radio( label="Runtime", choices=["CPU", "GPU"], value="GPU", info="GPU uses Zero GPU when available; CPU uses lazy-unpickle for low memory.", ) trust_remote_code = gr.Checkbox( label="Trust remote code", value=False, ) private = gr.Checkbox( label="Private repository", value=False, ) merge_btn = gr.Button("Merge & Upload", variant="primary", size="lg") logs = gr.Textbox( label="Terminal output", lines=20, max_lines=50, interactive=False, show_copy_button=True, ) build_btn.click( fn=update_yaml_from_builder, inputs=[model_a, model_b, base_model, num_layers, t_default, dtype], outputs=yaml_config, ) merge_btn.click( fn=merge_and_upload_text, inputs=[ yaml_config, model_name, hf_token, username, license_choice, runtime, trust_remote_code, private, ], outputs=logs, ) gr.Markdown(MARKDOWN_DOCS) os.environ.setdefault("GRADIO_SERVER_NAME", "0.0.0.0") demo.queue(default_concurrency_limit=1).launch( server_name="0.0.0.0", server_port=int(os.getenv("PORT", "7860")), )