model-merger / app.py
salihfurkaan's picture
Fix Space launch and disable example caching
7f56f88 verified
Raw History Blame Contribute Delete
13.5 kB
import os
import os
import pathlib
import random
import string
import tempfile
from typing import Iterable, List
import gradio as gr
import huggingface_hub
import spaces
import torch
import yaml
from log_runner import Log, LogsViewRunner, format_logs
from huggingface_hub import ModelCard
from mergekit.config import MergeConfiguration
from model_card import generate_model_card
MARKDOWN_HEADER = """
# Model Merger
Interactive model merging powered by [MergeKit](https://github.com/arcee-ai/mergekit).
Configure a merge using the **Quick Builder** or paste/edit a YAML config in **Advanced** mode, then click **Merge & Upload** to run the merge and publish the result to your Hugging Face account.
"""
MARKDOWN_DOCS = """
---
## Merge Methods
| Method | `merge_method` | Multi-Model | Uses base model |
|--------|----------------|-------------|-----------------|
| Linear | `linear` | Yes | No |
| SLERP | `slerp` | No (2 models) | Yes |
| Task Arithmetic | `task_arithmetic` | Yes | Yes |
| TIES | `ties` | Yes | Yes |
| DARE-TIES | `dare_ties` | Yes | Yes |
| DARE Linear | `dare_linear` | Yes | Yes |
See the [MergeKit docs](https://github.com/arcee-ai/mergekit) for full configuration options.
## Citation
```bibtex
@article{goddard2024arcee,
title={Arcee's MergeKit: A Toolkit for Merging Large Language Models},
author={Goddard, Charles and Siriwardhana, Shamane and Ehghaghi, Malikeh and Meyers, Luke and Karpukhin, Vlad and Benedict, Brian and McQuade, Mark and Solawetz, Jacob},
journal={arXiv preprint arXiv:2403.13257},
year={2024}
}
```
"""
DEFAULT_SLERP_YAML = pathlib.Path("examples/slerp.yaml").read_text(encoding="utf-8")
EXAMPLES = [[str(f)] for f in sorted(pathlib.Path("examples").glob("*.yaml"))]
def build_slerp_yaml(
model_a: str,
model_b: str,
base_model: str,
num_layers: int,
t_default: float,
dtype: str,
) -> str:
"""Build a SLERP merge YAML from form inputs."""
layer_range = [0, num_layers]
attn_values = [0, 0.5, 0.3, 0.7, 1]
mlp_values = [1, 0.5, 0.7, 0.3, 0]
config = {
"slices": [
{
"sources": [
{"model": model_a.strip(), "layer_range": layer_range},
{"model": model_b.strip(), "layer_range": layer_range},
]
}
],
"merge_method": "slerp",
"base_model": base_model.strip() or model_a.strip(),
"parameters": {
"t": [
{"filter": "self_attn", "value": attn_values},
{"filter": "mlp", "value": mlp_values},
{"value": t_default},
]
},
"dtype": dtype,
}
return yaml.dump(config, default_flow_style=False, sort_keys=False)
def update_yaml_from_builder(
model_a, model_b, base_model, num_layers, t_default, dtype
) -> str:
return build_slerp_yaml(model_a, model_b, base_model, num_layers, t_default, dtype)
def load_example(path: str) -> str:
return pathlib.Path(path).read_text(encoding="utf-8")
def prefetch_models(
runner: LogsViewRunner,
merge_config: MergeConfiguration,
hf_home: str,
lora_merge_cache: str,
):
for model in merge_config.referenced_models():
yield runner.log(f"Downloading {model}...")
merged = model.merged(cache_dir=lora_merge_cache, trust_remote_code=False)
local_path = merged.local_path(cache_dir=hf_home)
yield runner.log(f" -> {local_path}")
def run_merge_cpu(
runner: LogsViewRunner, cli: str, merged_path: str, tmpdirname: str
):
tmp_env = os.environ.copy()
tmp_env["HF_HOME"] = f"{tmpdirname}/.cache"
full_cli = (
cli
+ f" --lora-merge-cache {tmpdirname}/.lora_cache"
+ f" --transformers-cache {tmpdirname}/.cache"
)
yield from runner.run_command(full_cli.split(), cwd=merged_path, env=tmp_env)
yield ("done", runner.exit_code)
@spaces.GPU(duration=60 * 10)
def run_merge_gpu(
runner: LogsViewRunner, cli: str, merged_path: str, tmpdirname: str
):
yield from run_merge_cpu(
runner,
cli + " --cuda --low-cpu-memory --read-to-gpu",
merged_path,
tmpdirname,
)
def run_merge(
runner: LogsViewRunner,
cli: str,
merged_path: str,
tmpdirname: str,
use_gpu: bool,
):
if use_gpu:
yield from run_merge_gpu(runner, cli, merged_path, tmpdirname)
else:
yield from run_merge_cpu(runner, cli, merged_path, tmpdirname)
def merge_and_upload(
yaml_config: str,
model_name: str,
hf_token: str,
username: str,
license: str,
runtime: str,
trust_remote_code: bool,
private: bool,
) -> Iterable[List[Log]]:
runner = LogsViewRunner()
if not yaml_config.strip():
yield runner.log("Please provide a merge configuration.", level="ERROR")
return
if not hf_token.strip():
yield runner.log(
"A Hugging Face write token is required. Create one at https://huggingface.co/settings/tokens",
level="ERROR",
)
return
if not model_name.strip():
yield runner.log("Please provide a name for the merged model.", level="ERROR")
return
if not username.strip():
yield runner.log("Please provide your Hugging Face username.", level="ERROR")
return
try:
merge_config = MergeConfiguration.model_validate(yaml.safe_load(yaml_config))
except Exception as exc:
yield runner.log(f"Invalid YAML configuration: {exc}", level="ERROR")
return
api = huggingface_hub.HfApi(token=hf_token)
repo_id = f"{username.strip()}/{model_name.strip()}"
has_gpu = torch.cuda.is_available() and runtime == "GPU"
cli = "mergekit-yaml config.yaml merge --copy-tokenizer --allow-crimes -v"
if not has_gpu:
cli += " --out-shard-size 1B --lazy-unpickle"
if trust_remote_code:
cli += " --trust-remote-code"
yield runner.log(f"Runtime: {'GPU' if has_gpu else 'CPU'}")
yield runner.log(f"CLI: {cli}")
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdirname:
tmpdir = pathlib.Path(tmpdirname)
merged_path = tmpdir / "merged"
merged_path.mkdir(parents=True, exist_ok=True)
(merged_path / "config.yaml").write_text(yaml_config, encoding="utf-8")
try:
yield runner.log(f"Creating repository {repo_id}...")
repo_url = api.create_repo(
repo_id, exist_ok=True, private=private, repo_type="model"
)
yield runner.log(f"Repository ready: {repo_url}")
except Exception as exc:
yield runner.log(f"Failed to create repository: {exc}", level="ERROR")
return
yield runner.log("Prefetching source models...")
yield from prefetch_models(
runner,
merge_config,
hf_home=f"{tmpdirname}/.cache",
lora_merge_cache=f"{tmpdirname}/.lora_cache",
)
yield runner.log("Starting merge...")
exit_code = None
try:
for event in run_merge(
runner, cli, merged_path, tmpdirname, use_gpu=has_gpu
):
if isinstance(event, tuple) and event[0] == "done":
exit_code = event[1]
continue
yield event
except Exception as exc:
yield runner.log(f"Merge error: {exc}", level="ERROR")
yield runner.log("Cleaning up empty repository...", level="ERROR")
try:
api.delete_repo(repo_id)
except Exception:
pass
return
if exit_code != 0:
yield runner.log(f"Merge failed with exit code {exit_code}", level="ERROR")
try:
api.delete_repo(repo_id)
except Exception:
pass
return
yield runner.log("Merge complete. Generating model card...")
card_content = generate_model_card(
model_name=model_name.strip(),
yaml_config=yaml_config,
username=username.strip(),
license=license,
)
card = ModelCard(card_content)
merge_output = merged_path / "merge"
card.save(str(merge_output / "README.md"))
yield runner.log(f"Uploading to {repo_id}...")
yield from runner.run_python(
api.upload_folder,
repo_id=repo_id,
folder_path=str(merge_output),
)
yield runner.log(f"Success! Model uploaded to https://huggingface.co/{repo_id}")
def merge_and_upload_text(*args) -> Iterable[str]:
accumulated: list[str] = []
for batch in merge_and_upload(*args):
accumulated.append(format_logs(batch))
yield "".join(accumulated)
merge_and_upload.zerogpu = True
run_merge.zerogpu = True
with gr.Blocks(theme=gr.themes.Soft()) as demo:
gr.Markdown(MARKDOWN_HEADER)
with gr.Tabs():
with gr.Tab("Quick Builder (SLERP)"):
gr.Markdown(
"Build a SLERP merge between two models. The generated YAML is editable before merging."
)
with gr.Row():
model_a = gr.Textbox(
label="Model A",
value="OpenPipe/mistral-ft-optimized-1218",
placeholder="e.g. OpenPipe/mistral-ft-optimized-1218",
)
model_b = gr.Textbox(
label="Model B",
value="mlabonne/NeuralHermes-2.5-Mistral-7B",
placeholder="e.g. mlabonne/NeuralHermes-2.5-Mistral-7B",
)
with gr.Row():
base_model = gr.Textbox(
label="Base model",
value="OpenPipe/mistral-ft-optimized-1218",
info="Usually Model A for SLERP merges",
)
num_layers = gr.Slider(
label="Number of layers",
minimum=1,
maximum=80,
step=1,
value=32,
)
with gr.Row():
t_default = gr.Slider(
label="Default interpolation (t)",
minimum=0.0,
maximum=1.0,
step=0.05,
value=0.5,
)
dtype = gr.Dropdown(
label="dtype",
choices=["bfloat16", "float16", "float32"],
value="bfloat16",
)
build_btn = gr.Button("Generate YAML", variant="secondary")
with gr.Tab("Advanced (YAML)"):
gr.Markdown("Paste or edit a full MergeKit YAML configuration.")
yaml_config = gr.Code(
language="yaml",
lines=18,
label="Merge configuration (config.yaml)",
value=DEFAULT_SLERP_YAML,
)
example_path = gr.Textbox(visible=False)
gr.Examples(
examples=EXAMPLES,
fn=load_example,
run_on_click=True,
cache_examples=False,
label="Example configurations",
inputs=[example_path],
outputs=[yaml_config],
)
gr.Markdown("### Upload settings")
with gr.Row():
model_name = gr.Textbox(
label="Merged model name",
placeholder="NeuralPipe-7B-slerp",
value="NeuralPipe-7B-slerp",
)
username = gr.Textbox(
label="Hugging Face username",
placeholder="your-username",
value="salihfurkaan",
)
with gr.Row():
hf_token = gr.Textbox(
label="Hugging Face write token",
type="password",
info="Needs write access. https://huggingface.co/settings/tokens",
)
license_choice = gr.Dropdown(
label="License",
choices=["apache-2.0", "cc-by-nc-4.0", "mit", "openrail"],
value="apache-2.0",
allow_custom_value=True,
)
with gr.Row():
runtime = gr.Radio(
label="Runtime",
choices=["CPU", "GPU"],
value="GPU",
info="GPU uses Zero GPU when available; CPU uses lazy-unpickle for low memory.",
)
trust_remote_code = gr.Checkbox(
label="Trust remote code",
value=False,
)
private = gr.Checkbox(
label="Private repository",
value=False,
)
merge_btn = gr.Button("Merge & Upload", variant="primary", size="lg")
logs = gr.Textbox(
label="Terminal output",
lines=20,
max_lines=50,
interactive=False,
show_copy_button=True,
)
build_btn.click(
fn=update_yaml_from_builder,
inputs=[model_a, model_b, base_model, num_layers, t_default, dtype],
outputs=yaml_config,
)
merge_btn.click(
fn=merge_and_upload_text,
inputs=[
yaml_config,
model_name,
hf_token,
username,
license_choice,
runtime,
trust_remote_code,
private,
],
outputs=logs,
)
gr.Markdown(MARKDOWN_DOCS)
os.environ.setdefault("GRADIO_SERVER_NAME", "0.0.0.0")
demo.queue(default_concurrency_limit=1).launch(
server_name="0.0.0.0",
server_port=int(os.getenv("PORT", "7860")),
)