Nichijou_TTS / train.py
Respair's picture
Upload Nichijou
e065c61 verified
Raw History Blame Contribute Delete
26.2 kB
import json
import logging
import math
import os
import time
from copy import deepcopy
import bitsandbytes as bnb
import torch
from accelerate import Accelerator, DataLoaderConfiguration
from accelerate.utils import set_seed
from torchao.float8 import Float8LinearConfig, convert_to_float8_training
from dataset import extract_speaker_embs_from_batch, load_titanet, configure_length_training_stage
from discriminator import ConformerDiscirminator
from model_transformer import load_config, model_from_config
from training_utils import (
build_training_dataloader,
build_training_optimizers,
check_backward_finite,
checked_grad_norm,
cleanup_checkpoints,
collect_muon_norm_stats,
compute_discriminator_loss,
compute_loss,
configure_training_logging,
find_latest_checkpoint,
format_eta,
get_num_params,
load_checkpoint,
load_silence_latent,
load_training_tokenizer,
logger,
module_filter_fn,
parse_args,
reset_constant_lr,
resolve_latent_buckets,
resolve_objective_mix,
save_checkpoint,
save_nonfinite_diagnostic,
set_requires_grad,
unwrap_model,
update_ema,
)
logging.getLogger("TileLang").setLevel(logging.ERROR)
logging.getLogger("tilelang.cache.kernel_cache").setLevel(logging.ERROR)
for _http_logger in ("httpx", "httpcore"):
logging.getLogger(_http_logger).setLevel(logging.WARNING)
torch.set_float32_matmul_precision("high")
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
def main():
args = parse_args()
cfg = load_config(args.config)
model_cfg = cfg["model"]
data_cfg = cfg["data"]
train_cfg = cfg["training"]
train_cfg["length_training_stage"] = args.length_training_stage
if args.length_training_stage:
configure_length_training_stage(cfg, args.length_training_fraction)
train_cfg["length_index_cache_dir"] = os.path.join(args.exp_dir, "length_indices")
raw_max = data_cfg["max_audio_seconds"] * data_cfg["codec_rate_hz"]
max_latent_len = int(data_cfg.get("max_latent_length", int(raw_max)))
max_text_len = data_cfg["max_text_length"]
latent_dim = data_cfg["latent_dim"]
text_pad_id = data_cfg["text_pad_id"]
codec_rate_hz = float(data_cfg["codec_rate_hz"])
tokenizer_name, tokenizer = load_training_tokenizer(args, model_cfg, data_cfg)
latent_buckets = resolve_latent_buckets(data_cfg, max_latent_len)
latent_pad_value = data_cfg.get("latent_pad_value", data_cfg.get("pad_value", 0.0))
objective_cfg = resolve_objective_mix(train_cfg)
silence_latent_path = data_cfg.get(
"silence_latent_path",
"assets/silence.pt",
)
silence_latent = load_silence_latent(silence_latent_path, latent_dim)
accelerator = Accelerator(
dataloader_config=DataLoaderConfiguration(dispatch_batches=False),
gradient_accumulation_steps=train_cfg["grad_accum_steps"],
log_with=None,
)
os.makedirs(args.exp_dir, exist_ok=True)
norm_logger = configure_training_logging(args.exp_dir, accelerator)
if accelerator.is_main_process:
with open(os.path.join(args.exp_dir, "config.json"), "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=2, ensure_ascii=False)
f.write("\n")
set_seed(train_cfg["seed"])
logger.info(f"Experiment directory: {args.exp_dir}")
logger.info(
f"Stage: {'2' if args.stage2 else '1 (vanilla)'}"
f"{' + speaker conditioning' if args.use_speaker else ''}"
f"{' + adversarial' if args.use_adversarial else ''}"
)
logger.info(f"Max latent length: {max_latent_len}")
if args.length_training_stage:
logger.info(
"Length training: cached, non-overlapping groups; %.1f%% random starting samples; "
"2-8 consecutive clips / <=60s including Fish silence gaps; max_text_length=%s",
100 * train_cfg["length_training_fraction"],
max_text_len,
)
logger.info(
f"Tokenizer: {tokenizer_name} | class={type(tokenizer).__name__} | vocab={len(tokenizer)} | "
f"pad={tokenizer.pad_token_id} bos={tokenizer.bos_token_id} eos={tokenizer.eos_token_id} | "
f"add_bos={getattr(tokenizer, 'add_bos_token', None)} add_eos={getattr(tokenizer, 'add_eos_token', None)}"
)
logger.info(
f"Latent buckets: {latent_buckets} | pad_value={latent_pad_value} | "
f"upward_promotion=60/25/15,75/25,100 | silence=remaining bucket tail"
)
logger.info(
"Objective mix: "
f"full={objective_cfg['p_full']:.2f} | "
f"suffix={objective_cfg['p_suffix']:.2f} | "
f"infill={objective_cfg['p_infill']:.2f} | "
f"partial_frac=[{objective_cfg['partial_frac_min']:.2f}, "
f"{objective_cfg['partial_frac_max']:.2f}]"
)
logger.info(
f"Silence latent: {silence_latent_path} | "
f"frames={silence_latent.shape[0]} | dim={silence_latent.shape[1]}"
)
model = model_from_config(
cfg,
use_speaker_conditioning=args.use_speaker,
speaker_emb_dim=192,
)
if args.length_training_stage and hasattr(model.text_encoder, "ensure_nonempty_attention_rows"):
model.text_encoder.ensure_nonempty_attention_rows = True
logger.info("Length training: text SDPA masks protect fully masked query rows")
logger.info(f"Model parameters: {get_num_params(model)}")
logger.info("Patch/refinement architecture: %s", model.architecture_signature())
logger.info(
"Sequence lengths: frames=%d | global_tokens=%d | refinement_tokens=%d",
max_latent_len,
(max_latent_len + model.patch_size - 1) // model.patch_size,
max_latent_len if model.local_num_layers > 0 else 0,
)
ema = deepcopy(model)
ema.requires_grad_(False)
ema.eval()
if train_cfg.get("use_float8", True):
config = Float8LinearConfig.from_recipe_name("tensorwise")
convert_to_float8_training(
model,
config=config,
module_filter_fn=module_filter_fn,
)
logger.info("Linear precision: FP8 tensorwise (with Accelerate autocast)")
else:
logger.info("Linear precision: Accelerate %s; FP8 disabled", accelerator.mixed_precision)
if train_cfg.get("compile", False):
logger.info("Compiling model with torch.compile...")
model = torch.compile(model)
optimizer_name, optimizer, aux_optimizer, scheduler, aux_scheduler = build_training_optimizers(
args, model, train_cfg
)
titanet_model = None
if args.stage2 and args.use_speaker:
titanet_model = load_titanet(accelerator.device)
logger.info("Loaded frozen TitaNet-Large for on-the-fly speaker embedding extraction")
dataset, dataloader = build_training_dataloader(
args,
tokenizer,
data_cfg,
train_cfg,
max_latent_len,
latent_buckets,
latent_pad_value,
)
try:
logger.info(f"Dataset size: {len(dataset):,} samples")
except TypeError:
logger.info("Dataset size: streaming")
discriminator = None
disc_optimizer = None
if args.stage2 and args.use_adversarial:
base_model = unwrap_model(model)
disc_input_dim = base_model.num_layers * base_model.model_size
discriminator = ConformerDiscirminator(
input_dim=disc_input_dim,
channels=args.disc_channels,
num_layers=args.disc_num_layers,
num_heads=args.disc_num_heads,
depthwise_conv_kernel_size=args.disc_kernel,
use_group_norm=True,
)
disc_optimizer = bnb.optim.AdamW8bit(
discriminator.parameters(),
lr=args.disc_lr,
betas=(0.9, 0.95),
weight_decay=train_cfg["weight_decay"],
)
if args.stage2 and args.use_adversarial:
model, optimizer, dataloader, scheduler, discriminator, disc_optimizer = (
accelerator.prepare(
model, optimizer, dataloader, scheduler, discriminator, disc_optimizer
)
)
elif aux_optimizer is not None:
model, optimizer, aux_optimizer, dataloader, scheduler, aux_scheduler = accelerator.prepare(
model, optimizer, aux_optimizer, dataloader, scheduler, aux_scheduler
)
else:
model, optimizer, dataloader, scheduler = accelerator.prepare(
model, optimizer, dataloader, scheduler
)
ema = ema.to(accelerator.device)
silence_latent = silence_latent.to(accelerator.device)
if args.resume is None:
update_ema(ema, unwrap_model(model), decay=0)
global_step = 0
if args.resume is not None:
tag = args.resume
if tag == "latest":
tag = find_latest_checkpoint(args.exp_dir)
if tag is None:
logger.warning("No checkpoint found, training from scratch")
else:
global_step = load_checkpoint(
accelerator,
model,
ema,
args.exp_dir,
tag,
drop_optimizer_state=args.drop_optimizer_state,
)
else:
global_step = load_checkpoint(
accelerator,
model,
ema,
args.exp_dir,
tag,
drop_optimizer_state=args.drop_optimizer_state,
)
if args.reset_lr_scheduler:
if train_cfg.get("lr_schedule") != "constant" or aux_optimizer is not None:
raise ValueError("--reset_lr_scheduler requires constant LR and a single optimizer")
reset_constant_lr(optimizer, scheduler, train_cfg["lr"])
logger.info(
"Reset scheduler only: constant LR %.2e; optimizer moments and step %d retained",
train_cfg["lr"],
global_step,
)
try:
steps_per_epoch = len(dataloader) // train_cfg["grad_accum_steps"]
except TypeError:
steps_per_epoch = max(train_cfg["max_steps"] - global_step, 1)
max_steps = train_cfg["max_steps"]
log_every = train_cfg["log_every"]
save_every = train_cfg["save_every"]
keep_last_k = train_cfg["keep_last_k"]
local_keep_last_k = int(train_cfg.get("local_keep_last_k", keep_last_k))
if local_keep_last_k < 1:
raise ValueError("local_keep_last_k must be >= 1")
grad_clip = train_cfg["grad_clip"]
ema_decay = train_cfg["ema_decay"]
use_ckpt = train_cfg["use_checkpoint"]
logger.info(f"Training from step {global_step} to {max_steps}")
logger.info(f"Batch size per device: {train_cfg['batch_size']}")
logger.info(f"Gradient accumulation: {train_cfg['grad_accum_steps']}")
logger.info(
f"Effective batch size: "
f"{train_cfg['batch_size'] * train_cfg['grad_accum_steps'] * accelerator.num_processes}"
)
if args.stage2:
logger.info(
f"Stage 2 | use_speaker={args.use_speaker} | "
f"adversarial={args.use_adversarial} | "
f"adv_lambda={args.adv_lambda if args.use_adversarial else 0.0} | "
f"speaker_aux_weight={args.speaker_aux_weight} | "
f"speaker_drop_prob={args.speaker_drop_prob}"
)
model.train()
if args.stage2 and args.use_adversarial:
discriminator.train()
running_loss = 0.0
running_fm = 0.0
running_speech = 0.0
running_silence = 0.0
running_full_frac = 0.0
running_suffix_frac = 0.0
running_infill_frac = 0.0
running_zero_visible = 0.0
running_adv = 0.0
running_spk_aux = 0.0
running_disc = 0.0
log_steps = 0
last_grad_norm = float("nan")
pending_grad_stats = None
start_time = time.time()
epoch = global_step // steps_per_epoch if steps_per_epoch > 0 else 0
while global_step < max_steps:
try:
num_batches = len(dataloader)
except TypeError:
num_batches = None
grad_accum = train_cfg["grad_accum_steps"]
steps_per_epoch = (
num_batches // grad_accum
if num_batches is not None
else max(max_steps - global_step, 1)
)
epoch_step = 0
logger.info(f"Starting epoch {epoch} ({steps_per_epoch} steps)")
for batch_idx, batch in enumerate(dataloader):
accumulate_ctx = (
accelerator.accumulate(model, discriminator)
if args.stage2 and args.use_adversarial
else accelerator.accumulate(model)
)
with accumulate_ctx:
if args.length_training_stage:
batch["_rng_before_loss_cpu"] = torch.get_rng_state()
if accelerator.device.type == "cuda":
batch["_rng_before_loss_cuda"] = torch.cuda.get_rng_state(
accelerator.device
)
batch["_microbatch_index"] = torch.tensor(batch_idx)
if args.stage2 and args.use_speaker and titanet_model is not None:
batch["speaker_embs"] = extract_speaker_embs_from_batch(
titanet_model,
batch,
accelerator.device,
)
if args.stage2 and args.use_adversarial:
set_requires_grad(discriminator, False)
with accelerator.autocast():
loss, stats, disc_cache = compute_loss(
model=model,
batch=batch,
use_checkpoint=use_ckpt,
stage2=args.stage2,
use_speaker=args.use_speaker,
use_adversarial=(args.stage2 and args.use_adversarial),
discriminator=discriminator,
adv_lambda=(
args.adv_lambda if args.stage2 and args.use_adversarial else 0.0
),
speaker_aux_weight=args.speaker_aux_weight,
silence_latent=silence_latent,
objective_mix=objective_cfg,
)
if not torch.isfinite(loss.detach()).all():
save_nonfinite_diagnostic(
args.exp_dir, model, batch, global_step + 1, "forward loss"
)
raise FloatingPointError(
f"Non-finite forward loss before optimizer step {global_step + 1}; "
f'latent_shape={tuple(batch["latents"].shape)}, '
f'text_shape={tuple(batch["text_ids"].shape)}. '
"Stopped before backward, optimizer update, or checkpoint save."
)
accelerator.backward(loss)
if args.length_training_stage:
check_backward_finite(model, batch, args.exp_dir, global_step + 1)
if args.stage2 and args.use_adversarial:
set_requires_grad(discriminator, True)
disc_loss = compute_discriminator_loss(
model=model,
discriminator=discriminator,
cache=disc_cache,
)
accelerator.backward(disc_loss)
if accelerator.sync_gradients:
track_norms = (
optimizer_name == "muon" and (global_step + 1) % log_every == 0
)
if track_norms:
pending_grad_stats = collect_muon_norm_stats(model, grads=True)
last_grad_norm = checked_grad_norm(
accelerator, model, batch, args.exp_dir, global_step + 1, grad_clip
)
disc_grad_norm = checked_grad_norm(
accelerator,
discriminator,
batch,
args.exp_dir,
global_step + 1,
grad_clip,
)
if not math.isfinite(last_grad_norm) or not math.isfinite(disc_grad_norm):
save_nonfinite_diagnostic(
args.exp_dir, model, batch, global_step + 1, "gradient norm"
)
raise FloatingPointError(
f"Non-finite gradient norm at optimizer step {global_step + 1}; no update or checkpoint saved"
)
optimizer.step()
scheduler.step()
optimizer.zero_grad()
if aux_optimizer is not None:
aux_optimizer.step()
aux_scheduler.step()
aux_optimizer.zero_grad()
disc_optimizer.step()
disc_optimizer.zero_grad()
else:
if accelerator.sync_gradients:
track_norms = (
optimizer_name == "muon" and (global_step + 1) % log_every == 0
)
if track_norms:
pending_grad_stats = collect_muon_norm_stats(model, grads=True)
last_grad_norm = checked_grad_norm(
accelerator, model, batch, args.exp_dir, global_step + 1, grad_clip
)
if not math.isfinite(last_grad_norm):
save_nonfinite_diagnostic(
args.exp_dir, model, batch, global_step + 1, "gradient norm"
)
raise FloatingPointError(
f"Non-finite gradient norm at optimizer step {global_step + 1}; no update or checkpoint saved"
)
optimizer.step()
scheduler.step()
optimizer.zero_grad()
if aux_optimizer is not None:
aux_optimizer.step()
aux_scheduler.step()
aux_optimizer.zero_grad()
disc_loss = loss.new_zeros(())
if not accelerator.sync_gradients:
continue
global_step += 1
epoch_step += 1
update_ema(ema, unwrap_model(model), ema_decay)
running_loss += loss.detach().item()
running_fm += stats["fm_loss"].item()
running_speech += stats["speech_loss"].item()
running_silence += stats["silence_loss"].item()
running_full_frac += stats["full_tts_frac"].item()
running_suffix_frac += stats["suffix_frac"].item()
running_infill_frac += stats["infill_frac"].item()
running_zero_visible += stats["zero_visible_frac"].item()
running_adv += stats["gen_adv_loss"].item()
running_spk_aux += stats["speaker_aux_loss"].item()
running_disc += disc_loss.detach().item()
log_steps += 1
if global_step % log_every == 0:
n = log_steps
if optimizer_name == "muon":
lr_text = f"muon_lr={optimizer.param_groups[0]['lr']:.2e} | adamw_lr={aux_optimizer.param_groups[0]['lr']:.2e}"
else:
lr_text = f"lr={optimizer.param_groups[0]['lr']:.2e}"
elapsed = time.time() - start_time
sps = n / elapsed
remaining = (max_steps - global_step) / sps
mem_gb = torch.cuda.max_memory_allocated() / 1e9
speed = f"{sps:.2f} it/s" if sps >= 1.0 else f"{elapsed / n:.2f} s/it"
if optimizer_name == "muon" and norm_logger is not None:
weight_stats = collect_muon_norm_stats(model, grads=False)
grad_stats = pending_grad_stats or {}
norm_logger.info(
"step=%d | muon_lr=%.8g | adamw_lr=%.8g | grad_global_preclip=%.6g | "
"muon_grad_rms=%.6g muon_grad_max=%.6g muon_grad_l2=%.6g | "
"q_grad_rms=%.6g q_grad_max=%.6g q_grad_l2=%.6g | "
"k_grad_rms=%.6g k_grad_max=%.6g k_grad_l2=%.6g | "
"muon_w_rms=%.6g muon_w_max=%.6g muon_w_l2=%.6g | "
"q_w_rms=%.6g q_w_max=%.6g q_w_l2=%.6g | "
"k_w_rms=%.6g k_w_max=%.6g k_w_l2=%.6g | "
"peak_muon_grad=%.6g:%s peak_q_grad=%.6g:%s peak_k_grad=%.6g:%s | "
"peak_muon_w=%.6g:%s peak_q_w=%.6g:%s peak_k_w=%.6g:%s",
global_step,
optimizer.param_groups[0]["lr"],
aux_optimizer.param_groups[0]["lr"],
last_grad_norm,
grad_stats.get("muon_rms", 0.0),
grad_stats.get("muon_max", 0.0),
grad_stats.get("muon_l2", 0.0),
grad_stats.get("q_rms", 0.0),
grad_stats.get("q_max", 0.0),
grad_stats.get("q_l2", 0.0),
grad_stats.get("k_rms", 0.0),
grad_stats.get("k_max", 0.0),
grad_stats.get("k_l2", 0.0),
weight_stats["muon_rms"],
weight_stats["muon_max"],
weight_stats["muon_l2"],
weight_stats["q_rms"],
weight_stats["q_max"],
weight_stats["q_l2"],
weight_stats["k_rms"],
weight_stats["k_max"],
weight_stats["k_l2"],
grad_stats.get("muon_peak_l2", 0.0),
grad_stats.get("muon_peak_name", "none"),
grad_stats.get("q_peak_l2", 0.0),
grad_stats.get("q_peak_name", "none"),
grad_stats.get("k_peak_l2", 0.0),
grad_stats.get("k_peak_name", "none"),
weight_stats["muon_peak_l2"],
weight_stats["muon_peak_name"],
weight_stats["q_peak_l2"],
weight_stats["q_peak_name"],
weight_stats["k_peak_l2"],
weight_stats["k_peak_name"],
)
for handler in norm_logger.handlers:
handler.flush()
pending_grad_stats = None
if args.stage2:
logger.info(
f"step={global_step:>7d} | "
f"epoch={epoch} [{epoch_step}/{steps_per_epoch}] | "
f"loss={running_loss / n:.4f} | "
f"fm={running_fm / n:.4f} | "
f"speech={running_speech / n:.4f} | silence={running_silence / n:.4f} | "
f"modes={running_full_frac / n:.2f}/{running_suffix_frac / n:.2f}/{running_infill_frac / n:.2f} | "
f"zero_ctx={running_zero_visible / n:.2f} | "
f"spk_aux={running_spk_aux / n:.4f} | "
f"g_adv={running_adv / n:.4f} | "
f"d={running_disc / n:.4f} | "
f"{lr_text} | {speed} | "
f"mem={mem_gb:.1f}GB | eta={format_eta(remaining)}"
)
else:
logger.info(
f"step={global_step:>7d} | "
f"epoch={epoch} [{epoch_step}/{steps_per_epoch}] | "
f"loss={running_loss / n:.4f} | "
f"speech={running_speech / n:.4f} | silence={running_silence / n:.4f} | "
f"modes={running_full_frac / n:.2f}/{running_suffix_frac / n:.2f}/{running_infill_frac / n:.2f} | "
f"zero_ctx={running_zero_visible / n:.2f} | "
f"{lr_text} | {speed} | "
f"mem={mem_gb:.1f}GB | eta={format_eta(remaining)}"
)
running_loss = running_fm = running_speech = running_silence = 0.0
running_full_frac = running_suffix_frac = running_infill_frac = (
running_zero_visible
) = 0.0
running_adv = running_spk_aux = running_disc = 0.0
log_steps = 0
start_time = time.time()
if global_step % save_every == 0:
tag = f"step_{global_step:07d}"
save_checkpoint(
accelerator,
model,
ema,
global_step,
args.exp_dir,
tag,
checkpoint_bucket=train_cfg.get("checkpoint_bucket"),
keep_last_k=keep_last_k,
local_keep_last_k=local_keep_last_k,
)
if accelerator.is_main_process:
cleanup_checkpoints(args.exp_dir, local_keep_last_k)
if global_step >= max_steps:
break
epoch += 1
tag = f"step_{global_step:07d}"
save_checkpoint(
accelerator,
model,
ema,
global_step,
args.exp_dir,
tag,
checkpoint_bucket=train_cfg.get("checkpoint_bucket"),
keep_last_k=keep_last_k,
local_keep_last_k=local_keep_last_k,
)
if accelerator.is_main_process:
cleanup_checkpoints(args.exp_dir, local_keep_last_k)
logger.info("Training complete.")
if __name__ == "__main__":
main()