{
  "vocab_size": 8192,
  "d_model": 384,
  "n_layers": 6,
  "head_dim": 64,
  "n_experts": 160,
  "routed_experts": 8,
  "context": 1024,
  "rope_base": 1024.0,
  "qb_bias_space": "logit",
  "attention_backend": "torch",
  "expert_backend": "torch",
  "norm_lm": 39.765994980394794,
  "norm_max": 107.54856056583797,
  "lr_lm_head": 0.0036479407107242344,
  "lr_matrix": 0.016086860543376953,
  "lr_embed": 0.2494478865528685,
  "lr_norm": 0.7426652629829459,
  "adam_beta1": 0.9593251685185084,
  "adam_beta2": 0.9846493597229016,
  "muon_beta": 0.9365271129986884,
  "matrix_rms": 0.5101974318746972,
  "embed_init_std": 0.2491709558968777,
  "embed_norm": 3.7470434862324296,
  "residual_scale": 3.079348891976436,
  "norm_init": 0.9599666403748631,
  "router_temperature": 1.125857174559328,
  "warmup_steps": 77,
  "micro_batch_size": 128,
  "global_batch_tokens": 524288,
  "total_tokens": 1500000000,
  "compile": true,
  "data_dir": "data/fineweb10B_sp8192",
  "eval_every": 0,
  "save_every": 0,
  "qb_diagnostics": false,
  "qb_logit_histogram": "relative_compact",
  "qb_load_profile": "uniform",
  "qb_lower_slack": 0.0,
  "qb_upper_slack": 0.0,
  "qb_inverse_rank_offset": null
}