EXL3 4 bpw (experts 4 / attention 6) from bf16, code calibration

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/ai-rd-benchmarks.png filter=lfs diff=lfs merge=lfs -text
+assets/coding-benchmarks.png filter=lfs diff=lfs merge=lfs -text
+model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
+quantization_config.json filter=lfs diff=lfs merge=lfs -text

GATE_exllamav3.json

@@ -0,0 +1,87 @@
+{
+ "pack": "/mnt/models/Naive-N0.5-Flash-exl3-exp4-dense6",
+ "nll": {
+  "file": "/mnt/models/work/tests/fixtures/naive/bench_ref_text.txt",
+  "tokens": 977,
+  "mean_nll": 1.6335245370864868,
+  "seconds": 61.96650791168213
+ },
+ "prefixes": [
+  {
+   "file": "/mnt/models/work/tests/fixtures/naive/prompt_fib.txt",
+   "tokens": 77,
+   "top": [
+    {
+     "token": "       ",
+     "p": 0.9999997615814209
+    },
+    {
+     "token": " raise",
+     "p": 9.931192579415438e-08
+    },
+    {
+     "token": "      ",
+     "p": 7.380235444998107e-08
+    },
+    {
+     "token": "   ",
+     "p": 4.012564147615194e-08
+    },
+    {
+     "token": "raise",
+     "p": 1.92524929332194e-08
+    },
+    {
+     "token": "     ",
+     "p": 1.6467531338548724e-08
+    },
+    {
+     "token": "        ",
+     "p": 1.499385682279808e-08
+    },
+    {
+     "token": "\traise",
+     "p": 1.4761399214080484e-08
+    }
+   ]
+  },
+  {
+   "file": "/mnt/models/work/tests/fixtures/naive/prompt_json.txt",
+   "tokens": 168,
+   "top": [
+    {
+     "token": "age",
+     "p": 0.9999957084655762
+    },
+    {
+     "token": " age",
+     "p": 1.7061788639693987e-06
+    },
+    {
+     "token": "Age",
+     "p": 6.681492550342227e-07
+    },
+    {
+     "token": "name",
+     "p": 3.205815346518648e-07
+    },
+    {
+     "token": "ages",
+     "p": 2.1024249008405604e-07
+    },
+    {
+     "token": "old",
+     "p": 1.0246399995139654e-07
+    },
+    {
+     "token": "size",
+     "p": 7.614477937067932e-08
+    },
+    {
+     "token": "area",
+     "p": 7.265785484378284e-08
+    }
+   ]
+  }
+ ]
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Naive AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,121 @@
+---
+license: mit
+license_link: https://huggingface.co/NaiveAI/Naive-N0.5-Flash/blob/main/LICENSE
+base_model: NaiveAI/Naive-N0.5-Flash
+tags:
+  - exl3
+  - exllamav3
+  - trellis
+  - 4bpw
+  - mixed-bitrate
+  - moe
+  - long-context
+  - dgx-spark
+  - naive-n0.5-flash
+---
+
+# Naive-N0.5-Flash — EXL3 4 bpw (experts 4 / attention 6)
+
+> **Original model:** [NaiveAI/Naive-N0.5-Flash](https://huggingface.co/NaiveAI/Naive-N0.5-Flash) by the
+> NaiveAI Team, [MIT License](https://huggingface.co/NaiveAI/Naive-N0.5-Flash/blob/main/LICENSE).
+> This artifact is quantized from the **bf16 master** release (not the FP8 one).
+> It contains **modified weights** (EXL3 trellis quantization); the original model is © its authors.
+> Tokenizer and chat template are unchanged from the original.
+
+A sibling of [doth4580/Naive-N0.5-Flash-EXL3-3.5bpw](https://huggingface.co/doth4580/Naive-N0.5-Flash-EXL3-3.5bpw):
+an **EXL3 (ExLlamaV3 trellis) mixed-bitrate** quantization of **Naive-N0.5-Flash** — 309B parameters,
+15.5B active, native 1M-token context — with the **routed experts at 4 bpw** and **attention, the DSA
+indexer, the dense layer-0 MLP and `lm_head` at 6 bpw** (4.03 bpw overall). The pack is **~157 GB
+(~147 GiB) against ~315 GB for the FP8 release and ~618 GB for bf16**; it fits across **two NVIDIA DGX
+Spark (GB10)** machines with less headroom than the 3.5 bpw sibling, or across four comfortably.
+
+Quantized with stock [exllamav3](https://github.com/turboderp-org/exllamav3) **1.5.3** plus the
+architecture definition from
+[sf-stav/Naive-N0.5-Flash-EXL3-2x-DGX-Spark](https://github.com/sf-stav/Naive-N0.5-Flash-EXL3-2x-DGX-Spark),
+run on **8× RTX PRO 6000 (Blackwell)** in about 1.5 hours, with a **498-row × 2,048-token calibration set
+built from code repositories, agent-style tool-calling conversations rendered through the model's own
+chat template, and general text** (instead of the converter's default 250-row general set).
+
+## How it compares
+
+Forward-only gate in exllamav3 (teacher-forced NLL on a 977-token technical text; next-token
+probability after two code/JSON prefixes), same prompts for every row:
+
+| model | source | NLL (nats/token) | p(8-space indent after `if n < 0:`) | p(`age`) on a JSON key |
+|---|---|---|---|---|
+| Naive-N0.5-Flash bf16, `transformers` | bf16 | 1.627 | 1.000 | 1.000 |
+| **this pack** (4.03 bpw) | bf16 | **1.634** | 1.000 | 1.000 |
+| sibling 3.5 bpw pack | FP8 | 1.619 | 1.000 | 1.000 |
+
+So this pack is within 0.007 nats of its own unquantized source. It is **not** better than the 3.5 bpw
+sibling on that text: the FP8 release and the bf16 master are different checkpoints (their embedding
+tables differ by ~14 % relative), and on this particular text the FP8-derived pack scores marginally lower.
+Where this pack may pay is code and tool-call output, which its calibration set targets, and in the
+headroom that 4-bit experts leave against outliers; measure on your own workload. Note that a pack built
+by **mixing** the two sources (attention from bf16, experts copied from the FP8-derived pack) scored 2.48
+nats — the two releases do not mix.
+
+## Serving
+
+- **ExLlamaV3 directly:** upstream exllamav3 ≥ 1.5.3 with the architecture definition from the
+  [recipe repository](https://github.com/sf-stav/Naive-N0.5-Flash-EXL3-2x-DGX-Spark) applied. The
+  pack carries the native exllamav3 metadata (`config.json` with its `quantization_config`,
+  `quantization_config.json`, `model.safetensors.index.json`).
+- **veloGB10** ([sf-stav/veloGB10](https://github.com/sf-stav/veloGB10)), the from-scratch GB10 engine
+  this pack was made for: it loads per-tensor bit rates (3 / 3.5 / 4 / 5 / 6 bpw) from the first release
+  that includes the Naive-N0.5 mixed-rate loader.
+- **The vLLM + vllm-exl3 recipe used by the 3.5 bpw sibling has not been tested with this pack.** That
+  stack was validated for a *uniform* 3.5 bpw pack; a mixed-rate pack may need plugin changes. The
+  vLLM-rewritten `config.json` / `.native` pair of the sibling is therefore not shipped here.
+
+Important for any runtime: layer 47's routed `up_proj` is stored **divided by 128** (exllamav3's
+`interm_div` for this architecture — that layer's expert activations overflow f16) and the forward must
+scale that layer's routing weights by 128, as exllamav3 does. A runtime that applies the pack as stored
+silently produces a last-layer MoE 128× too small (the symptom is loops and wrong token choices, not a crash).
+
+## Specifications
+
+| | |
+|---|---|
+| Base model | [NaiveAI/Naive-N0.5-Flash](https://huggingface.co/NaiveAI/Naive-N0.5-Flash) (bf16 master as the direct source) |
+| Architecture | MoE, 309B total / 15.5B active, 48 layers, hidden 4096, 64 heads / 4 KV heads, head_dim 192 |
+| Attention | hybrid 39 SWA layers (window 128) + 9 DSA layers (top-2048 selection), GQA4, no full attention |
+| MoE | 256 routed experts, 8 per token, expert intermediate 2048 |
+| Context / vocab | 1,048,576 tokens / 152,576 |
+| Quantization | EXL3 trellis, `mul1` codebook — routed experts 4 bpw; attention, DSA indexer, dense MLP and `lm_head` 6 bpw (4.03 bpw overall); embeddings unquantized; calibration 498×2048 (code / agent / general) |
+| Format / size | native `exl3` — ~157 GB (~147 GiB), 24 shards |
+| License | MIT (inherited from the original model) |
+
+## Files
+
+| File | Notes |
+|---|---|
+| `model-00001..00024-of-00024.safetensors` | the quantized weights |
+| `model.safetensors.index.json`, `quantization_config.json` | tensor index and the full EXL3 metadata |
+| `config.json` | the native ExLlamaV3 pack config (carries `quantization_config`) |
+| `RECIPE.yaml` | the per-tensor bit-rate recipe the converter was run with |
+| `GATE_exllamav3.json`, `CONVERT.log` | the gate numbers above and the converter's full log |
+| `SHA256SUMS` | checksums of every file in the pack |
+| `modeling_naive_n05_flash.py`, `configuration_naive_n05_flash.py` | model code, loaded via `trust_remote_code` |
+| tokenizer + `chat_template.jinja`, `generation_config.json` | unchanged from the original |
+
+## Notes
+
+- **Beyond 2,048 tokens the DSA layers are approximate** in runtimes that run them as dense attention
+  (exact up to 2,048 tokens); the 1M native context needs a sparse attention backend.
+- **Larger than the sibling:** ~14 % more bytes than the 3.5 bpw pack, so expect proportionally lower
+  decode speed on bandwidth-bound hardware such as the DGX Spark.
+- Sampling defaults from the original `generation_config.json`: temperature 1.0, top_p 0.95.
+
+## License and credits
+
+- **Original model:** [NaiveAI/Naive-N0.5-Flash](https://huggingface.co/NaiveAI/Naive-N0.5-Flash) —
+  © the NaiveAI Team, [MIT](https://huggingface.co/NaiveAI/Naive-N0.5-Flash/blob/main/LICENSE); a copy
+  ships here as `LICENSE`. Built on Xiaomi's
+  [MiMo-V2.5](https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Base) and DeepSeek Sparse Attention.
+- **This artifact:** EXL3 mixed-bitrate quantization produced with
+  [exllamav3](https://github.com/turboderp-org/exllamav3) 1.5.3 and the architecture definition from
+  the [recipe repository](https://github.com/sf-stav/Naive-N0.5-Flash-EXL3-2x-DGX-Spark). Weights are
+  modified; tokenizer and chat template are not.
+- **Sibling:** [doth4580/Naive-N0.5-Flash-EXL3-3.5bpw](https://huggingface.co/doth4580/Naive-N0.5-Flash-EXL3-3.5bpw)
+  (uniform 3.5 bpw from the FP8 release, with the vLLM recipe).

SHA256SUMS

@@ -0,0 +1,37 @@
+42e3af1e796d1684d557a4ea5c86a16a44ac2a0f22d8e4c6f037a1cb619666d4  model-00001-of-00024.safetensors
+b3456afef81b352652047f566537776b6148eafd08427767e0038964b354ce39  model-00002-of-00024.safetensors
+2561992afcaed488332eddf0517cfadb3041535d2fde74a07f99c52877d65719  model-00003-of-00024.safetensors
+070ec1b74e56de7a2c25b993a68f5b47dc3541dc999a68dd603357d1d3323b98  model-00004-of-00024.safetensors
+53d1da81fc0add15870d77c6820b0db36b4ea0c25e30f3ee21901d32f639dd37  model-00005-of-00024.safetensors
+74941483a1a84fcab3ec46b6142354238ad91bf0c09700782f758fb64e5e811b  model-00006-of-00024.safetensors
+10237cb3d2f6a9b6bc4feddb10d181c837f7274eda45ab48a4b19fb8e636991e  model-00007-of-00024.safetensors
+42b3ffe3e7fbec40e8bf892b0a5ba8b1a10b28e4abf4773788b48c4e085510c5  model-00008-of-00024.safetensors
+2c24818edeb51f28057abcb232ab6fad44933e94115edfb503bedd37fb508881  model-00009-of-00024.safetensors
+504a0693ce37f002c20e9f0ecbf81a12d443547950b6d4c0799d80a39b0ef997  model-00010-of-00024.safetensors
+fb6b4b7b549592f937286a985814a0305bd3ab20fe70878b896a3b8723965403  model-00011-of-00024.safetensors
+654176edda31382f12001024555023264ff157a998aea7c7dc050ef560e3b04f  model-00012-of-00024.safetensors
+3b07fde58523badd56f1fb2ea71c8d93dbae0567b9cf96771f800e4fce1e8ce7  model-00013-of-00024.safetensors
+7d3d5ebae01af1c480cbc21b4e54230a06a53521ec481b8a31ad3dede773eda6  model-00014-of-00024.safetensors
+2c815df50218f15e97371ad7965563af8378f9ecfbf45d284b8f25536eed2a5e  model-00015-of-00024.safetensors
+826ece4eeb60c44b88812998d307a7e40fcf4a0437f2babb53201cdb054df9b0  model-00016-of-00024.safetensors
+956d917971973e211f3128368eaa42c178ee3824fbfc3a5d29caaece0164616c  model-00017-of-00024.safetensors
+700a7579879ca33bba532162ce999fd0ee8649a5db65208726506e768e7af832  model-00018-of-00024.safetensors
+c91381268219ec82c8771c1644411e64f18a3bdc7b82fb199e2ccfc626023141  model-00019-of-00024.safetensors
+121d86aaa9b0547cd68e6fc7e89c1041f5ebde48b752e90af7b29e70e5879cf5  model-00020-of-00024.safetensors
+499e3e378115912207a89cc2f07ee7bbd952a14f4e7ea13483e7a404192cb85a  model-00021-of-00024.safetensors
+816e1a8c012fe45c1365374b04ce8704d28f9e3eb7c2c82d718abe36235add83  model-00022-of-00024.safetensors
+4c9b0bb5671a96c959ae40321c50660778ebaabe17d816b3aa99358dd1fc8e79  model-00023-of-00024.safetensors
+a6368e5481852c67d9495de04fdfc8514a971ffacc345e0b1247576aa312bb5e  model-00024-of-00024.safetensors
+2833a8b6950d0054daf7290696503833fb3f25bf409cdc742aad9b3fd1ca59b9  config.json
+32fe0aac6da16419c6cfa28688978ad5ce54b7fd576ddd406ab7df60587a08cd  generation_config.json
+1fb058ed285464a798f34a53445d24b2b488d3c768574e322c717205607c3021  model.safetensors.index.json
+7c7110f110f3a2df203835d6c95f4d4cd145bf6536bfe3b527d91e60f89b8d1a  quantization_config.json
+bd994a943a5a44bb8c70c4f27d412a83eba9308c5e8689412219da448a2bd2ed  special_tokens_map.json
+2efef62c60dedec64b55ab8b6e6838ca5cee391c00c62f2c554ba5de3d33e030  tokenizer.json
+a6970b2958597d6463ea3c6737b77797de1261e43dadddb5a6b737c377bec8dd  tokenizer_config.json
+ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910  vocab.json
+e7ecc00653da360dd32df60e7f278f0e7533b3d7d8e70bdb8a877c9a1660770e  chat_template.jinja
+8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5  merges.txt
+510adea6465e3fa40925c04769abac67a05cef4540d8451c4e7f892c213d0e8e  configuration_naive_n05_flash.py
+e192cea4fa3d01f494d91e6c97ace2c25b907d92e942484e025d86ae2df806c7  modeling_naive_n05_flash.py
+a8d3b45fb752ec36b122e4de6b61c96eecc0fc9657b881b1a60def1424afed26  README.md

chat_template.jinja

@@ -0,0 +1,156 @@
+{%- if not add_generation_prompt is defined -%}
+    {%- set add_generation_prompt = false -%}
+{%- endif -%}
+{%- if not enable_thinking is defined -%}
+    {%- set enable_thinking = true -%}
+{%- endif -%}
+{%- if not keep_all_reasoning is defined -%}
+    {%- set keep_all_reasoning = true -%}
+{%- endif -%}
+{%- macro render_extra_keys(json_dict, handled_keys) -%}
+    {%- if json_dict is mapping %}
+        {%- for json_key in json_dict if json_key not in handled_keys %}
+            {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
+                {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
+            {%- else %}
+                {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
+            {%- endif %}
+        {%- endfor %}
+    {%- endif %}
+{%- endmacro -%}
+{%- macro render_content(message_content) -%}
+    {%- if message_content is string -%}
+        {{- message_content -}}
+    {%- else -%}
+        {%- for content in message_content -%}
+            {%- if 'text' in content -%}
+                {{- content['text'] -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if messages[0]["role"] == "system" %}
+    {%- set system_message = messages[0]["content"] %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- set ns = namespace(last_user_index=-1) %}
+{%- for m in loop_messages %}
+    {%- if m.role == 'user' %}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{%- if not tools is defined %}
+    {%- set tools = [] %}
+{%- endif %}
+{%- if system_message is defined %}
+    {{- "<|im_start|>system\n" + render_content(system_message) }}
+{%- else %}
+    {{- "<|im_start|>system\nYou are Naive-N0.5-Flash, a helpful AI assistant engineered by Naive AI." }}
+{%- endif %}
+{%- if tools is iterable and tools | length > 0 %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou have access to the following functions:\n\n" }}
+    {{- "<tools>" }}
+    {%- for tool in tools %}
+        {%- if tool.function is defined %}
+            {%- set tool = tool.function %}
+        {%- endif %}
+        {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
+        {%- if tool.description is defined %}
+            {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
+        {%- endif %}
+        {{- '\n<parameters>' }}
+        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
+            {%- for param_name, param_fields in tool.parameters.properties|items %}
+                {{- '\n<parameter>' }}
+                {{- '\n<name>' ~ param_name ~ '</name>' }}
+                {%- if param_fields.type is defined %}
+                    {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
+                {%- endif %}
+                {%- if param_fields.description is defined %}
+                    {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
+                {%- endif %}
+                {%- set handled_keys = ['name', 'type', 'description'] %}
+                {{- render_extra_keys(param_fields, handled_keys) }}
+                {{- '\n</parameter>' }}
+            {%- endfor %}
+        {%- endif %}
+        {%- set handled_keys = ['type', 'properties'] %}
+        {{- render_extra_keys(tool.parameters, handled_keys) }}
+        {{- '\n</parameters>' }}
+        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
+        {{- render_extra_keys(tool, handled_keys) }}
+        {{- '\n</function>' }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nFor each function call, output the function name and arguments in the following format:\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>value_1</parameter>\n<parameter=example_parameter_2>This is the value for the second parameter\nthat can span\nmultiple lines</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- DO NOT use function calls inside <think></think> tags.\n- The value enclosed between parameter tags is preserved exactly as-is, including newlines and spaces.\n</IMPORTANT>' }}
+{%- endif %}
+{{- '<|im_end|>' }}
+{%- for message in loop_messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = render_content(message.content) %}
+    {%- endif %}
+    {%- if message.role == "assistant" %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- set reasoning_content = '' %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
+                {%- set content = content.split('</think>')[-1] %}
+            {%- endif %}
+        {%- endif %}
+        {%- if (keep_all_reasoning or loop.index0 > ns.last_user_index) and reasoning_content -%}
+            {{- '<|im_start|>' + message.role + '\n<think>' + reasoning_content + '</think>' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n<think></think>' + content }}
+        {%- endif %}
+        {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>' }}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
+    {%- elif message.role == "system" %}
+        {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>tool\n' }}
+        {%- endif %}
+        {{- '<tool_response>\n' }}
+        {{- render_content(message.content) }}
+        {{- '\n</tool_response>\n' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>' }}
+        {%- endif %}
+    {%- else %}
+        {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if not enable_thinking -%}
+        {{- '<think></think>' -}}
+    {%- else -%}
+        {{- '' -}}
+    {%- endif -%}
+{%- endif %}

config.json

@@ -0,0 +1,179 @@
+{
+    "add_full_attention_sink_bias": false,
+    "add_swa_attention_sink_bias": true,
+    "architectures": [
+        "NaiveN05FlashForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_chunk_size": 128,
+    "attention_dropout": 0.0,
+    "attention_projection_layout": "split",
+    "attention_value_scale": 0.707,
+    "auto_map": {
+        "AutoConfig": "configuration_naive_n05_flash.NaiveN05FlashConfig",
+        "AutoModel": "modeling_naive_n05_flash.NaiveN05FlashModel",
+        "AutoModelForCausalLM": "modeling_naive_n05_flash.NaiveN05FlashForCausalLM"
+    },
+    "dtype": "bfloat16",
+    "eos_token_id": 151645,
+    "head_dim": 192,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "hybrid_block_size": null,
+    "hybrid_layer_pattern": [
+        0,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        0
+    ],
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "layernorm_epsilon": 1e-05,
+    "max_position_embeddings": 1048576,
+    "model_type": "naive_n05_flash",
+    "moe_intermediate_size": 2048,
+    "moe_layer_freq": [
+        0,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1,
+        1
+    ],
+    "n_group": 1,
+    "n_routed_experts": 256,
+    "n_shared_experts": null,
+    "norm_topk_prob": true,
+    "num_attention_heads": 64,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 4,
+    "pad_token_id": 151643,
+    "partial_rotary_factor": 0.334,
+    "rope_scaling": {
+        "rope_type": "default",
+        "type": "default"
+    },
+    "rope_theta": 10000000,
+    "routed_scaling_factor": null,
+    "scoring_func": "sigmoid",
+    "sliding_window": 128,
+    "sliding_window_size": 128,
+    "swa_head_dim": 192,
+    "swa_num_attention_heads": 64,
+    "swa_num_key_value_heads": 8,
+    "swa_rope_theta": 10000,
+    "swa_v_head_dim": 128,
+    "tie_word_embeddings": false,
+    "topk_group": 1,
+    "topk_method": "noaux_tc",
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.57.1",
+    "use_cache": true,
+    "v_head_dim": 128,
+    "vocab_size": 152576,
+    "enable_dsa": true,
+    "index_top_k": 2048,
+    "index_head_dim": 128,
+    "index_n_heads": 16,
+    "index_n_kv_heads": 1,
+    "indexer_activation_dtype": "fp8_e4m3",
+    "quantization_config": {
+        "quant_method": "exl3",
+        "version": "1.5.3",
+        "bits": 4.03,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 498,
+            "cols": 2048
+        },
+        "out_scales": "always",
+        "codebook": "mul1"
+    }
+}

configuration_naive_n05_flash.py

@@ -0,0 +1,155 @@
+# Copyright 2026 Naive AI.
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class NaiveN05FlashConfig(PretrainedConfig):
+    model_type = "naive_n05_flash"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {"num_local_experts": "n_routed_experts"}
+
+    def __init__(
+        self,
+        vocab_size: int = 152576,
+        hidden_size: int = 4096,
+        intermediate_size: int = 16384,
+        num_hidden_layers: int = 48,
+        num_attention_heads: int = 64,
+        num_key_value_heads: int = 4,
+        head_dim: int = 192,
+        v_head_dim: int = 128,
+        swa_num_attention_heads: int = 64,
+        swa_num_key_value_heads: int = 8,
+        swa_head_dim: int = 192,
+        swa_v_head_dim: int = 128,
+        hidden_act: str = "silu",
+        max_position_embeddings: int = 1048576,
+        initializer_range: float = 0.02,
+        layernorm_epsilon: float = 1e-5,
+        rope_theta: float = 10000000.0,
+        swa_rope_theta: float = 10000.0,
+        partial_rotary_factor: float = 0.334,
+        sliding_window: int = 128,
+        hybrid_layer_pattern: list[int] | None = None,
+        attention_bias: bool = False,
+        attention_projection_layout: str = "split",
+        attention_value_scale: float | None = 0.707,
+        add_full_attention_sink_bias: bool = False,
+        add_swa_attention_sink_bias: bool = True,
+        n_routed_experts: int = 256,
+        moe_intermediate_size: int = 2048,
+        num_experts_per_tok: int = 8,
+        routed_scaling_factor: float | None = 1.0,
+        scoring_func: str = "sigmoid",
+        topk_method: str = "noaux_tc",
+        n_group: int = 1,
+        topk_group: int = 1,
+        norm_topk_prob: bool = True,
+        moe_layer_freq: list[int] | None = None,
+        enable_dsa: bool = True,
+        index_top_k: int = 2048,
+        index_head_dim: int = 128,
+        index_n_heads: int = 16,
+        index_n_kv_heads: int = 1,
+        indexer_activation_dtype: str = "fp8_e4m3",
+        use_cache: bool = True,
+        tie_word_embeddings: bool = False,
+        pad_token_id: int | None = 151643,
+        eos_token_id: int | list[int] | None = 151645,
+        **kwargs,
+    ) -> None:
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.v_head_dim = v_head_dim
+        self.swa_num_attention_heads = swa_num_attention_heads
+        self.swa_num_key_value_heads = swa_num_key_value_heads
+        self.swa_head_dim = swa_head_dim
+        self.swa_v_head_dim = swa_v_head_dim
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.layernorm_epsilon = layernorm_epsilon
+        self.rope_theta = rope_theta
+        self.swa_rope_theta = swa_rope_theta
+        self.partial_rotary_factor = partial_rotary_factor
+        self.sliding_window = sliding_window
+        self.hybrid_layer_pattern = hybrid_layer_pattern or [
+            int(i != 0 and i % 6 != 5) for i in range(num_hidden_layers)
+        ]
+        self.attention_bias = attention_bias
+        self.attention_projection_layout = attention_projection_layout
+        self.attention_value_scale = attention_value_scale
+        self.add_full_attention_sink_bias = add_full_attention_sink_bias
+        self.add_swa_attention_sink_bias = add_swa_attention_sink_bias
+        self.n_routed_experts = n_routed_experts
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_experts_per_tok = num_experts_per_tok
+        self.routed_scaling_factor = routed_scaling_factor or 1.0
+        self.scoring_func = scoring_func
+        self.topk_method = topk_method
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.norm_topk_prob = norm_topk_prob
+        self.moe_layer_freq = moe_layer_freq or ([0] + [1] * (num_hidden_layers - 1))
+        self.enable_dsa = enable_dsa
+        self.index_top_k = index_top_k
+        self.index_head_dim = index_head_dim
+        self.index_n_heads = index_n_heads
+        self.index_n_kv_heads = index_n_kv_heads
+        self.indexer_activation_dtype = indexer_activation_dtype
+        self.use_cache = use_cache
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings, pad_token_id=pad_token_id, eos_token_id=eos_token_id, **kwargs
+        )
+        self.validate()
+
+    def validate(self):
+        if self.attention_projection_layout != "split" or self.index_n_kv_heads != 1:
+            raise ValueError("NaiveN05Flash requires split Q/K/V and one indexer KV head")
+        if not self.enable_dsa or self.index_top_k <= 0:
+            raise ValueError("NaiveN05Flash requires DSA with a positive index_top_k")
+        if self.indexer_activation_dtype not in ("bf16", "fp8_e4m3"):
+            raise ValueError("indexer_activation_dtype must be bf16 or fp8_e4m3")
+        if self.scoring_func != "sigmoid" or self.topk_method != "noaux_tc":
+            raise ValueError("NaiveN05Flash requires sigmoid routing with correction bias")
+        if self.n_group != 1 or self.topk_group != 1:
+            raise ValueError("NaiveN05Flash uses one expert group")
+        for prefix in ("", "swa_"):
+            heads = getattr(self, prefix + "num_attention_heads")
+            kv_heads = getattr(self, prefix + "num_key_value_heads")
+            head_dim = getattr(self, prefix + "head_dim")
+            rotary_dim = int(head_dim * self.partial_rotary_factor)
+            if kv_heads <= 0 or heads % kv_heads or rotary_dim % 2 or not 0 < rotary_dim <= head_dim:
+                raise ValueError("Invalid attention head or rotary dimensions")
+        if int(self.head_dim * self.partial_rotary_factor) > self.index_head_dim:
+            raise ValueError("Indexer head dimension must cover the rotary dimension")
+        if self._attn_implementation not in (None, "eager"):
+            raise ValueError("NaiveN05Flash supports eager attention only")
+        if len(self.hybrid_layer_pattern) != self.num_hidden_layers:
+            raise ValueError("hybrid_layer_pattern must contain one entry per layer")
+        if len(self.moe_layer_freq) != self.num_hidden_layers:
+            raise ValueError("moe_layer_freq must contain one entry per layer")
+        self.layer_types = [
+            "sliding_attention" if is_swa else "deepseek_sparse_attention" for is_swa in self.hybrid_layer_pattern
+        ]
+
+
+__all__ = ["NaiveN05FlashConfig"]

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "do_sample": true,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "repetition_penalty": 1.0,
+  "eos_token_id": 151645,
+  "pad_token_id": 151643,
+  "max_new_tokens": 131072
+}

model-00001-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42e3af1e796d1684d557a4ea5c86a16a44ac2a0f22d8e4c6f037a1cb619666d4
+size 8087010992

model-00002-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3456afef81b352652047f566537776b6148eafd08427767e0038964b354ce39
+size 6611149486

model-00003-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2561992afcaed488332eddf0517cfadb3041535d2fde74a07f99c52877d65719
+size 6613536806

model-00004-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:070ec1b74e56de7a2c25b993a68f5b47dc3541dc999a68dd603357d1d3323b98
+size 6611149486

model-00005-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53d1da81fc0add15870d77c6820b0db36b4ea0c25e30f3ee21901d32f639dd37
+size 6611152579

model-00006-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74941483a1a84fcab3ec46b6142354238ad91bf0c09700782f758fb64e5e811b
+size 6613543005

model-00007-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10237cb3d2f6a9b6bc4feddb10d181c837f7274eda45ab48a4b19fb8e636991e
+size 6611155672

model-00008-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42b3ffe3e7fbec40e8bf892b0a5ba8b1a10b28e4abf4773788b48c4e085510c5
+size 6611155672

model-00009-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c24818edeb51f28057abcb232ab6fad44933e94115edfb503bedd37fb508881
+size 6613543005

model-00010-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504a0693ce37f002c20e9f0ecbf81a12d443547950b6d4c0799d80a39b0ef997
+size 6611155672

model-00011-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb6b4b7b549592f937286a985814a0305bd3ab20fe70878b896a3b8723965403
+size 6611155672

model-00012-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:654176edda31382f12001024555023264ff157a998aea7c7dc050ef560e3b04f
+size 6613543005

model-00013-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b07fde58523badd56f1fb2ea71c8d93dbae0567b9cf96771f800e4fce1e8ce7
+size 6611155672

model-00014-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d3d5ebae01af1c480cbc21b4e54230a06a53521ec481b8a31ad3dede773eda6
+size 6611155672

model-00015-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c815df50218f15e97371ad7965563af8378f9ecfbf45d284b8f25536eed2a5e
+size 6613543005

model-00016-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:826ece4eeb60c44b88812998d307a7e40fcf4a0437f2babb53201cdb054df9b0
+size 6611155672

model-00017-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:956d917971973e211f3128368eaa42c178ee3824fbfc3a5d29caaece0164616c
+size 6611155672

model-00018-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:700a7579879ca33bba532162ce999fd0ee8649a5db65208726506e768e7af832
+size 6613543005

model-00019-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c91381268219ec82c8771c1644411e64f18a3bdc7b82fb199e2ccfc626023141
+size 6611155672

model-00020-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:121d86aaa9b0547cd68e6fc7e89c1041f5ebde48b752e90af7b29e70e5879cf5
+size 6611155672

model-00021-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:499e3e378115912207a89cc2f07ee7bbd952a14f4e7ea13483e7a404192cb85a
+size 6613543005

model-00022-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816e1a8c012fe45c1365374b04ce8704d28f9e3eb7c2c82d718abe36235add83
+size 6611155672

model-00023-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9b0bb5671a96c959ae40321c50660778ebaabe17d816b3aa99358dd1fc8e79
+size 6611155672

model-00024-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6368e5481852c67d9495de04fdfc8514a971ffacc345e0b1247576aa312bb5e
+size 3776999554

model.safetensors.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fb058ed285464a798f34a53445d24b2b488d3c768574e322c717205607c3021
+size 13521151

modeling_naive_n05_flash.py

@@ -0,0 +1,243 @@
+# Copyright 2026 Naive AI.
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import copy
+
+import torch
+from torch import nn
+from transformers.cache_utils import DynamicCache
+from transformers.conversion_mapping import get_checkpoint_conversion_mapping, register_checkpoint_conversion_mapping
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.deepseek_v3.modeling_deepseek_v3 import DeepseekV3Experts, DeepseekV3TopkRouter
+from transformers.models.deepseek_v3.modeling_deepseek_v3 import DeepseekV3MLP as NaiveN05FlashMLP
+from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXRotaryEmbedding as NaiveN05FlashRotaryEmbedding
+from transformers.models.gpt_neox.modeling_gpt_neox import apply_rotary_pos_emb
+from transformers.models.llama.modeling_llama import LlamaRMSNorm as NaiveN05FlashRMSNorm
+from transformers.models.llama.modeling_llama import repeat_kv
+
+from .configuration_naive_n05_flash import NaiveN05FlashConfig
+
+# Fuse experts weights
+# mlp.experts.{i}.{gate,up,down}_proj.weight[_scale_inv] -> mlp.experts.{gate,up,down}_proj.weight[_scale_inv]
+register_checkpoint_conversion_mapping("naive_n05_flash", get_checkpoint_conversion_mapping("deepseek_v3"), overwrite=True)
+
+
+def round_indexer_fp8(states):
+    states = states.float()
+    scale = states.abs().amax(dim=-1, keepdim=True).clamp_min(1e-4) / 448.0
+    return (states / scale).clamp(-448, 448).to(torch.float8_e4m3fn).float() * scale
+
+
+class NaiveN05FlashMoE(nn.Module):
+    """DeepSeek-V3-style routed experts without a shared expert."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.experts = DeepseekV3Experts(config)
+        self.gate = DeepseekV3TopkRouter(config)
+
+    def forward(self, states):
+        original_shape = states.shape
+        _, topk_weights, topk_indices = self.gate(states)
+        states = states.view(-1, states.shape[-1])
+        states = self.experts(states, topk_indices, topk_weights)
+        return states.view(*original_shape)
+
+
+class NaiveN05FlashIndexer(nn.Module):
+    def __init__(self, config, layer_idx):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.wq = nn.Linear(config.hidden_size, config.index_n_heads * config.index_head_dim, bias=False)
+        self.wk = nn.Linear(config.hidden_size, config.index_head_dim, bias=False)
+        self.k_norm = nn.LayerNorm(config.index_head_dim, eps=1e-5)
+        self.weights_proj = nn.Linear(config.hidden_size, config.index_n_heads, bias=False)
+
+    def forward(self, states, position_embeddings, past_key_values=None):
+        c = self.config
+        batch, length, _ = states.shape
+        query = self.wq(states).view(batch, length, c.index_n_heads, c.index_head_dim).transpose(1, 2)
+        key = self.k_norm(self.wk(states)).unsqueeze(1)
+        query, key = apply_rotary_pos_emb(query, key, *position_embeddings)
+        if c.indexer_activation_dtype == "fp8_e4m3":
+            query, key = round_indexer_fp8(query), round_indexer_fp8(key)
+        if past_key_values is not None:
+            key = past_key_values.update_indexer(key.squeeze(1), self.layer_idx).unsqueeze(1)
+        scores = (query.float() @ key.float().transpose(-1, -2)).relu()
+        weights = self.weights_proj(states) * c.index_n_heads**-0.5
+        return (scores * weights.transpose(1, 2).unsqueeze(-1).float()).sum(dim=1)
+
+
+class NaiveN05FlashAttention(nn.Module):
+    def __init__(self, config, layer_idx):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.is_swa = bool(config.hybrid_layer_pattern[layer_idx])
+        prefix = "swa_" if self.is_swa else ""
+        self.heads = getattr(config, prefix + "num_attention_heads")
+        self.kv_heads = getattr(config, prefix + "num_key_value_heads")
+        self.head_dim = getattr(config, prefix + "head_dim")
+        self.v_dim = getattr(config, prefix + "v_head_dim")
+        rope_config = copy(config)
+        rope_config.head_dim = self.head_dim
+        rope_config.rope_parameters = {
+            "rope_type": "default",
+            "rope_theta": getattr(config, prefix + "rope_theta"),
+            "partial_rotary_factor": config.partial_rotary_factor,
+        }
+        self.rotary_emb = NaiveN05FlashRotaryEmbedding(rope_config)
+        self.q_proj = nn.Linear(config.hidden_size, self.heads * self.head_dim, bias=config.attention_bias)
+        self.k_proj = nn.Linear(
+            config.hidden_size,
+            self.kv_heads * self.head_dim,
+            bias=config.attention_bias,
+        )
+        self.v_proj = nn.Linear(config.hidden_size, self.kv_heads * self.v_dim, bias=config.attention_bias)
+        self.o_proj = nn.Linear(self.heads * self.v_dim, config.hidden_size, bias=False)
+        self.indexer = None if self.is_swa else NaiveN05FlashIndexer(config, layer_idx)
+        sink = config.add_swa_attention_sink_bias if self.is_swa else config.add_full_attention_sink_bias
+        self.attention_sink_bias = nn.Parameter(torch.zeros(self.heads), requires_grad=False) if sink else None
+
+    def forward(self, states, positions, allowed, past_key_values=None):
+        batch, length, _ = states.shape
+        past_length = 0 if past_key_values is None else past_key_values.get_seq_length(self.layer_idx)
+        key_offset = 0 if past_key_values is None else past_key_values.get_mask_sizes(length, self.layer_idx)[1]
+        query = self.q_proj(states).view(batch, length, self.heads, self.head_dim).transpose(1, 2)
+        key = self.k_proj(states).view(batch, length, self.kv_heads, self.head_dim).transpose(1, 2)
+        value = self.v_proj(states).view(batch, length, self.kv_heads, self.v_dim).transpose(1, 2)
+        position_embeddings = self.rotary_emb(states, positions)
+        query, key = apply_rotary_pos_emb(query, key, *position_embeddings)
+        if past_key_values is not None:
+            key, value = past_key_values.update(key, value, self.layer_idx)
+        key_length = key.shape[-2]
+        allowed = allowed[..., key_offset : key_offset + key_length]
+        if self.is_swa:
+            query_offsets = past_length + torch.arange(length, device=states.device)
+            key_offsets = key_offset + torch.arange(key_length, device=states.device)
+            allowed = allowed & (query_offsets[:, None] - key_offsets[None, :] < self.config.sliding_window)
+        else:
+            scores = self.indexer(states, position_embeddings, past_key_values).masked_fill(~allowed, -torch.inf)
+            # Stable ties keep padding from changing which equal-score keys win.
+            selected = scores.argsort(dim=-1, descending=True, stable=True)[..., : self.config.index_top_k]
+            allowed = allowed & torch.zeros_like(scores, dtype=torch.bool).scatter(-1, selected, True)
+        key = repeat_kv(key, self.heads // self.kv_heads)
+        value = repeat_kv(value, self.heads // self.kv_heads)
+        if self.config.attention_value_scale is not None:
+            value = value * self.config.attention_value_scale
+        logits = (query @ key.transpose(-1, -2)) * self.head_dim**-0.5
+        logits = logits.masked_fill(~allowed[:, None], -torch.inf)
+        if self.attention_sink_bias is not None:
+            sink = self.attention_sink_bias[None, :, None, None].expand(batch, -1, length, -1)
+            logits = torch.cat((logits, sink), dim=-1)
+        probabilities = logits.float().softmax(-1).nan_to_num(0.0)[..., :key_length].to(value.dtype)
+        output = (probabilities @ value).transpose(1, 2).reshape(batch, length, -1)
+        return self.o_proj(output)
+
+
+class NaiveN05FlashDecoderLayer(nn.Module):
+    def __init__(self, config, layer_idx):
+        super().__init__()
+        self.self_attn = NaiveN05FlashAttention(config, layer_idx)
+        self.mlp = NaiveN05FlashMoE(config) if config.moe_layer_freq[layer_idx] else NaiveN05FlashMLP(config)
+        self.input_layernorm = NaiveN05FlashRMSNorm(config.hidden_size, config.layernorm_epsilon)
+        self.post_attention_layernorm = NaiveN05FlashRMSNorm(config.hidden_size, config.layernorm_epsilon)
+
+    def forward(self, states, positions, allowed, past_key_values=None):
+        states = states + self.self_attn(self.input_layernorm(states), positions, allowed, past_key_values)
+        return states + self.mlp(self.post_attention_layernorm(states))
+
+
+class NaiveN05FlashModel(PreTrainedModel):
+    config_class = NaiveN05FlashConfig
+    base_model_prefix = "model"
+    _can_set_experts_implementation_cached_value = True
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.layers = nn.ModuleList([NaiveN05FlashDecoderLayer(config, i) for i in range(config.num_hidden_layers)])
+        self.norm = NaiveN05FlashRMSNorm(config.hidden_size, config.layernorm_epsilon)
+
+    def forward(self, states, positions, allowed, past_key_values=None):
+        for layer in self.layers:
+            states = layer(states, positions, allowed, past_key_values)
+        return self.norm(states)
+
+
+class NaiveN05FlashForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = NaiveN05FlashConfig
+    base_model_prefix = "model"
+    _no_split_modules = ["NaiveN05FlashDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _keep_in_fp32_modules_strict = ["mlp.gate.weight", "mlp.gate.e_score_correction_bias"]
+    _can_set_experts_implementation_cached_value = True
+
+    def __init__(self, config):
+        config.validate()
+        super().__init__(config)
+        self.model = NaiveN05FlashModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+        self.eval().requires_grad_(False)
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    @torch.inference_mode()
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        inputs_embeds=None,
+        use_cache=None,
+        past_key_values=None,
+        **kwargs,
+    ):
+        use_cache = self.config.use_cache if use_cache is None else use_cache
+        if past_key_values is not None and not use_cache:
+            raise ValueError("Pass use_cache=True when providing past_key_values")
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+        if past_key_values is not None and not isinstance(past_key_values, DynamicCache):
+            raise ValueError("NaiveN05Flash supports DynamicCache only")
+        if (input_ids is None) == (inputs_embeds is None):
+            raise ValueError("Provide exactly one of input_ids or inputs_embeds")
+        states = self.model.embed_tokens(input_ids) if inputs_embeds is None else inputs_embeds
+        batch, length, _ = states.shape
+        past_length = 0 if past_key_values is None else past_key_values.get_seq_length()
+        total_length = past_length + length
+        if attention_mask is None:
+            attention_mask = torch.ones(batch, total_length, device=states.device, dtype=torch.bool)
+        if attention_mask.shape != (batch, total_length):
+            raise ValueError("Expected a 2-D attention_mask covering the cached prefix and new tokens")
+        if position_ids is None:
+            position_ids = (attention_mask.long().cumsum(-1) - 1).clamp_min(0)[:, -length:]
+        query_offsets = past_length + torch.arange(length, device=states.device)
+        key_offsets = torch.arange(total_length, device=states.device)
+        allowed = query_offsets[:, None] >= key_offsets[None, :]
+        allowed = allowed[None] & attention_mask[:, None].bool()
+        logits = self.lm_head(self.model(states, position_ids, allowed, past_key_values))
+        return CausalLMOutputWithPast(logits=logits, past_key_values=past_key_values)
+
+
+__all__ = ["NaiveN05FlashForCausalLM", "NaiveN05FlashModel"]

quantization_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7110f110f3a2df203835d6c95f4d4cd145bf6536bfe3b527d91e60f89b8d1a
+size 44681475

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|endoftext|>",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<|audio_pad|>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,248 @@
+{
+  "_comment": "Tokenizer copied from the open-source MiMo 2.5",
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<|audio_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<|audio_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 1048576,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

vocab.README.md

@@ -0,0 +1,3 @@
+# vocab.json
+
+Tokenizer copied from the open-source MiMo 2.5