Zero-Shot Classification
Transformers
qwen3_5
feature-extraction
decision-model
system-one
semantic-router
vela
token-classification
named-entity-recognition
relation-extraction
pii-detection
hallucination-detection
content-safety
multimodal
custom_code
Eval Results
Instructions to use vllm-sr/vela3-turbo with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use vllm-sr/vela3-turbo with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("zero-shot-classification", model="vllm-sr/vela3-turbo", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModel processor = AutoProcessor.from_pretrained("vllm-sr/vela3-turbo", trust_remote_code=True) model = AutoModel.from_pretrained("vllm-sr/vela3-turbo", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Commit ·
c39e558
0
Parent(s):
Vela 3.0
Browse files- .gitattributes +38 -0
- EVALUATION.md +195 -0
- LICENSE +201 -0
- README.md +275 -0
- SHA256SUMS +19 -0
- assets/banner.png +3 -0
- assets/family.png +3 -0
- assets/glance.png +3 -0
- calibration.json +1677 -0
- config.json +148 -0
- decision_df_lora.safetensors +3 -0
- decision_lora.safetensors +3 -0
- label_definitions.json +13 -0
- modeling_vela3.py +471 -0
- ner_head.safetensors +3 -0
- ner_lora.safetensors +3 -0
- relation_head.safetensors +3 -0
- relation_lora.safetensors +3 -0
- router_questions.json +220 -0
- span_decode_v2.py +272 -0
- span_head.safetensors +3 -0
- span_lora.safetensors +3 -0
- vela3_adapters.py +87 -0
- vela3_config.json +302 -0
- vela3_labeldef.py +144 -0
- vela3_span.py +232 -0
.gitattributes
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
assets/banner.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
assets/family.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
assets/glance.png filter=lfs diff=lfs merge=lfs -text
|
EVALUATION.md
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# vela3-turbo (27B): evaluation
|
| 2 |
+
|
| 3 |
+
All numbers were measured through this package, with `system_one`, one request at a time and the shipped thresholds, on an AMD Instinct MI325X (20 Oct 2026; bitwise check and latency 21 Oct). The base is vllm-sr/d3-turbo at revision `da17b5d0c860f7e5be6e2afebe67228b4bd24d0c`. Other models' numbers are as reported by their authors.
|
| 4 |
+
|
| 5 |
+
## 1. Public test sets: unsupported claims and held-out decisions
|
| 6 |
+
|
| 7 |
+
Vela 2.0 9B values are as published on its model card ([vllm-sr/Vela-2.0-9B](https://huggingface.co/vllm-sr/Vela-2.0-9B), EVALUATION.md; PII with its shipped calibration). We hold no per-row outputs of its release run, so this comparison has no paired differences or confidence intervals; in the README, bold marks a lead of at least 0.02 over the published value.
|
| 8 |
+
|
| 9 |
+
| benchmark | metric | vela3-turbo | best other model | Vela 2.0 9B (published) | n |
|
| 10 |
+
|---|---|---:|---|---:|---:|
|
| 11 |
+
| Hallucination spans, 10,698 examples | example-F1 | 0.900 | LettuceDetect v2 qwen-2b 0.921 | 0.885 | 10698 |
|
| 12 |
+
| RAGTruth | example-F1 | 0.779 | HDM-2 (3B) 0.850 | 0.774 | 2700 |
|
| 13 |
+
| MASSIVE unseen scenarios, intent | accuracy | 0.833 | – | 0.763 | 14911 |
|
| 14 |
+
| MASSIVE unseen scenarios, scenario | accuracy | 0.937 | – | 0.871 | 14911 |
|
| 15 |
+
| Super-NI task 1568 (held out) | accuracy | 0.636 | – | 0.574 | 162 |
|
| 16 |
+
|
| 17 |
+
MASSIVE: three scenarios never used in our router training (iot, transport, cooking), 52 locales; accuracy on the 14,911 test utterances that match no training text of the model's lineage. The 342 excluded utterances occur verbatim in upstream training mixtures (tasksource MASSIVE / HWU64 rows). Vela 2.0 values are on the full 15,253, so these rows are not bolded.
|
| 18 |
+
|
| 19 |
+
QASC omitted: its items are in the d3 training mixture. Super-NI 1568: 79 to 90 of its 162 ProPara passages share a 13-gram with other tasks' training rows.
|
| 20 |
+
|
| 21 |
+
Best other model, same test set and metric only (– where no such number exists): Hallucination spans, 10,698 examples: LettuceDetect v2 qwen-2b 0.921 (reported, KRLabsOrg/lettucedect-v2-qwen-2b card (unified test set, 10,698, example-F1)); RAGTruth: HDM-2 (3B) 0.850 (reported, HalluciNot arXiv 2504.07069 Table 2, example-level F1 on the RAGTruth test set; its training set includes RAGTruth contexts. Next: RAG-HAT 0.839, EMNLP 2024 Industry Table 2). Also reported on RAGTruth: LettuceDetect v2 qwen-2b 0.818 and mmBERT-base 0.743 (their cards).
|
| 22 |
+
|
| 23 |
+
### Router signals: public router signal suite
|
| 24 |
+
|
| 25 |
+
[vllm-sr/router-signal-suite](https://huggingface.co/datasets/vllm-sr/router-signal-suite) at `fa08b2a6`, scored with the dataset's own scoring code, one `system_one` call per row with the shipped thresholds. Every test file of the 8 router signals this package was trained on (supervised, as in Vela 2.0). Vela 2.0 9B was run by us through its public release on the same rows; diff = ours minus Vela 2.0 with the 95% paired cluster bootstrap CI (2,000 rounds). Decision-1.0-Route-0.6B is scored from the suite's published predictions (trained on the suite's train split). No-overlap rows: the test rows that share no text (exact or a 13-gram) with this model's training data.
|
| 26 |
+
|
| 27 |
+
| signal | file | metric | rows | vela3-turbo | Vela 2.0 9B | diff [95% CI] | Decision-1.0-Route-0.6B | no-overlap rows: vela3-turbo (rows) | no-overlap diff [95% CI] |
|
| 28 |
+
|---|---|---|---:|---:|---:|---|---:|---:|---|
|
| 29 |
+
| Harmful request | test | AUC | 2,230 | 0.958 | 0.957 | +0.001 [-0.006, +0.008] | 0.963 | 0.953 (1,254) | -0.005 [-0.015, +0.005] |
|
| 30 |
+
| Harmful request | test-aegis2 | AUC | 1,749 | 0.939 | 0.935 | +0.004 [-0.004, +0.011] | 0.927 | 0.931 (1,120) | +0.005 [-0.005, +0.015] |
|
| 31 |
+
| Harmful request | test-nemotron-v3 | AUC | 2,000 | 0.915 | 0.913 | +0.002 [-0.009, +0.012] | 0.902 | 0.917 (1,843) | +0.003 [-0.008, +0.015] |
|
| 32 |
+
| Harmful request | test-polyguardprompts | AUC | 2,000 | 0.961 | 0.941 | +0.021 [+0.012, +0.029] | 0.894 | 0.962 (1,955) | +0.020 [+0.011, +0.029] |
|
| 33 |
+
| Harmful request | test-wildguardmix | AUC | 1,696 | 0.965 | 0.957 | +0.008 [+0.002, +0.014] | 0.927 | 0.965 (1,671) | +0.008 [+0.002, +0.014] |
|
| 34 |
+
| Jailbreak or prompt injection | test | AUC | 1,878 | 0.967 | 0.919 | +0.047 [+0.037, +0.060] | 0.991 | 0.965 (944) | +0.063 [+0.048, +0.077] |
|
| 35 |
+
| Personal data in the request | test | AUC | 3,000 | 0.971 | 0.960 | +0.011 [+0.004, +0.019] | 0.994 | 0.973 (2,802) | +0.013 [+0.005, +0.021] |
|
| 36 |
+
| Personal data in the request | test-abcd | AUC | 1,486 | 0.990 | 0.897 | +0.093 [+0.075, +0.112] | 0.999 | 0.988 (1,190) | +0.103 [+0.084, +0.123] |
|
| 37 |
+
| Personal data in the request | test-mapa | AUC | 1,679 | 0.934 | 0.787 | +0.147 [+0.037, +0.187] | 0.984 | 0.934 (1,678) | +0.147 [+0.043, +0.187] |
|
| 38 |
+
| Personal data in the request | test-tab | AUC | 1,461 | 0.987 | 0.977 | +0.010 [+0.002, +0.020] | 0.994 | 0.985 (1,372) | +0.013 [+0.003, +0.024] |
|
| 39 |
+
| Answer not supported by its context | test | AUC | 3,000 | 0.783 | 0.765 | +0.018 [+0.004, +0.032] | 0.791 | 0.727 (1,743) | -0.017 [-0.036, +0.002] |
|
| 40 |
+
| Answer not supported by its context | test-psiloqa | AUC | 1,265 | 0.907 | 0.862 | +0.045 [+0.025, +0.065] | 0.872 | 0.918 (1,146) | +0.041 [+0.020, +0.061] |
|
| 41 |
+
| Answer not supported by its context | test-ragbench | AUC | 1,311 | 0.755 | 0.721 | +0.034 [-0.001, +0.069] | 0.628 | 0.753 (1,223) | +0.032 [-0.004, +0.066] |
|
| 42 |
+
| Answer not supported by its context | test-ragtruth | AUC | 1,528 | 0.894 | 0.865 | +0.028 [+0.009, +0.048] | 0.669 | 0.893 (1,209) | +0.029 [+0.005, +0.052] |
|
| 43 |
+
| Fact-check needed | test | AUC | 800 | 0.870 | 0.809 | +0.062 [+0.042, +0.083] | 0.942 | 0.863 (690) | +0.058 [+0.036, +0.080] |
|
| 44 |
+
| Output modality (text or image) | test | AUC | 3,000 | 0.993 | 0.933 | +0.060 [+0.052, +0.069] | 0.999 | 0.993 (2,916) | +0.063 [+0.054, +0.072] |
|
| 45 |
+
| Output modality (text or image) | test-dolly | specificity at dev threshold | 2,000 | 0.999 | 1.000 | -0.001 (no CI: one-class file) | 0.965 | 0.998 (1,950) | – |
|
| 46 |
+
| Output modality (text or image) | test-realedit | recall at dev threshold | 2,000 | 0.986 | 0.417 | +0.569 (no CI: one-class file) | 1.000 | 0.986 (2,000) | – |
|
| 47 |
+
| Domain (subject area) | test | accuracy | 2,996 | 0.884 | 0.765 | +0.119 [+0.104, +0.135] | 0.863 | 0.885 (2,916) | +0.123 [+0.107, +0.137] |
|
| 48 |
+
| Domain (subject area) | test-exams | accuracy | 1,927 | 0.830 | 0.791 | +0.039 [+0.024, +0.056] | 0.815 | 0.830 (1,907) | +0.039 [+0.023, +0.055] |
|
| 49 |
+
| User feedback on the previous answer | test | accuracy | 2,842 | 0.852 | 0.666 | +0.186 [+0.166, +0.206] | 0.848 | 0.843 (2,295) | +0.212 [+0.188, +0.236] |
|
| 50 |
+
| User feedback on the previous answer | test-sgd | accuracy | 1,998 | 0.966 | 0.807 | +0.159 [+0.141, +0.177] | 0.973 | 0.963 (1,794) | +0.172 [+0.153, +0.191] |
|
| 51 |
+
|
| 52 |
+
Domain and feedback are also trained on the suite's own train split (with the public router data Vela 2.0 used; the suite's test and hold rows, and anything sharing an exact text or a 13-gram with them, are kept out), as Decision-1.0-Route is, so their test files are in-distribution for both. The domain question has a 15th option, religion, which the suite maps to philosophy. Feedback has NO_FEEDBACK as a trained option. Every other signal answers exactly as before (identity check on dev rows).
|
| 53 |
+
Not shown: hazard and tool need, which have no trained question in this package; the hold files (no Vela 2.0 pairing).
|
| 54 |
+
|
| 55 |
+
## Hallucination by source (example-F1)
|
| 56 |
+
|
| 57 |
+
- lettucedetect-acl: n=440 ours 0.971
|
| 58 |
+
- lettucedetect-code-agent: n=2015 ours 0.792
|
| 59 |
+
- lettucedetect-readme: n=641 ours 0.977
|
| 60 |
+
- lettucedetect-tool-output: n=617 ours 0.897
|
| 61 |
+
- lettucedetect-wikipedia: n=1388 ours 0.977
|
| 62 |
+
- psiloqa: n=2897 ours 0.964
|
| 63 |
+
- ragtruth: n=2700 ours 0.779
|
| 64 |
+
|
| 65 |
+
Long answers are read in windows of at most 1,800 tokens (stride 1,536) when they exceed 2,048 tokens; every window sees the whole context.
|
| 66 |
+
|
| 67 |
+
## 2. Package checks
|
| 68 |
+
|
| 69 |
+
- **Non-router questions vs plain d3-turbo** (revision `da17b5d0`, the snapshot users download; 260 requests, the request set of the family's bitwise check (seed 0, 200 zero-shot + 60 mixed requests with router questions), exact equality of every returned answer and probability; d3-turbo's runtime does not expose per-pass logits): answers identical on 561 / 561 non-router questions; router answers changed by the decision adapter 92 / 92.
|
| 70 |
+
- **Span answers, shipped runtime vs training runtime:** identical on 6 / 6 requests.
|
| 71 |
+
- **Image path:** no image adapter at this size. Box and track questions, assets, videos and long-context requests are answered by d3-turbo itself, adapters off.
|
| 72 |
+
- **Files:** `SHA256SUMS` covers the package files (adapters, heads, code, configs, licence).
|
| 73 |
+
|
| 74 |
+
## 3. Zero-shot NER, OOD-7
|
| 75 |
+
|
| 76 |
+
Strict entity micro-F1 per set, average of the 7 sets.
|
| 77 |
+
|
| 78 |
+
| mode | avg | 95% CI | AI | Lit | Music | Pol | Sci | Movie | Rest |
|
| 79 |
+
|---|---|---|---|---|---|---|---|---|---|
|
| 80 |
+
| label names only (default; span adapter) | **64.58** | 63.73-65.44 | 66.8 | 61.2 | 73.1 | 66.5 | 70.9 | 65.0 | 48.5 |
|
| 81 |
+
|
| 82 |
+
Reported on the same 7 sets with label names only: B2NER-20B, standardised label names 72.1 (B2NER, arXiv 2406.11192 Table 3; the best reported entry of this size class), GLiFormer-base 60.40, GLiFormer-large 59.05 (GLiFormer paper). Vela 2.0 9B, names only: 43.5 (published).
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
Label-name overlap: no CrossNER or MIT text is in the training data, but 35 of CrossNER's 41 type names (mostly generic types such as person, organisation, location and country) also occur as label names in the NER training data.
|
| 86 |
+
|
| 87 |
+
## 4. NER, 26 IE_INSTRUCTIONS datasets
|
| 88 |
+
|
| 89 |
+
Label names only, mean F1 **63.7**. Not all 26 sets are zero-shot.
|
| 90 |
+
|
| 91 |
+
| ACE_2004 | ACE_2005 | AnatEM | Broad_Tweet_Corpus | CoNLL_2003 | CrossNER_AI | CrossNER_literature | CrossNER_music | CrossNER_politics | CrossNER_science | FabNER | FindVehicle | GENIA_NER |
|
| 92 |
+
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
| 93 |
+
| 24.9 | 22.7 | 81.7 | 79.0 | 77.3 | 66.8 | 61.2 | 73.1 | 66.5 | 70.9 | 54.1 | 93.0 | 69.6 |
|
| 94 |
+
|
| 95 |
+
| HarveyNER | MultiNERD | Ontonotes | PolyglotNER | TweetNER7 | WikiANN_en | WikiNeural | bc2gm | bc4chemd | bc5cdr | mit-movie | mit-restaurant | ncbi |
|
| 96 |
+
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
| 97 |
+
| 43.6 | 42.3 | 62.6 | 51.1 | 38.6 | 69.3 | 86.4 | 70.9 | 78.9 | 83.6 | 65.0 | 48.5 | 74.3 |
|
| 98 |
+
|
| 99 |
+
## 5. Zero-shot classification (GLiFormer paper suite)
|
| 100 |
+
|
| 101 |
+
Label names only, no tuning. These questions run on d3-turbo with every adapter off. GLiFormer-large numbers as reported in the GLiFormer paper (Table 5).
|
| 102 |
+
|
| 103 |
+
| dataset | n | vela3-turbo | GLiFormer-large |
|
| 104 |
+
|---|---:|---:|---:|
|
| 105 |
+
| CR | 376 | 93.7 | 90.5 |
|
| 106 |
+
| SST-2 | 1,821 | 96.3 | 93.0 |
|
| 107 |
+
| SST-5 | 2,210 | 44.3 | 40.3 |
|
| 108 |
+
| IMDB | 25,000 | 96.5 | 93.9 |
|
| 109 |
+
| 20 Newsgroups | 7,532 | 71.5 | 57.2 |
|
| 110 |
+
| Enron spam | 2,000 | 90.8 | 98.0 |
|
| 111 |
+
| MASSIVE | 2,974 | 83.7 | 70.0 |
|
| 112 |
+
| Banking77 | 3,076 | 93.5 | 70.6 |
|
| 113 |
+
| Financial PhraseBank | 2,264 | 94.9 | 96.8 |
|
| 114 |
+
| AG News | 7,600 | 90.1 | 81.5 |
|
| 115 |
+
| Emotion | 2,000 | 52.6 | 48.1 |
|
| 116 |
+
| CAP SOTU | 23,040 | 58.7 | 49.0 |
|
| 117 |
+
| Rotten Tomatoes | 1,066 | 91.4 | 86.7 |
|
| 118 |
+
| **mean** | | **81.4** | **75.0** |
|
| 119 |
+
|
| 120 |
+
Reported means over the same 13 sets: GPT-5-mini 79.8, Gemma-4-31B-IT 79.7, GPT-OSS-120B 77.8, GLiFormer-large 75.0, GLiClass Multilang ultra 71.0, GLiNER2 large v1 66.2, GLiNER2.5 64.9.
|
| 121 |
+
|
| 122 |
+
- The Banking77 copy has 3,076 rows; the paper has 3,080.
|
| 123 |
+
- Financial PhraseBank is the AllAgree subset (2,264 rows); the paper reports 1,129 rows.
|
| 124 |
+
|
| 125 |
+
## 6. Relation extraction
|
| 126 |
+
|
| 127 |
+
**Head-conditioned** (gold head given; tail micro-F1):
|
| 128 |
+
|
| 129 |
+
| set | mode | F1 | 95% CI |
|
| 130 |
+
|---|---|---:|---|
|
| 131 |
+
| CoNLL04 | label names (default) | 51.7 | 47.1-56.1 |
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
Vela 2.0 9B: 10.1 on CoNLL04, label names (published).
|
| 135 |
+
|
| 136 |
+
**End to end, GLiFormer-paper protocol** (entities predicted; strict micro-F1)
|
| 137 |
+
|
| 138 |
+
| set | mode | strict F1 | P / R |
|
| 139 |
+
|---|---|---:|---|
|
| 140 |
+
| CoNLL04 | per head | 42.8 | 38.0 / 48.9 |
|
| 141 |
+
| CoNLL04 | per (head, relation) | 40.4 | 33.0 / 52.1 |
|
| 142 |
+
| CrossRE (6 domains) | per head | 39.2 | 45.1 / 34.6 |
|
| 143 |
+
| CrossRE | per (head, relation) | 32.3 | 27.3 / 39.6 |
|
| 144 |
+
| DocRED (validation) | per head | 33.7 | 29.8 / 38.9 |
|
| 145 |
+
| FewRel (split 0) | per head | 24.0 | 15.6 / 51.4 |
|
| 146 |
+
| FewRel (split 0) | per (head, relation) | 27.8 | 21.2 / 40.1 |
|
| 147 |
+
|
| 148 |
+
The README table uses the GLiFormer Table 4 protocol: per head for CoNLL04, DocRED and CrossRE, per (head, relation) for FewRel. Reported (CoNLL04 / DocRED / FewRel / CrossRE / avg): GLiNER-Relex 40.4 / 31.3 / 12.5 / 18.1 / 25.6; Gemma-4-31B-IT 42.3 / 18.0 / 30.2 / 9.8 / 25.1; GPT-5-mini 42.4 / 18.6 / 15.0 / 12.4 / 22.1; GLiFormer-large 35.8 / 12.8 / 24.1 / 12.6 / 21.3; GLiNER2 32.9 / 11.7 / 20.8 / 6.0 / 17.8. FewRel split 0 here has 3,423 sentences, and CrossRE covers all six domains.
|
| 149 |
+
|
| 150 |
+
**End to end, typed and generic sets** (strict triplet micro-F1)
|
| 151 |
+
|
| 152 |
+
| CoNLL04 | SciERC | NYT11 | FewRel-0 | avg |
|
| 153 |
+
|---:|---:|---:|---:|---:|
|
| 154 |
+
| 42.8 | 0.2 | 0.5 | 24.0 | 16.9 |
|
| 155 |
+
|
| 156 |
+
SciERC, NYT11 and FewRel use a generic `entity` type. CoNLL04 is zero-shot; the DocRED, CrossRE and FewRel train splits are in the relation training data (their test sets and FewRel's evaluation relations are held out).
|
| 157 |
+
|
| 158 |
+
## 7. Evidence
|
| 159 |
+
|
| 160 |
+
| benchmark | word-F1 | 95% CI |
|
| 161 |
+
|---|---:|---|
|
| 162 |
+
| QASPER evidence (1,141 questions, whole papers; zero-shot) | **39.3** | 37.3-41.4 |
|
| 163 |
+
| ACL-Verbatim evidence (100 rows; in-domain) | **49.1** | 34.6-61.8 |
|
| 164 |
+
|
| 165 |
+
ACL-Verbatim is in-domain: its silver training split is in the training data.
|
| 166 |
+
|
| 167 |
+
## 8. Image-grounded hallucination spans
|
| 168 |
+
|
| 169 |
+
No image adapter at this size: there is no image-grounded hallucination result.
|
| 170 |
+
|
| 171 |
+
## 9. Latency (AMD Instinct MI325X, batch 1, measured alone on one GPU, no other process on it)
|
| 172 |
+
|
| 173 |
+
| case | 200 tokens | 1K tokens | 3K tokens |
|
| 174 |
+
|---|---:|---:|---:|
|
| 175 |
+
| 4 router decisions, adapter off | 179.4 ms (p90 180.3) | 258.2 ms (p90 264.1) | 461.5 ms (p90 465.9) |
|
| 176 |
+
| 4 router decisions, adapter on | 468.8 ms (p90 469.8) | 951.5 ms (p90 957.0) | 2237.6 ms (p90 2244.1) |
|
| 177 |
+
| 1 span (10 labels, names only) | 83.5 ms (p90 83.7) | 181.6 ms (p90 187.1) | 530.7 ms (p90 533.8) |
|
| 178 |
+
| 4 decisions + 1 span | 565.7 ms (p90 585.6) | 1137.5 ms (p90 1143.2) | 2767.7 ms (p90 2773.7) |
|
| 179 |
+
|
| 180 |
+
- Multi-passage answer spans (~3K tokens, 100 questions): mean 560 ms, p50 548 ms, p95 659 ms.
|
| 181 |
+
|
| 182 |
+
## 10. Decisions (Decision Index 0.4.1, 2026-10-21)
|
| 183 |
+
|
| 184 |
+
| model | public index | full score | full-score rank | note |
|
| 185 |
+
|---|---:|---:|---:|---|
|
| 186 |
+
| Fastino GLiDE V1.2 | 71.99 | 71.52 | 1 | |
|
| 187 |
+
| Perplexity Decider v1.2 (27B) | 67.74 | 67.98 | 4 | tied rank |
|
| 188 |
+
| vLLM-SR d3-turbo v1.1 (27B) | 71.62 | 67.31 | 10 | the public d3-turbo main; not this package's base; tied rank |
|
| 189 |
+
| vLLM-SR d3 (27B) | 65.45 | 67.15 | 11 | tied rank |
|
| 190 |
+
| OneJev1.6-0.8B | 48.76 | 29.58 | 155 | tied rank |
|
| 191 |
+
| StartLux-Decision v1.1 0.8B | 53.79 | 28.88 | 158 | tied rank |
|
| 192 |
+
| **General decisions = d3-turbo** | – | – | – | d3-turbo v1.0, revision da17b5d0: not on the 0.4.1 board |
|
| 193 |
+
| GLiClass v4 Edge (0.4B) | 47.49 | – | – | own card, public part; not on the board |
|
| 194 |
+
|
| 195 |
+
General decisions are the base model's own answers, bit for bit; no Vela package was run on the board. Full score (the board's ranking): 60% private same-skill tests and 40% private new-domain tasks; public index: the 37 public benchmarks, chance-corrected. All values reported on the Decision Index 0.4.1 board (2026-10-21; multimodalart/jev-decision-index, data generated 2026-10-22) unless noted; tied ranks share a number. d3-turbo v1.0, this package's base (revision `da17b5d0`), is not on the Decision Index 0.4.1 board; the public d3-turbo main is v1.1, which is on the board at 67.31 (rank 10, tied).
|
LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
pipeline_tag: zero-shot-classification
|
| 4 |
+
library_name: transformers
|
| 5 |
+
base_model:
|
| 6 |
+
- vllm-sr/d3-turbo
|
| 7 |
+
base_model_relation: adapter
|
| 8 |
+
tags:
|
| 9 |
+
- decision-model
|
| 10 |
+
- system-one
|
| 11 |
+
- semantic-router
|
| 12 |
+
- vela
|
| 13 |
+
- zero-shot-classification
|
| 14 |
+
- token-classification
|
| 15 |
+
- named-entity-recognition
|
| 16 |
+
- relation-extraction
|
| 17 |
+
- pii-detection
|
| 18 |
+
- hallucination-detection
|
| 19 |
+
- content-safety
|
| 20 |
+
- multimodal
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+

|
| 24 |
+
|
| 25 |
+
<p align="center"><b>Name it. Vela finds it.</b><br>
|
| 26 |
+
<sub>vLLM Semantic Router × KR Labs · Vela 3.0 family · <b>vela3-turbo</b> · 27B · base <code>d3-turbo</code> · Apache-2.0</sub></p>
|
| 27 |
+
|
| 28 |
+
<p align="center">
|
| 29 |
+
<a href="#what-it-does">What it does</a> ·
|
| 30 |
+
<a href="#quick-start">Quick start</a> ·
|
| 31 |
+
<a href="#evaluation">Evaluation</a> ·
|
| 32 |
+
<a href="#the-family">The family</a> ·
|
| 33 |
+
<a href="https://huggingface.co/collections/vllm-sr/vela-30-6ad1c430eb1718eb4754edcd">📡 Collection</a>
|
| 34 |
+
</p>
|
| 35 |
+
|
| 36 |
+
---
|
| 37 |
+
|
| 38 |
+
**vela3-turbo** is the 27B member of the Vela 3.0 family, built on [d3-turbo](https://huggingface.co/vllm-sr/d3-turbo). In one `system_one` call it answers routing and general decisions and finds the exact spans you name (entities, relations, personal data, unsupported claims, evidence), with probabilities and character offsets and no generated text.
|
| 39 |
+
|
| 40 |
+
| | |
|
| 41 |
+
| --- | --- |
|
| 42 |
+
| **Parameters** | 26.09B with vision encoder (base) + 5044 MB of adapters and heads |
|
| 43 |
+
| **Base** | [vllm-sr/d3-turbo](https://huggingface.co/vllm-sr/d3-turbo) **v1.0**, revision `da17b5d0c860f7e5be6e2afebe67228b4bd24d0c`, unchanged (the public d3-turbo main is now v1.1; this package stays on v1.0) |
|
| 44 |
+
| **Inputs** | Text or JSON, plus images and videos for d3-turbo's own question types |
|
| 45 |
+
| **Answers** | Choice · Yes / No · Score · Span |
|
| 46 |
+
| **Licence** | Apache-2.0 |
|
| 47 |
+
|
| 48 |
+
## At a glance
|
| 49 |
+
|
| 50 |
+
| | |
|
| 51 |
+
| --- | --- |
|
| 52 |
+
| **What it is** | A 26B decision-and-extraction model for LLM routing and guardrails. It runs [vllm-sr/d3-turbo](https://huggingface.co/vllm-sr/d3-turbo), the largest d3 decision model, pinned to d3-turbo v1.0 (revision da17b5d0; the public main is now v1.1), unchanged, and adds the exact spans you name: entities, relations, personal data, unsupported claims and evidence. |
|
| 53 |
+
| **How it was trained** | d3-turbo stays frozen, so every ordinary decision is d3-turbo's own answer, bit for bit. Vela adds small gated LoRA adapters on top, one per job: spans (unsupported claims, personal data, evidence), entities, relations and router signals. Each adapter switches on only for its own questions. The adapters were trained on vela3-flash's recipe and ported to d3-turbo: span, entity and router adapters re-thresholded, the relation adapter retrained. Router signals are trained supervised, as in Vela 2.0, on the public router datasets Vela 2.0 used (for example AEGIS 2.0, PolyGuardMix and Nemotron-Safety-Guard v3 for safety, Global-MMLU for domain, WildFeedback for feedback, DiffusionDB for modality, MASSIVE for intents); domain and feedback also on the public router signal suite's train split. Other training data: public NER, relation, hallucination, evidence and PII datasets, plus web and document text labelled by a large teacher model. No test split and no Decision Index item is trained on; some router-signal-suite test rows also occur in public training sources, so the router table also reports the rows without that overlap. |
|
| 54 |
+
| **What it is good at** | Zero-shot classification; zero-shot relations; decisions, bit for bit; router signals above Vela 2.0 9B on 7 of 8 signals of the public router signal suite. Pick it for the most accurate answers, as the family's reference. |
|
| 55 |
+
| **How we measure it** | Public test sets only, run through this package one request at a time with the shipped thresholds: CoNLL04, CoNLL04 · DocRED · FewRel · CrossRE (avg), OOD-7 (CrossNER + MIT), RAGTruth, Super-NI task 1568, the 13-set GLiFormer classification suite, the LettuceDetect v2 test set, the public router signal suite, the Decision Index 0.4.1. Other models' numbers are as their authors report them on the same test set and metric, or scored by us on the same rows where a table says so. |
|
| 56 |
+
|
| 57 |
+
## Why use it
|
| 58 |
+
|
| 59 |
+
- **Zero-shot classification:** 81.4 on the 13-set GLiFormer classification suite ✦, beats Gemma-4-31B-IT (31B): 79.69 (reported, GLiFormer paper Tab 5).
|
| 60 |
+
- **Zero-shot relations:** 35.9 on CoNLL04 · DocRED · FewRel · CrossRE (avg) ✦, beats Gemma-4-31B-IT, prompted (31B): 25.08 (reported, GLiFormer paper Tab 4).
|
| 61 |
+
- **Decisions, bit for bit:** every non-router question is answered by `d3-turbo@da17b5d0` itself (561 of 561 answers identical in our check).
|
| 62 |
+
|
| 63 |
+

|
| 64 |
+
<sub>vela3-turbo (27B) against reported numbers. Accent = vela3-turbo; grey = other models (reported).</sub>
|
| 65 |
+
|
| 66 |
+
## What it does
|
| 67 |
+
|
| 68 |
+
| Capability | Ask (one question of a `system_one` call) | Get |
|
| 69 |
+
| --- | --- | --- |
|
| 70 |
+
| **Decisions** | `{"type": "choice", "instructions": "Which topic?", "criteria": {"billing": None, "tech": None}}` (also `noul`, `score`) | a probability per option, from d3-turbo itself |
|
| 71 |
+
| **Router signals** | a built-in router question id from `router_questions.json` | the signal's answer from the gated decision adapter |
|
| 72 |
+
| **Named entities** | `{"type": "span", "criteria": ["drug", "dosage"]}` | spans with label, offsets and score |
|
| 73 |
+
| **Relations** | `{"type": "span", "instructions": "Which spans are the founder of Apple?", "criteria": {"founder": None}}` | the related spans |
|
| 74 |
+
| **Personal data** | `{"type": "span", "criteria": ["PERSON", "EMAIL_ADDRESS"]}` | spans of the 17 trained PII types, any subset |
|
| 75 |
+
| **Unsupported claims** | state `{"context": ..., "answer": ...}` and `{"type": "span", "criteria": {"unsupported": None}}` | the words of the answer the context does not support |
|
| 76 |
+
| **Evidence** | `{"type": "span", "criteria": ["evidence"]}` | the sentences that support the answer |
|
| 77 |
+
| **Boxes, tracks, images in questions** | d3-turbo's `box` / `track` questions, `assets` | answered by d3-turbo itself, adapters off ([d3-turbo card](https://huggingface.co/vllm-sr/d3-turbo)) |
|
| 78 |
+
|
| 79 |
+
## Quick start
|
| 80 |
+
|
| 81 |
+
```python
|
| 82 |
+
# pip install "transformers==5.17.0" torch torchvision pillow opencv-python-headless safetensors accelerate
|
| 83 |
+
from transformers import AutoModel
|
| 84 |
+
|
| 85 |
+
m = AutoModel.from_pretrained("vllm-sr/vela3-turbo", trust_remote_code=True) # downloads d3-turbo at the pinned revision
|
| 86 |
+
|
| 87 |
+
# Name any labels; the spans come back with offsets and scores.
|
| 88 |
+
r = m.system_one(
|
| 89 |
+
state="Geoffrey Hinton presented capsule networks at NeurIPS in Montreal.",
|
| 90 |
+
questions={"ents": {"type": "span", "criteria": ["researcher", "algorithm", "conference", "location"]}})
|
| 91 |
+
# researcher "Geoffrey Hinton", algorithm "capsule networks", conference "NeurIPS", location "Montreal"
|
| 92 |
+
|
| 93 |
+
# Unsupported claims in an answer, against a context or an image.
|
| 94 |
+
r = m.system_one(
|
| 95 |
+
state={"context": "The Eiffel Tower is 330 metres tall and was completed in 1889.",
|
| 96 |
+
"answer": "The Eiffel Tower is 330 metres tall and was completed in 1925 by Gustave Eiffel's son."},
|
| 97 |
+
questions={"halu": {"type": "span", "criteria": {"unsupported": None}}})
|
| 98 |
+
# flags "1925 by Gustave Eiffel's son"; add images=["photo.jpg"] to check an answer against a photo
|
| 99 |
+
|
| 100 |
+
# PII, relations and evidence use the same call.
|
| 101 |
+
r = m.system_one(
|
| 102 |
+
state="Steve Jobs founded Apple in Cupertino together with Steve Wozniak.",
|
| 103 |
+
questions={"pii": {"type": "span", "criteria": ["PERSON"]},
|
| 104 |
+
"rel": {"type": "span", "instructions": "Which spans are the founder of Apple?", "criteria": {"founder": None}}})
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
## Evaluation
|
| 108 |
+
|
| 109 |
+
All numbers on public test sets, measured through this package with `system_one`, one request at a time and the shipped thresholds. Other models' numbers are as reported by their authors (or scored in the same harness where marked). Full per-set tables: [EVALUATION.md](EVALUATION.md).
|
| 110 |
+
|
| 111 |
+
<details open><summary><b>Named entities</b></summary>
|
| 112 |
+
|
| 113 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 114 |
+
|---|---:|---|---:|---|
|
| 115 |
+
| OOD-7, label names only (default; F1) | 64.6 | B2NER-20B 72.1 (standardised label names; B2NER Tab 3) | 43.5 | B2NER-20B, standardised label names 72.1 (B2NER, Table 3) |
|
| 116 |
+
| NER, 26 datasets of the GLiFormer paper (label names; not all zero-shot) | 63.7 | – | – | – |
|
| 117 |
+
|
| 118 |
+
Label names are the only input. Training holds no CrossNER or MIT text, though many generic CrossNER type names occur as training labels.
|
| 119 |
+
|
| 120 |
+
</details>
|
| 121 |
+
|
| 122 |
+
<details open><summary><b>Relations</b></summary>
|
| 123 |
+
|
| 124 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 125 |
+
|---|---:|---|---:|---|
|
| 126 |
+
| CoNLL04, head entity given, label names (F1) | **51.7** | GLiFormer-base 8.6 (same harness, Vela 2.0 card) | 10.1 | – |
|
| 127 |
+
| End to end, GLiFormer Table 4 protocol, avg of 4 (strict F1) | **35.9** | Qwen3.5-9B, prompted 25.7 (GLiFormer paper Tab 4) | – | Gemma-4-31B-IT, prompted 25.1 |
|
| 128 |
+
| End to end: CoNLL04 · DocRED · FewRel · CrossRE (CoNLL04 zero-shot, others in-domain) | 42.8 · 33.7 · 27.8 · 39.2 | GPT-5-mini 42.4 · GLiNER-Relex 31.3 · Qwen3.5-9B 71.6 · GLiNER-Relex 18.1 (GLiFormer paper Tab 4) | – | – |
|
| 129 |
+
|
| 130 |
+
CrossRE 39.2 (in-domain) is above every model reported in the GLiFormer paper (best: GLiNER-Relex 18.1).
|
| 131 |
+
|
| 132 |
+
</details>
|
| 133 |
+
|
| 134 |
+
<details open><summary><b>Evidence</b></summary>
|
| 135 |
+
|
| 136 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 137 |
+
|---|---:|---|---:|---|
|
| 138 |
+
| QASPER, word-F1 (zero-shot) | 39.3 | – | – | – |
|
| 139 |
+
| ACL-Verbatim, word-F1 (in-domain) | 49.1 | ACL-Verbatim ModernBERT 53.4 (in-domain, evidence only; Vela 2.0 card) | 24.5 | – |
|
| 140 |
+
|
| 141 |
+
ACL-Verbatim is in-domain: its silver training split is in the training data.
|
| 142 |
+
|
| 143 |
+
</details>
|
| 144 |
+
|
| 145 |
+
<details open><summary><b>Unsupported claims (hallucination spans)</b></summary>
|
| 146 |
+
|
| 147 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 148 |
+
|---|---:|---|---:|---|
|
| 149 |
+
| LettuceDetect v2 test set, 10,698 answers (example-F1) | 0.900 | LettuceDetect v2 qwen-2b 0.921 (its card) | 0.885 | – |
|
| 150 |
+
| RAGTruth (example-F1) | 0.779 | HDM-2 (3B) 0.850 (HalluciNot Tab 2) | 0.774 | Llama-2-13B fine-tuned 0.787 (reported, RAGTruth Tab 5) |
|
| 151 |
+
|
| 152 |
+
</details>
|
| 153 |
+
|
| 154 |
+
<details open><summary><b>Zero-shot classification</b></summary>
|
| 155 |
+
|
| 156 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 157 |
+
|---|---:|---|---:|---|
|
| 158 |
+
| GLiFormer suite, 13 sets, label names only (mean macro-F1) | **81.4** | GPT-5-mini 79.8 (GLiFormer paper Tab 5) | – | Gemma-4-31B-IT 79.7 |
|
| 159 |
+
|
| 160 |
+
These questions run on d3-turbo with every adapter off.
|
| 161 |
+
|
| 162 |
+
</details>
|
| 163 |
+
|
| 164 |
+
<details open><summary><b>Router signals</b> (public router signal suite; above Vela 2.0 9B on 7 of 8)</summary>
|
| 165 |
+
|
| 166 |
+
| Signal (metric, test rows) | vela3-turbo | Vela 2.0 9B, same rows | Decision-1.0-Route-0.6B, same rows | Rows sharing no text with training: vela3-turbo / Vela 2.0 9B |
|
| 167 |
+
|---|---:|---:|---:|---|
|
| 168 |
+
| Harmful request (AUC, 2,230) | 0.958 | 0.957 | 0.963 | 0.953 / 0.958 (1,254) |
|
| 169 |
+
| Jailbreak or prompt injection (AUC, 1,878) | **0.967** | 0.919 | 0.991 | **0.965** / 0.902 (944) |
|
| 170 |
+
| Personal data in the request (AUC, 3,000) | **0.971** | 0.960 | 0.994 | **0.973** / 0.960 (2,802) |
|
| 171 |
+
| Answer not supported by its context (AUC, 3,000) | **0.783** | 0.765 | 0.791 | 0.727 / 0.744 (1,743) |
|
| 172 |
+
| Fact-check needed (AUC, 800) | **0.870** | 0.809 | 0.942 | **0.863** / 0.805 (690) |
|
| 173 |
+
| Output modality (text or image) (AUC, 3,000) | **0.993** | 0.933 | 0.999 | **0.993** / 0.930 (2,916) |
|
| 174 |
+
| Domain (subject area) (accuracy, 2,996) | **0.884** | 0.765 | 0.863 | **0.885** / 0.763 (2,916) |
|
| 175 |
+
| User feedback on the previous answer (accuracy, 2,842) | **0.852** | 0.666 | 0.848 | **0.843** / 0.631 (2,295) |
|
| 176 |
+
|
| 177 |
+
[vllm-sr/router-signal-suite](https://huggingface.co/datasets/vllm-sr/router-signal-suite) (public), the `test` file of each signal, scored with the dataset's own scoring code. Each signal is a router question this package was trained on (supervised, as in Vela 2.0), asked one request at a time with the shipped thresholds.
|
| 178 |
+
|
| 179 |
+
Vela 2.0 9B was run by us through its public release on the same rows. Decision-1.0-Route-0.6B is scored from the suite's published predictions; it was trained on the suite's own train split, so these test files are in-distribution for it.
|
| 180 |
+
|
| 181 |
+
Bold: the paired 95% bootstrap CI against Vela 2.0 excludes a tie. The last column drops the test rows that share text (exact or a 13-gram) with this model's training data. Safety and jailbreak are scored on the rows whose text the suite publishes (2,230 of 3,003 and 1,878 of 3,000).
|
| 182 |
+
|
| 183 |
+
Domain and feedback are also trained on the suite's own train split (with the public router data Vela 2.0 used; the suite's test and hold rows, and anything sharing an exact text or a 13-gram with them, are kept out), as Decision-1.0-Route is, so their test files are in-distribution for both. The domain question has a 15th option, religion, which the suite maps to philosophy. Feedback has NO_FEEDBACK as a trained option.
|
| 184 |
+
|
| 185 |
+
</details>
|
| 186 |
+
|
| 187 |
+
<details open><summary><b>Decisions</b> (general decisions are the base model's, bit for bit)</summary>
|
| 188 |
+
|
| 189 |
+
| Decision Index 0.4.1 (2026-10-21) | Public index | Full score (board rank) |
|
| 190 |
+
|---|---:|---:|
|
| 191 |
+
| Fastino GLiDE V1.2 | 71.99 | 71.52 (1) |
|
| 192 |
+
| Perplexity Decider v1.2 (27B) (tied rank) | 67.74 | 67.98 (4) |
|
| 193 |
+
| vLLM-SR d3-turbo v1.1 (27B) (the public d3-turbo main; not this package's base; tied rank) | 71.62 | 67.31 (10) |
|
| 194 |
+
| vLLM-SR d3 (27B) (tied rank) | 65.45 | 67.15 (11) |
|
| 195 |
+
| OneJev1.6-0.8B (tied rank) | 48.76 | 29.58 (155) |
|
| 196 |
+
| StartLux-Decision v1.1 0.8B (tied rank) | 53.79 | 28.88 (158) |
|
| 197 |
+
| **General decisions = d3-turbo** (d3-turbo v1.0, revision da17b5d0: not on the 0.4.1 board) | – | – |
|
| 198 |
+
| GLiClass v4 Edge (0.4B) (own card, public part; not on the board) | 47.49 | – |
|
| 199 |
+
|
| 200 |
+
General decisions are the base model's own answers, bit for bit; no Vela package was run on the board. Full score (the board's ranking): 60% private same-skill tests and 40% private new-domain tasks; public index: the 37 public benchmarks, chance-corrected. All values reported on the Decision Index 0.4.1 board (2026-10-21; multimodalart/jev-decision-index, data generated 2026-10-22) unless noted; tied ranks share a number. d3-turbo v1.0, this package's base (revision `da17b5d0`), is not on the Decision Index 0.4.1 board; the public d3-turbo main is v1.1, which is on the board at 67.31 (rank 10, tied).
|
| 201 |
+
|
| 202 |
+
| Benchmark | vela3-turbo | Best other model (reported) | Vela 2.0 9B | Size-class best (>10B, reported) |
|
| 203 |
+
|---|---:|---|---:|---|
|
| 204 |
+
| Super-NI task 1568, held out (accuracy) | 0.636 | – | 0.574 | – |
|
| 205 |
+
| MASSIVE, three unseen scenarios: intent / scenario (accuracy; 14,911 decontaminated rows) | 0.833 / 0.937 | – | 0.763 / 0.871 (full set) | – |
|
| 206 |
+
|
| 207 |
+
MASSIVE: three scenarios never used in our router training (iot, transport, cooking), 52 locales; accuracy on the 14,911 test utterances that match no training text of the model's lineage. The 342 excluded utterances occur verbatim in upstream training mixtures (tasksource MASSIVE / HWU64 rows). Vela 2.0 values are on the full 15,253, so these rows are not bolded. Super-NI 1568: 79 to 90 of its 162 ProPara passages share a 13-gram with other tasks' training rows.
|
| 208 |
+
|
| 209 |
+
</details>
|
| 210 |
+
|
| 211 |
+
<details><summary><b>Latency</b></summary>
|
| 212 |
+
|
| 213 |
+
| Median, one request, AMD Instinct MI325X (measured alone on one GPU, no other process on it) | 200 tokens | 1K tokens | 3K tokens |
|
| 214 |
+
|---|---:|---:|---:|
|
| 215 |
+
| 4 router decisions | 469 ms | 952 ms | 2238 ms |
|
| 216 |
+
| 1 span question | 84 ms | 182 ms | 531 ms |
|
| 217 |
+
| 4 decisions + 1 span | 566 ms | 1138 ms | 2768 ms |
|
| 218 |
+
|
| 219 |
+
Span questions with label names only.
|
| 220 |
+
|
| 221 |
+
</details>
|
| 222 |
+
|
| 223 |
+
### Benchmarks
|
| 224 |
+
|
| 225 |
+
- **OOD-7** (CrossNER AI, literature, music, politics, science; MIT movie, restaurant): find the entities of the named types in text from domains the model was not trained on (strict entity F1).
|
| 226 |
+
- **NER, 26 datasets of the GLiFormer paper** (IE_INSTRUCTIONS): entities of named types across 26 public NER sets, not all zero-shot (strict entity F1).
|
| 227 |
+
- **CoNLL04, head entity given:** given one entity of a sentence, find the entities it is related to by the named relation (F1 over those tail entities).
|
| 228 |
+
- **Relations end to end, GLiFormer Table 4 protocol** (CoNLL04, DocRED, FewRel, CrossRE): find the entity pairs and their relation from raw text (strict triplet F1).
|
| 229 |
+
- **QASPER:** find the sentences of a research paper that answer a question about it (word-F1).
|
| 230 |
+
- **ACL-Verbatim** ([KRLabsOrg/acl-verbatim-spans](https://huggingface.co/datasets/KRLabsOrg/acl-verbatim-spans)): find the passage of an ACL paper that supports an answer (word-F1).
|
| 231 |
+
- **LettuceDetect v2 test set** (10,698 answers, as in the LettuceDetect v2 paper and card): does an answer contain claims its context does not support (example-F1: the answer is flagged or not).
|
| 232 |
+
- **RAGTruth:** does a retrieval-augmented answer contain claims its retrieved context does not support (example-F1).
|
| 233 |
+
- **MASSIVE, three unseen scenarios** (iot, transport, cooking; 52 languages): pick the intent and the scenario of a user command the model never saw in training (accuracy).
|
| 234 |
+
- **Super-NI task 1568** (ProPara): answer a multiple-choice question about how a process changes an entity (accuracy).
|
| 235 |
+
- **Zero-shot classification, 13 sets** (the GLiFormer paper's suite: SST-2, IMDB, AG News, Banking77, and more): pick the label of a text from label names only (mean macro-F1).
|
| 236 |
+
- **Decision Index 0.4.1** ([board](https://huggingface.co/spaces/multimodalart/jev-decision-index), 2026-10-21): 37 public decision benchmarks plus private tests; the Full score ranks on the private same-skill and new-domain tests (chance-corrected skill, 0 = guessing).
|
| 237 |
+
- **Router signal suite** ([vllm-sr/router-signal-suite](https://huggingface.co/datasets/vllm-sr/router-signal-suite)): the vLLM Semantic Router's public test of its request signals (harmful requests, jailbreaks, personal data, unsupported answers, fact-check need, modality, domain, feedback); each label comes from public corpora (AUC or accuracy).
|
| 238 |
+
|
| 239 |
+
## The family
|
| 240 |
+
|
| 241 |
+

|
| 242 |
+
|
| 243 |
+
| Model | Size | Base | Best for |
|
| 244 |
+
|---|---|---|---|
|
| 245 |
+
| [vela3-fast](https://huggingface.co/vllm-sr/vela3-fast) | 0.4B | UltraBERT-380M encoder | the default router; fastest: decisions + spans in one ~13 ms encoder pass, English, 32K |
|
| 246 |
+
| [vela3-fast-vision](https://huggingface.co/vllm-sr/vela3-fast-vision) | 0.3B | UltraVBERT encoder | image decisions, image-grounded hallucination spans, region pointing (preview) |
|
| 247 |
+
| [vela3-lite](https://huggingface.co/vllm-sr/vela3-lite) | 0.8B | d3-lite | small GPUs, low latency |
|
| 248 |
+
| [vela3-mini](https://huggingface.co/vllm-sr/vela3-mini) | 4B | d3-mini | balanced accuracy and speed |
|
| 249 |
+
| [vela3-flash](https://huggingface.co/vllm-sr/vela3-flash) | 9B | d3-flash | high-accuracy routing |
|
| 250 |
+
| [vela3-turbo](https://huggingface.co/vllm-sr/vela3-turbo) | 27B | d3-turbo | the reference model |
|
| 251 |
+
|
| 252 |
+
## How it works
|
| 253 |
+
|
| 254 |
+
- **Frozen base.** `d3-turbo@da17b5d0` runs unchanged; ordinary questions are its own answers.
|
| 255 |
+
- **Gated LoRA adapters.** Each switches on only for its own questions, and every answer names the one it used (`head_reason`): decision (router signals), NER (entity labels with definitions), relation, image (hallucination spans with images), span (evidence, PII, unsupported claims, toxic spans, names-only labels).
|
| 256 |
+
- **Spans as decisions.** The labels are presented as options; a span head scores every word against every label, and agreeing neighbours form a span with character offsets.
|
| 257 |
+
- **Label names in.** Labels are read by name at this size; no definitions are written.
|
| 258 |
+
- **Specification.** Adds 5044 MB of adapters and heads; text or JSON state; requests with images, videos, assets, box or track questions, or long_context go to the base with every adapter off. Full evaluation: [EVALUATION.md](EVALUATION.md).
|
| 259 |
+
|
| 260 |
+
## Trade-offs
|
| 261 |
+
|
| 262 |
+
- **Unsupported claims:** hallucination spans 0.900 against 0.921 for LettuceDetect v2 qwen-2b, RAGTruth 0.779 against 0.850 for HDM-2 (3B) (reported).
|
| 263 |
+
- **NER from label names:** 64.6 on OOD-7, above the best reported encoder (gliner-multitask-v1.0 63.25) and below B2NER-20B, standardised label names 72.1 (reported).
|
| 264 |
+
- **End-to-end relations on generic types:** 35.9 average on the four typed sets, above every model in GLiFormer Table 4 (reported), but 0.2 on SciERC and 0.5 on NYT11, where every entity has the generic type `entity`.
|
| 265 |
+
|
| 266 |
+
## Citation
|
| 267 |
+
|
| 268 |
+
```bibtex
|
| 269 |
+
@misc{vela3_turbo,
|
| 270 |
+
title = {vela3-turbo (Vela 3.0, 27B): zero-shot extraction and router signals on Decision 3.0},
|
| 271 |
+
author = {{KR Labs} and {vLLM Semantic Router}},
|
| 272 |
+
year = {2026},
|
| 273 |
+
howpublished = {\url{https://huggingface.co/vllm-sr/vela3-turbo}}
|
| 274 |
+
}
|
| 275 |
+
```
|
SHA256SUMS
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 LICENSE
|
| 2 |
+
81ff9bfd1c4de347c00eaa78d53f6d37f7aa250c70682583e96427e0af309c24 calibration.json
|
| 3 |
+
8fa1f8e03f3a39e6cfc808db1151163fe1118a8e7a89cd040573814680b7f4ed config.json
|
| 4 |
+
05b8d0cf65b4292a9c2764b8316d53a66f813b8f6742b78d1f75c7c3aa36167e decision_df_lora.safetensors
|
| 5 |
+
f6ceb0a0e0ea66d95db7d32ebd0a01631f2596870ebeb52d83300e2f2fef68b6 decision_lora.safetensors
|
| 6 |
+
4cd699d86c9d41a7947e19ae944541d9d552fc68e94b7c11d40b9b67b1afe62c label_definitions.json
|
| 7 |
+
792f5458e868bb57190ed256ef38993d1ea64969d72bb4965e3fb737e465239d modeling_vela3.py
|
| 8 |
+
8bc4c9848358782594e5f508bec047c1e49b9049429d13137672d68e1be6d87a ner_head.safetensors
|
| 9 |
+
fd9d297b4cc07fb35e45636a3e926137f2e1d7b464abdb48ba088a50d3d6f65a ner_lora.safetensors
|
| 10 |
+
b3ec955d3a2144eb5b2359dc7ed3a3f53373ae7eb55cfcad974d81d8c6b02531 relation_head.safetensors
|
| 11 |
+
c61459f28b85f6c5c4fa25fcb850b501400312cefb4e0bba1125ab7c5635a64d relation_lora.safetensors
|
| 12 |
+
8ed409d39cf03e49ccc5c5868643d73b72aa078497c848d214a370dd317fe3ec router_questions.json
|
| 13 |
+
d2c5f2f446cf12d2f8a24dbf634452477088523b0973e1e07df244c851421efb span_decode_v2.py
|
| 14 |
+
5b450a2b20d077f385b7c06042ff06c86797f05692f8b84755032920349f4dd1 span_head.safetensors
|
| 15 |
+
94a02a79c37c1a7d53d3e9771b31515c94af19011e2a75c1c51ea37196b26383 span_lora.safetensors
|
| 16 |
+
eb69e58c18593527a0eb0bade7bc05eec315f3039fe92e1c500ba9c5f0a36d31 vela3_adapters.py
|
| 17 |
+
7d37c276ba35b15aab9a91ff2cdb5bf5eb57973b07e7bef187b5b551ca39c8b0 vela3_config.json
|
| 18 |
+
b7fcefdd1a0b3e8ac22ac12338c72e76925688a989b5af6c70a104fa70122a39 vela3_labeldef.py
|
| 19 |
+
97d2448765a7ee5411dff44c601c3a073fdad9a9921f9818bac4135916081c20 vela3_span.py
|
assets/banner.png
ADDED
|
Git LFS Details
|
assets/family.png
ADDED
|
Git LFS Details
|
assets/glance.png
ADDED
|
Git LFS Details
|
calibration.json
ADDED
|
@@ -0,0 +1,1677 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"span_pick": {
|
| 3 |
+
"rule": "ner_shift VALID: (ckpt, head) by eval F1 in defs mode; thresholds from the thr halves",
|
| 4 |
+
"all": {
|
| 5 |
+
"ckpt-024000/final": {
|
| 6 |
+
"threshold": 0.6,
|
| 7 |
+
"thr_f1": 0.75564605701592,
|
| 8 |
+
"eval_f1": 0.7622234616099647
|
| 9 |
+
}
|
| 10 |
+
},
|
| 11 |
+
"choice": {
|
| 12 |
+
"ckpt": "ckpt-024000",
|
| 13 |
+
"head": "final",
|
| 14 |
+
"threshold_defs": 0.6,
|
| 15 |
+
"threshold_names": 0.55
|
| 16 |
+
},
|
| 17 |
+
"names_at_choice": {
|
| 18 |
+
"threshold": 0.55,
|
| 19 |
+
"thr_f1": 0.7392610924518296,
|
| 20 |
+
"eval_f1": 0.753922967189729
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"evidence_pick": {
|
| 24 |
+
"setting": {
|
| 25 |
+
"ckpt": "ckpt-024000",
|
| 26 |
+
"head": "final",
|
| 27 |
+
"threshold_defs": 0.6,
|
| 28 |
+
"threshold_names": 0.55
|
| 29 |
+
},
|
| 30 |
+
"choice": {
|
| 31 |
+
"decode": "plain",
|
| 32 |
+
"gap": 0,
|
| 33 |
+
"minc": 10,
|
| 34 |
+
"threshold": 0.5,
|
| 35 |
+
"thr": {
|
| 36 |
+
"acl_val": 0.7021252311274454,
|
| 37 |
+
"vsdev": 0.7685352622061482
|
| 38 |
+
},
|
| 39 |
+
"objective": 0.7353302466667968
|
| 40 |
+
},
|
| 41 |
+
"eval_at_choice": {
|
| 42 |
+
"acl_val": 0.6795241661244582,
|
| 43 |
+
"vsdev": 0.7205161290322583
|
| 44 |
+
},
|
| 45 |
+
"eval_plain_at_its_best_threshold": {
|
| 46 |
+
"threshold": 0.5,
|
| 47 |
+
"acl_val": 0.6805612976969284,
|
| 48 |
+
"vsdev": 0.7183206594670585
|
| 49 |
+
},
|
| 50 |
+
"eval_plain_06": {
|
| 51 |
+
"acl_val": 0.6464071352442925,
|
| 52 |
+
"vsdev": 0.7059601104156137
|
| 53 |
+
},
|
| 54 |
+
"rule": "max mean thr-half word F1 over acl_val + vsdev; ties -> higher threshold",
|
| 55 |
+
"grid": {
|
| 56 |
+
"plain/g0/m0/t0.05": {
|
| 57 |
+
"decode": "plain",
|
| 58 |
+
"gap": 0,
|
| 59 |
+
"minc": 0,
|
| 60 |
+
"threshold": 0.05,
|
| 61 |
+
"thr": {
|
| 62 |
+
"acl_val": 0.49804282485612433,
|
| 63 |
+
"vsdev": 0.5368626834146012
|
| 64 |
+
},
|
| 65 |
+
"objective": 0.5174527541353627
|
| 66 |
+
},
|
| 67 |
+
"plain/g0/m0/t0.1": {
|
| 68 |
+
"decode": "plain",
|
| 69 |
+
"gap": 0,
|
| 70 |
+
"minc": 0,
|
| 71 |
+
"threshold": 0.1,
|
| 72 |
+
"thr": {
|
| 73 |
+
"acl_val": 0.5401435486563179,
|
| 74 |
+
"vsdev": 0.5760208514335361
|
| 75 |
+
},
|
| 76 |
+
"objective": 0.558082200044927
|
| 77 |
+
},
|
| 78 |
+
"plain/g0/m0/t0.15": {
|
| 79 |
+
"decode": "plain",
|
| 80 |
+
"gap": 0,
|
| 81 |
+
"minc": 0,
|
| 82 |
+
"threshold": 0.15,
|
| 83 |
+
"thr": {
|
| 84 |
+
"acl_val": 0.5716617446693733,
|
| 85 |
+
"vsdev": 0.6116173413075284
|
| 86 |
+
},
|
| 87 |
+
"objective": 0.5916395429884509
|
| 88 |
+
},
|
| 89 |
+
"plain/g0/m0/t0.2": {
|
| 90 |
+
"decode": "plain",
|
| 91 |
+
"gap": 0,
|
| 92 |
+
"minc": 0,
|
| 93 |
+
"threshold": 0.2,
|
| 94 |
+
"thr": {
|
| 95 |
+
"acl_val": 0.5979189590364293,
|
| 96 |
+
"vsdev": 0.6456984273820537
|
| 97 |
+
},
|
| 98 |
+
"objective": 0.6218086932092415
|
| 99 |
+
},
|
| 100 |
+
"plain/g0/m0/t0.25": {
|
| 101 |
+
"decode": "plain",
|
| 102 |
+
"gap": 0,
|
| 103 |
+
"minc": 0,
|
| 104 |
+
"threshold": 0.25,
|
| 105 |
+
"thr": {
|
| 106 |
+
"acl_val": 0.6234659901696528,
|
| 107 |
+
"vsdev": 0.6791974169741698
|
| 108 |
+
},
|
| 109 |
+
"objective": 0.6513317035719113
|
| 110 |
+
},
|
| 111 |
+
"plain/g0/m0/t0.3": {
|
| 112 |
+
"decode": "plain",
|
| 113 |
+
"gap": 0,
|
| 114 |
+
"minc": 0,
|
| 115 |
+
"threshold": 0.3,
|
| 116 |
+
"thr": {
|
| 117 |
+
"acl_val": 0.6445885677322712,
|
| 118 |
+
"vsdev": 0.705739447312481
|
| 119 |
+
},
|
| 120 |
+
"objective": 0.6751640075223762
|
| 121 |
+
},
|
| 122 |
+
"plain/g0/m0/t0.35": {
|
| 123 |
+
"decode": "plain",
|
| 124 |
+
"gap": 0,
|
| 125 |
+
"minc": 0,
|
| 126 |
+
"threshold": 0.35,
|
| 127 |
+
"thr": {
|
| 128 |
+
"acl_val": 0.6636997747584101,
|
| 129 |
+
"vsdev": 0.7220617497913858
|
| 130 |
+
},
|
| 131 |
+
"objective": 0.692880762274898
|
| 132 |
+
},
|
| 133 |
+
"plain/g0/m0/t0.4": {
|
| 134 |
+
"decode": "plain",
|
| 135 |
+
"gap": 0,
|
| 136 |
+
"minc": 0,
|
| 137 |
+
"threshold": 0.4,
|
| 138 |
+
"thr": {
|
| 139 |
+
"acl_val": 0.678196381473299,
|
| 140 |
+
"vsdev": 0.7409347574664943
|
| 141 |
+
},
|
| 142 |
+
"objective": 0.7095655694698966
|
| 143 |
+
},
|
| 144 |
+
"plain/g0/m0/t0.45": {
|
| 145 |
+
"decode": "plain",
|
| 146 |
+
"gap": 0,
|
| 147 |
+
"minc": 0,
|
| 148 |
+
"threshold": 0.45,
|
| 149 |
+
"thr": {
|
| 150 |
+
"acl_val": 0.6944342211800334,
|
| 151 |
+
"vsdev": 0.7613341033785734
|
| 152 |
+
},
|
| 153 |
+
"objective": 0.7278841622793034
|
| 154 |
+
},
|
| 155 |
+
"plain/g0/m0/t0.5": {
|
| 156 |
+
"decode": "plain",
|
| 157 |
+
"gap": 0,
|
| 158 |
+
"minc": 0,
|
| 159 |
+
"threshold": 0.5,
|
| 160 |
+
"thr": {
|
| 161 |
+
"acl_val": 0.7013930539681104,
|
| 162 |
+
"vsdev": 0.7663813229571984
|
| 163 |
+
},
|
| 164 |
+
"objective": 0.7338871884626543
|
| 165 |
+
},
|
| 166 |
+
"plain/g0/m0/t0.55": {
|
| 167 |
+
"decode": "plain",
|
| 168 |
+
"gap": 0,
|
| 169 |
+
"minc": 0,
|
| 170 |
+
"threshold": 0.55,
|
| 171 |
+
"thr": {
|
| 172 |
+
"acl_val": 0.7040013659036307,
|
| 173 |
+
"vsdev": 0.7549156081433791
|
| 174 |
+
},
|
| 175 |
+
"objective": 0.7294584870235049
|
| 176 |
+
},
|
| 177 |
+
"plain/g0/m0/t0.6": {
|
| 178 |
+
"decode": "plain",
|
| 179 |
+
"gap": 0,
|
| 180 |
+
"minc": 0,
|
| 181 |
+
"threshold": 0.6,
|
| 182 |
+
"thr": {
|
| 183 |
+
"acl_val": 0.6822220485161156,
|
| 184 |
+
"vsdev": 0.7282051282051282
|
| 185 |
+
},
|
| 186 |
+
"objective": 0.7052135883606219
|
| 187 |
+
},
|
| 188 |
+
"plain/g0/m0/t0.65": {
|
| 189 |
+
"decode": "plain",
|
| 190 |
+
"gap": 0,
|
| 191 |
+
"minc": 0,
|
| 192 |
+
"threshold": 0.65,
|
| 193 |
+
"thr": {
|
| 194 |
+
"acl_val": 0.651708607082352,
|
| 195 |
+
"vsdev": 0.6918902622851976
|
| 196 |
+
},
|
| 197 |
+
"objective": 0.6717994346837748
|
| 198 |
+
},
|
| 199 |
+
"plain/g0/m0/t0.7": {
|
| 200 |
+
"decode": "plain",
|
| 201 |
+
"gap": 0,
|
| 202 |
+
"minc": 0,
|
| 203 |
+
"threshold": 0.7,
|
| 204 |
+
"thr": {
|
| 205 |
+
"acl_val": 0.6133449376269219,
|
| 206 |
+
"vsdev": 0.6369310233509092
|
| 207 |
+
},
|
| 208 |
+
"objective": 0.6251379804889156
|
| 209 |
+
},
|
| 210 |
+
"plain/g0/m0/t0.75": {
|
| 211 |
+
"decode": "plain",
|
| 212 |
+
"gap": 0,
|
| 213 |
+
"minc": 0,
|
| 214 |
+
"threshold": 0.75,
|
| 215 |
+
"thr": {
|
| 216 |
+
"acl_val": 0.5729350807680585,
|
| 217 |
+
"vsdev": 0.5785641142337012
|
| 218 |
+
},
|
| 219 |
+
"objective": 0.57574959750088
|
| 220 |
+
},
|
| 221 |
+
"plain/g0/m0/t0.8": {
|
| 222 |
+
"decode": "plain",
|
| 223 |
+
"gap": 0,
|
| 224 |
+
"minc": 0,
|
| 225 |
+
"threshold": 0.8,
|
| 226 |
+
"thr": {
|
| 227 |
+
"acl_val": 0.5265596293633676,
|
| 228 |
+
"vsdev": 0.5130738643210025
|
| 229 |
+
},
|
| 230 |
+
"objective": 0.519816746842185
|
| 231 |
+
},
|
| 232 |
+
"plain/g0/m0/t0.85": {
|
| 233 |
+
"decode": "plain",
|
| 234 |
+
"gap": 0,
|
| 235 |
+
"minc": 0,
|
| 236 |
+
"threshold": 0.85,
|
| 237 |
+
"thr": {
|
| 238 |
+
"acl_val": 0.47060204595002514,
|
| 239 |
+
"vsdev": 0.46027742749054223
|
| 240 |
+
},
|
| 241 |
+
"objective": 0.4654397367202837
|
| 242 |
+
},
|
| 243 |
+
"plain/g0/m0/t0.9": {
|
| 244 |
+
"decode": "plain",
|
| 245 |
+
"gap": 0,
|
| 246 |
+
"minc": 0,
|
| 247 |
+
"threshold": 0.9,
|
| 248 |
+
"thr": {
|
| 249 |
+
"acl_val": 0.3798414941767895,
|
| 250 |
+
"vsdev": 0.3790523690773067
|
| 251 |
+
},
|
| 252 |
+
"objective": 0.3794469316270481
|
| 253 |
+
},
|
| 254 |
+
"plain/g0/m0/t0.95": {
|
| 255 |
+
"decode": "plain",
|
| 256 |
+
"gap": 0,
|
| 257 |
+
"minc": 0,
|
| 258 |
+
"threshold": 0.95,
|
| 259 |
+
"thr": {
|
| 260 |
+
"acl_val": 0.22448818897637796,
|
| 261 |
+
"vsdev": 0.26957842215633815
|
| 262 |
+
},
|
| 263 |
+
"objective": 0.24703330556635805
|
| 264 |
+
},
|
| 265 |
+
"plain/g0/m10/t0.05": {
|
| 266 |
+
"decode": "plain",
|
| 267 |
+
"gap": 0,
|
| 268 |
+
"minc": 10,
|
| 269 |
+
"threshold": 0.05,
|
| 270 |
+
"thr": {
|
| 271 |
+
"acl_val": 0.5088676752763116,
|
| 272 |
+
"vsdev": 0.5461197237846984
|
| 273 |
+
},
|
| 274 |
+
"objective": 0.5274936995305051
|
| 275 |
+
},
|
| 276 |
+
"plain/g0/m10/t0.1": {
|
| 277 |
+
"decode": "plain",
|
| 278 |
+
"gap": 0,
|
| 279 |
+
"minc": 10,
|
| 280 |
+
"threshold": 0.1,
|
| 281 |
+
"thr": {
|
| 282 |
+
"acl_val": 0.5488734119990303,
|
| 283 |
+
"vsdev": 0.5840941913690549
|
| 284 |
+
},
|
| 285 |
+
"objective": 0.5664838016840426
|
| 286 |
+
},
|
| 287 |
+
"plain/g0/m10/t0.15": {
|
| 288 |
+
"decode": "plain",
|
| 289 |
+
"gap": 0,
|
| 290 |
+
"minc": 10,
|
| 291 |
+
"threshold": 0.15,
|
| 292 |
+
"thr": {
|
| 293 |
+
"acl_val": 0.5782624285503364,
|
| 294 |
+
"vsdev": 0.6200166545227439
|
| 295 |
+
},
|
| 296 |
+
"objective": 0.5991395415365401
|
| 297 |
+
},
|
| 298 |
+
"plain/g0/m10/t0.2": {
|
| 299 |
+
"decode": "plain",
|
| 300 |
+
"gap": 0,
|
| 301 |
+
"minc": 10,
|
| 302 |
+
"threshold": 0.2,
|
| 303 |
+
"thr": {
|
| 304 |
+
"acl_val": 0.6043906540933389,
|
| 305 |
+
"vsdev": 0.6526344751685239
|
| 306 |
+
},
|
| 307 |
+
"objective": 0.6285125646309314
|
| 308 |
+
},
|
| 309 |
+
"plain/g0/m10/t0.25": {
|
| 310 |
+
"decode": "plain",
|
| 311 |
+
"gap": 0,
|
| 312 |
+
"minc": 10,
|
| 313 |
+
"threshold": 0.25,
|
| 314 |
+
"thr": {
|
| 315 |
+
"acl_val": 0.6290759118443098,
|
| 316 |
+
"vsdev": 0.6863363100449426
|
| 317 |
+
},
|
| 318 |
+
"objective": 0.6577061109446262
|
| 319 |
+
},
|
| 320 |
+
"plain/g0/m10/t0.3": {
|
| 321 |
+
"decode": "plain",
|
| 322 |
+
"gap": 0,
|
| 323 |
+
"minc": 10,
|
| 324 |
+
"threshold": 0.3,
|
| 325 |
+
"thr": {
|
| 326 |
+
"acl_val": 0.6493385375289281,
|
| 327 |
+
"vsdev": 0.710611031057424
|
| 328 |
+
},
|
| 329 |
+
"objective": 0.679974784293176
|
| 330 |
+
},
|
| 331 |
+
"plain/g0/m10/t0.35": {
|
| 332 |
+
"decode": "plain",
|
| 333 |
+
"gap": 0,
|
| 334 |
+
"minc": 10,
|
| 335 |
+
"threshold": 0.35,
|
| 336 |
+
"thr": {
|
| 337 |
+
"acl_val": 0.6674155234324648,
|
| 338 |
+
"vsdev": 0.7265536356858042
|
| 339 |
+
},
|
| 340 |
+
"objective": 0.6969845795591345
|
| 341 |
+
},
|
| 342 |
+
"plain/g0/m10/t0.4": {
|
| 343 |
+
"decode": "plain",
|
| 344 |
+
"gap": 0,
|
| 345 |
+
"minc": 10,
|
| 346 |
+
"threshold": 0.4,
|
| 347 |
+
"thr": {
|
| 348 |
+
"acl_val": 0.681955590566688,
|
| 349 |
+
"vsdev": 0.7457010468508667
|
| 350 |
+
},
|
| 351 |
+
"objective": 0.7138283187087773
|
| 352 |
+
},
|
| 353 |
+
"plain/g0/m10/t0.45": {
|
| 354 |
+
"decode": "plain",
|
| 355 |
+
"gap": 0,
|
| 356 |
+
"minc": 10,
|
| 357 |
+
"threshold": 0.45,
|
| 358 |
+
"thr": {
|
| 359 |
+
"acl_val": 0.6967976524623628,
|
| 360 |
+
"vsdev": 0.7625867897778907
|
| 361 |
+
},
|
| 362 |
+
"objective": 0.7296922211201268
|
| 363 |
+
},
|
| 364 |
+
"plain/g0/m10/t0.5": {
|
| 365 |
+
"decode": "plain",
|
| 366 |
+
"gap": 0,
|
| 367 |
+
"minc": 10,
|
| 368 |
+
"threshold": 0.5,
|
| 369 |
+
"thr": {
|
| 370 |
+
"acl_val": 0.7021252311274454,
|
| 371 |
+
"vsdev": 0.7685352622061482
|
| 372 |
+
},
|
| 373 |
+
"objective": 0.7353302466667968
|
| 374 |
+
},
|
| 375 |
+
"plain/g0/m10/t0.55": {
|
| 376 |
+
"decode": "plain",
|
| 377 |
+
"gap": 0,
|
| 378 |
+
"minc": 10,
|
| 379 |
+
"threshold": 0.55,
|
| 380 |
+
"thr": {
|
| 381 |
+
"acl_val": 0.7045848122720182,
|
| 382 |
+
"vsdev": 0.7549088359046283
|
| 383 |
+
},
|
| 384 |
+
"objective": 0.7297468240883233
|
| 385 |
+
},
|
| 386 |
+
"plain/g0/m10/t0.6": {
|
| 387 |
+
"decode": "plain",
|
| 388 |
+
"gap": 0,
|
| 389 |
+
"minc": 10,
|
| 390 |
+
"threshold": 0.6,
|
| 391 |
+
"thr": {
|
| 392 |
+
"acl_val": 0.6800340514701069,
|
| 393 |
+
"vsdev": 0.7257667550170391
|
| 394 |
+
},
|
| 395 |
+
"objective": 0.7029004032435731
|
| 396 |
+
},
|
| 397 |
+
"plain/g0/m10/t0.65": {
|
| 398 |
+
"decode": "plain",
|
| 399 |
+
"gap": 0,
|
| 400 |
+
"minc": 10,
|
| 401 |
+
"threshold": 0.65,
|
| 402 |
+
"thr": {
|
| 403 |
+
"acl_val": 0.6497498410592365,
|
| 404 |
+
"vsdev": 0.6903885892848143
|
| 405 |
+
},
|
| 406 |
+
"objective": 0.6700692151720253
|
| 407 |
+
},
|
| 408 |
+
"plain/g0/m10/t0.7": {
|
| 409 |
+
"decode": "plain",
|
| 410 |
+
"gap": 0,
|
| 411 |
+
"minc": 10,
|
| 412 |
+
"threshold": 0.7,
|
| 413 |
+
"thr": {
|
| 414 |
+
"acl_val": 0.6102100001457322,
|
| 415 |
+
"vsdev": 0.6332522303325223
|
| 416 |
+
},
|
| 417 |
+
"objective": 0.6217311152391273
|
| 418 |
+
},
|
| 419 |
+
"plain/g0/m10/t0.75": {
|
| 420 |
+
"decode": "plain",
|
| 421 |
+
"gap": 0,
|
| 422 |
+
"minc": 10,
|
| 423 |
+
"threshold": 0.75,
|
| 424 |
+
"thr": {
|
| 425 |
+
"acl_val": 0.5707604126862821,
|
| 426 |
+
"vsdev": 0.5749828806208629
|
| 427 |
+
},
|
| 428 |
+
"objective": 0.5728716466535725
|
| 429 |
+
},
|
| 430 |
+
"plain/g0/m10/t0.8": {
|
| 431 |
+
"decode": "plain",
|
| 432 |
+
"gap": 0,
|
| 433 |
+
"minc": 10,
|
| 434 |
+
"threshold": 0.8,
|
| 435 |
+
"thr": {
|
| 436 |
+
"acl_val": 0.5240645140603457,
|
| 437 |
+
"vsdev": 0.5082244799225931
|
| 438 |
+
},
|
| 439 |
+
"objective": 0.5161444969914695
|
| 440 |
+
},
|
| 441 |
+
"plain/g0/m10/t0.85": {
|
| 442 |
+
"decode": "plain",
|
| 443 |
+
"gap": 0,
|
| 444 |
+
"minc": 10,
|
| 445 |
+
"threshold": 0.85,
|
| 446 |
+
"thr": {
|
| 447 |
+
"acl_val": 0.4677129064271417,
|
| 448 |
+
"vsdev": 0.4563800847725692
|
| 449 |
+
},
|
| 450 |
+
"objective": 0.46204649559985544
|
| 451 |
+
},
|
| 452 |
+
"plain/g0/m10/t0.9": {
|
| 453 |
+
"decode": "plain",
|
| 454 |
+
"gap": 0,
|
| 455 |
+
"minc": 10,
|
| 456 |
+
"threshold": 0.9,
|
| 457 |
+
"thr": {
|
| 458 |
+
"acl_val": 0.3752737193524069,
|
| 459 |
+
"vsdev": 0.37334235453315284
|
| 460 |
+
},
|
| 461 |
+
"objective": 0.37430803694277986
|
| 462 |
+
},
|
| 463 |
+
"plain/g0/m10/t0.95": {
|
| 464 |
+
"decode": "plain",
|
| 465 |
+
"gap": 0,
|
| 466 |
+
"minc": 10,
|
| 467 |
+
"threshold": 0.95,
|
| 468 |
+
"thr": {
|
| 469 |
+
"acl_val": 0.2187771545935698,
|
| 470 |
+
"vsdev": 0.26694669756662803
|
| 471 |
+
},
|
| 472 |
+
"objective": 0.24286192608009893
|
| 473 |
+
},
|
| 474 |
+
"gap20/g20/m10/t0.05": {
|
| 475 |
+
"decode": "gap20",
|
| 476 |
+
"gap": 20,
|
| 477 |
+
"minc": 10,
|
| 478 |
+
"threshold": 0.05,
|
| 479 |
+
"thr": {
|
| 480 |
+
"acl_val": 0.48406010557263873,
|
| 481 |
+
"vsdev": 0.5233538441325246
|
| 482 |
+
},
|
| 483 |
+
"objective": 0.5037069748525816
|
| 484 |
+
},
|
| 485 |
+
"gap20/g20/m10/t0.1": {
|
| 486 |
+
"decode": "gap20",
|
| 487 |
+
"gap": 20,
|
| 488 |
+
"minc": 10,
|
| 489 |
+
"threshold": 0.1,
|
| 490 |
+
"thr": {
|
| 491 |
+
"acl_val": 0.5299878672653816,
|
| 492 |
+
"vsdev": 0.5637153105531433
|
| 493 |
+
},
|
| 494 |
+
"objective": 0.5468515889092624
|
| 495 |
+
},
|
| 496 |
+
"gap20/g20/m10/t0.15": {
|
| 497 |
+
"decode": "gap20",
|
| 498 |
+
"gap": 20,
|
| 499 |
+
"minc": 10,
|
| 500 |
+
"threshold": 0.15,
|
| 501 |
+
"thr": {
|
| 502 |
+
"acl_val": 0.5647771633051399,
|
| 503 |
+
"vsdev": 0.5987990854960682
|
| 504 |
+
},
|
| 505 |
+
"objective": 0.581788124400604
|
| 506 |
+
},
|
| 507 |
+
"gap20/g20/m10/t0.2": {
|
| 508 |
+
"decode": "gap20",
|
| 509 |
+
"gap": 20,
|
| 510 |
+
"minc": 10,
|
| 511 |
+
"threshold": 0.2,
|
| 512 |
+
"thr": {
|
| 513 |
+
"acl_val": 0.5907094080292288,
|
| 514 |
+
"vsdev": 0.6360715433222662
|
| 515 |
+
},
|
| 516 |
+
"objective": 0.6133904756757476
|
| 517 |
+
},
|
| 518 |
+
"gap20/g20/m10/t0.25": {
|
| 519 |
+
"decode": "gap20",
|
| 520 |
+
"gap": 20,
|
| 521 |
+
"minc": 10,
|
| 522 |
+
"threshold": 0.25,
|
| 523 |
+
"thr": {
|
| 524 |
+
"acl_val": 0.6174080393944352,
|
| 525 |
+
"vsdev": 0.6712881413716689
|
| 526 |
+
},
|
| 527 |
+
"objective": 0.6443480903830521
|
| 528 |
+
},
|
| 529 |
+
"gap20/g20/m10/t0.3": {
|
| 530 |
+
"decode": "gap20",
|
| 531 |
+
"gap": 20,
|
| 532 |
+
"minc": 10,
|
| 533 |
+
"threshold": 0.3,
|
| 534 |
+
"thr": {
|
| 535 |
+
"acl_val": 0.6394975388459829,
|
| 536 |
+
"vsdev": 0.6993103448275862
|
| 537 |
+
},
|
| 538 |
+
"objective": 0.6694039418367845
|
| 539 |
+
},
|
| 540 |
+
"gap20/g20/m10/t0.35": {
|
| 541 |
+
"decode": "gap20",
|
| 542 |
+
"gap": 20,
|
| 543 |
+
"minc": 10,
|
| 544 |
+
"threshold": 0.35,
|
| 545 |
+
"thr": {
|
| 546 |
+
"acl_val": 0.660314333049747,
|
| 547 |
+
"vsdev": 0.7162354729408784
|
| 548 |
+
},
|
| 549 |
+
"objective": 0.6882749029953127
|
| 550 |
+
},
|
| 551 |
+
"gap20/g20/m10/t0.4": {
|
| 552 |
+
"decode": "gap20",
|
| 553 |
+
"gap": 20,
|
| 554 |
+
"minc": 10,
|
| 555 |
+
"threshold": 0.4,
|
| 556 |
+
"thr": {
|
| 557 |
+
"acl_val": 0.6738052212695606,
|
| 558 |
+
"vsdev": 0.7354790972992971
|
| 559 |
+
},
|
| 560 |
+
"objective": 0.7046421592844287
|
| 561 |
+
},
|
| 562 |
+
"gap20/g20/m10/t0.45": {
|
| 563 |
+
"decode": "gap20",
|
| 564 |
+
"gap": 20,
|
| 565 |
+
"minc": 10,
|
| 566 |
+
"threshold": 0.45,
|
| 567 |
+
"thr": {
|
| 568 |
+
"acl_val": 0.6920388591874519,
|
| 569 |
+
"vsdev": 0.7584579071597168
|
| 570 |
+
},
|
| 571 |
+
"objective": 0.7252483831735843
|
| 572 |
+
},
|
| 573 |
+
"gap20/g20/m10/t0.5": {
|
| 574 |
+
"decode": "gap20",
|
| 575 |
+
"gap": 20,
|
| 576 |
+
"minc": 10,
|
| 577 |
+
"threshold": 0.5,
|
| 578 |
+
"thr": {
|
| 579 |
+
"acl_val": 0.7008406761373853,
|
| 580 |
+
"vsdev": 0.763363478930152
|
| 581 |
+
},
|
| 582 |
+
"objective": 0.7321020775337687
|
| 583 |
+
},
|
| 584 |
+
"gap20/g20/m10/t0.55": {
|
| 585 |
+
"decode": "gap20",
|
| 586 |
+
"gap": 20,
|
| 587 |
+
"minc": 10,
|
| 588 |
+
"threshold": 0.55,
|
| 589 |
+
"thr": {
|
| 590 |
+
"acl_val": 0.7031983407924899,
|
| 591 |
+
"vsdev": 0.7554289168069767
|
| 592 |
+
},
|
| 593 |
+
"objective": 0.7293136287997333
|
| 594 |
+
},
|
| 595 |
+
"gap20/g20/m10/t0.6": {
|
| 596 |
+
"decode": "gap20",
|
| 597 |
+
"gap": 20,
|
| 598 |
+
"minc": 10,
|
| 599 |
+
"threshold": 0.6,
|
| 600 |
+
"thr": {
|
| 601 |
+
"acl_val": 0.6839368833223125,
|
| 602 |
+
"vsdev": 0.7280812619950383
|
| 603 |
+
},
|
| 604 |
+
"objective": 0.7060090726586754
|
| 605 |
+
},
|
| 606 |
+
"gap20/g20/m10/t0.65": {
|
| 607 |
+
"decode": "gap20",
|
| 608 |
+
"gap": 20,
|
| 609 |
+
"minc": 10,
|
| 610 |
+
"threshold": 0.65,
|
| 611 |
+
"thr": {
|
| 612 |
+
"acl_val": 0.6535705460835187,
|
| 613 |
+
"vsdev": 0.6929425298854599
|
| 614 |
+
},
|
| 615 |
+
"objective": 0.6732565379844893
|
| 616 |
+
},
|
| 617 |
+
"gap20/g20/m10/t0.7": {
|
| 618 |
+
"decode": "gap20",
|
| 619 |
+
"gap": 20,
|
| 620 |
+
"minc": 10,
|
| 621 |
+
"threshold": 0.7,
|
| 622 |
+
"thr": {
|
| 623 |
+
"acl_val": 0.6150554423096384,
|
| 624 |
+
"vsdev": 0.639378515938923
|
| 625 |
+
},
|
| 626 |
+
"objective": 0.6272169791242808
|
| 627 |
+
},
|
| 628 |
+
"gap20/g20/m10/t0.75": {
|
| 629 |
+
"decode": "gap20",
|
| 630 |
+
"gap": 20,
|
| 631 |
+
"minc": 10,
|
| 632 |
+
"threshold": 0.75,
|
| 633 |
+
"thr": {
|
| 634 |
+
"acl_val": 0.5751018175186918,
|
| 635 |
+
"vsdev": 0.581010650351235
|
| 636 |
+
},
|
| 637 |
+
"objective": 0.5780562339349634
|
| 638 |
+
},
|
| 639 |
+
"gap20/g20/m10/t0.8": {
|
| 640 |
+
"decode": "gap20",
|
| 641 |
+
"gap": 20,
|
| 642 |
+
"minc": 10,
|
| 643 |
+
"threshold": 0.8,
|
| 644 |
+
"thr": {
|
| 645 |
+
"acl_val": 0.5296373502219818,
|
| 646 |
+
"vsdev": 0.516694691328369
|
| 647 |
+
},
|
| 648 |
+
"objective": 0.5231660207751754
|
| 649 |
+
},
|
| 650 |
+
"gap20/g20/m10/t0.85": {
|
| 651 |
+
"decode": "gap20",
|
| 652 |
+
"gap": 20,
|
| 653 |
+
"minc": 10,
|
| 654 |
+
"threshold": 0.85,
|
| 655 |
+
"thr": {
|
| 656 |
+
"acl_val": 0.47460572039561616,
|
| 657 |
+
"vsdev": 0.46528868505371623
|
| 658 |
+
},
|
| 659 |
+
"objective": 0.46994720272466617
|
| 660 |
+
},
|
| 661 |
+
"gap20/g20/m10/t0.9": {
|
| 662 |
+
"decode": "gap20",
|
| 663 |
+
"gap": 20,
|
| 664 |
+
"minc": 10,
|
| 665 |
+
"threshold": 0.9,
|
| 666 |
+
"thr": {
|
| 667 |
+
"acl_val": 0.384585219778261,
|
| 668 |
+
"vsdev": 0.3849664429530201
|
| 669 |
+
},
|
| 670 |
+
"objective": 0.38477583136564053
|
| 671 |
+
},
|
| 672 |
+
"gap20/g20/m10/t0.95": {
|
| 673 |
+
"decode": "gap20",
|
| 674 |
+
"gap": 20,
|
| 675 |
+
"minc": 10,
|
| 676 |
+
"threshold": 0.95,
|
| 677 |
+
"thr": {
|
| 678 |
+
"acl_val": 0.22994505494505493,
|
| 679 |
+
"vsdev": 0.27132846926244314
|
| 680 |
+
},
|
| 681 |
+
"objective": 0.250636762103749
|
| 682 |
+
},
|
| 683 |
+
"gap50/g50/m10/t0.05": {
|
| 684 |
+
"decode": "gap50",
|
| 685 |
+
"gap": 50,
|
| 686 |
+
"minc": 10,
|
| 687 |
+
"threshold": 0.05,
|
| 688 |
+
"thr": {
|
| 689 |
+
"acl_val": 0.4734027397260274,
|
| 690 |
+
"vsdev": 0.5117762932511133
|
| 691 |
+
},
|
| 692 |
+
"objective": 0.49258951648857036
|
| 693 |
+
},
|
| 694 |
+
"gap50/g50/m10/t0.1": {
|
| 695 |
+
"decode": "gap50",
|
| 696 |
+
"gap": 50,
|
| 697 |
+
"minc": 10,
|
| 698 |
+
"threshold": 0.1,
|
| 699 |
+
"thr": {
|
| 700 |
+
"acl_val": 0.5209060727455377,
|
| 701 |
+
"vsdev": 0.554428793539554
|
| 702 |
+
},
|
| 703 |
+
"objective": 0.5376674331425458
|
| 704 |
+
},
|
| 705 |
+
"gap50/g50/m10/t0.15": {
|
| 706 |
+
"decode": "gap50",
|
| 707 |
+
"gap": 50,
|
| 708 |
+
"minc": 10,
|
| 709 |
+
"threshold": 0.15,
|
| 710 |
+
"thr": {
|
| 711 |
+
"acl_val": 0.5564907931303357,
|
| 712 |
+
"vsdev": 0.5900965585540975
|
| 713 |
+
},
|
| 714 |
+
"objective": 0.5732936758422166
|
| 715 |
+
},
|
| 716 |
+
"gap50/g50/m10/t0.2": {
|
| 717 |
+
"decode": "gap50",
|
| 718 |
+
"gap": 50,
|
| 719 |
+
"minc": 10,
|
| 720 |
+
"threshold": 0.2,
|
| 721 |
+
"thr": {
|
| 722 |
+
"acl_val": 0.5850439987174463,
|
| 723 |
+
"vsdev": 0.6276890589943249
|
| 724 |
+
},
|
| 725 |
+
"objective": 0.6063665288558856
|
| 726 |
+
},
|
| 727 |
+
"gap50/g50/m10/t0.25": {
|
| 728 |
+
"decode": "gap50",
|
| 729 |
+
"gap": 50,
|
| 730 |
+
"minc": 10,
|
| 731 |
+
"threshold": 0.25,
|
| 732 |
+
"thr": {
|
| 733 |
+
"acl_val": 0.6111456010055721,
|
| 734 |
+
"vsdev": 0.663598866410393
|
| 735 |
+
},
|
| 736 |
+
"objective": 0.6373722337079826
|
| 737 |
+
},
|
| 738 |
+
"gap50/g50/m10/t0.3": {
|
| 739 |
+
"decode": "gap50",
|
| 740 |
+
"gap": 50,
|
| 741 |
+
"minc": 10,
|
| 742 |
+
"threshold": 0.3,
|
| 743 |
+
"thr": {
|
| 744 |
+
"acl_val": 0.6352007932573128,
|
| 745 |
+
"vsdev": 0.6941295186086544
|
| 746 |
+
},
|
| 747 |
+
"objective": 0.6646651559329836
|
| 748 |
+
},
|
| 749 |
+
"gap50/g50/m10/t0.35": {
|
| 750 |
+
"decode": "gap50",
|
| 751 |
+
"gap": 50,
|
| 752 |
+
"minc": 10,
|
| 753 |
+
"threshold": 0.35,
|
| 754 |
+
"thr": {
|
| 755 |
+
"acl_val": 0.6565348469175751,
|
| 756 |
+
"vsdev": 0.7111571285140563
|
| 757 |
+
},
|
| 758 |
+
"objective": 0.6838459877158157
|
| 759 |
+
},
|
| 760 |
+
"gap50/g50/m10/t0.4": {
|
| 761 |
+
"decode": "gap50",
|
| 762 |
+
"gap": 50,
|
| 763 |
+
"minc": 10,
|
| 764 |
+
"threshold": 0.4,
|
| 765 |
+
"thr": {
|
| 766 |
+
"acl_val": 0.6705533258516729,
|
| 767 |
+
"vsdev": 0.7319106368789596
|
| 768 |
+
},
|
| 769 |
+
"objective": 0.7012319813653163
|
| 770 |
+
},
|
| 771 |
+
"gap50/g50/m10/t0.45": {
|
| 772 |
+
"decode": "gap50",
|
| 773 |
+
"gap": 50,
|
| 774 |
+
"minc": 10,
|
| 775 |
+
"threshold": 0.45,
|
| 776 |
+
"thr": {
|
| 777 |
+
"acl_val": 0.6908421290866819,
|
| 778 |
+
"vsdev": 0.756150378075189
|
| 779 |
+
},
|
| 780 |
+
"objective": 0.7234962535809355
|
| 781 |
+
},
|
| 782 |
+
"gap50/g50/m10/t0.5": {
|
| 783 |
+
"decode": "gap50",
|
| 784 |
+
"gap": 50,
|
| 785 |
+
"minc": 10,
|
| 786 |
+
"threshold": 0.5,
|
| 787 |
+
"thr": {
|
| 788 |
+
"acl_val": 0.6992094421556618,
|
| 789 |
+
"vsdev": 0.7594427362107581
|
| 790 |
+
},
|
| 791 |
+
"objective": 0.72932608918321
|
| 792 |
+
},
|
| 793 |
+
"gap50/g50/m10/t0.55": {
|
| 794 |
+
"decode": "gap50",
|
| 795 |
+
"gap": 50,
|
| 796 |
+
"minc": 10,
|
| 797 |
+
"threshold": 0.55,
|
| 798 |
+
"thr": {
|
| 799 |
+
"acl_val": 0.7032488018688438,
|
| 800 |
+
"vsdev": 0.756326268464997
|
| 801 |
+
},
|
| 802 |
+
"objective": 0.7297875351669203
|
| 803 |
+
},
|
| 804 |
+
"gap50/g50/m10/t0.6": {
|
| 805 |
+
"decode": "gap50",
|
| 806 |
+
"gap": 50,
|
| 807 |
+
"minc": 10,
|
| 808 |
+
"threshold": 0.6,
|
| 809 |
+
"thr": {
|
| 810 |
+
"acl_val": 0.68566493955095,
|
| 811 |
+
"vsdev": 0.7306830907054872
|
| 812 |
+
},
|
| 813 |
+
"objective": 0.7081740151282185
|
| 814 |
+
},
|
| 815 |
+
"gap50/g50/m10/t0.65": {
|
| 816 |
+
"decode": "gap50",
|
| 817 |
+
"gap": 50,
|
| 818 |
+
"minc": 10,
|
| 819 |
+
"threshold": 0.65,
|
| 820 |
+
"thr": {
|
| 821 |
+
"acl_val": 0.6556266054544462,
|
| 822 |
+
"vsdev": 0.6957691732695469
|
| 823 |
+
},
|
| 824 |
+
"objective": 0.6756978893619965
|
| 825 |
+
},
|
| 826 |
+
"gap50/g50/m10/t0.7": {
|
| 827 |
+
"decode": "gap50",
|
| 828 |
+
"gap": 50,
|
| 829 |
+
"minc": 10,
|
| 830 |
+
"threshold": 0.7,
|
| 831 |
+
"thr": {
|
| 832 |
+
"acl_val": 0.618599099811622,
|
| 833 |
+
"vsdev": 0.6428952137298796
|
| 834 |
+
},
|
| 835 |
+
"objective": 0.6307471567707508
|
| 836 |
+
},
|
| 837 |
+
"gap50/g50/m10/t0.75": {
|
| 838 |
+
"decode": "gap50",
|
| 839 |
+
"gap": 50,
|
| 840 |
+
"minc": 10,
|
| 841 |
+
"threshold": 0.75,
|
| 842 |
+
"thr": {
|
| 843 |
+
"acl_val": 0.5777266396172712,
|
| 844 |
+
"vsdev": 0.584719481251762
|
| 845 |
+
},
|
| 846 |
+
"objective": 0.5812230604345165
|
| 847 |
+
},
|
| 848 |
+
"gap50/g50/m10/t0.8": {
|
| 849 |
+
"decode": "gap50",
|
| 850 |
+
"gap": 50,
|
| 851 |
+
"minc": 10,
|
| 852 |
+
"threshold": 0.8,
|
| 853 |
+
"thr": {
|
| 854 |
+
"acl_val": 0.5329041260981257,
|
| 855 |
+
"vsdev": 0.5218899521531101
|
| 856 |
+
},
|
| 857 |
+
"objective": 0.5273970391256179
|
| 858 |
+
},
|
| 859 |
+
"gap50/g50/m10/t0.85": {
|
| 860 |
+
"decode": "gap50",
|
| 861 |
+
"gap": 50,
|
| 862 |
+
"minc": 10,
|
| 863 |
+
"threshold": 0.85,
|
| 864 |
+
"thr": {
|
| 865 |
+
"acl_val": 0.4773757163801146,
|
| 866 |
+
"vsdev": 0.46867371847030104
|
| 867 |
+
},
|
| 868 |
+
"objective": 0.4730247174252078
|
| 869 |
+
},
|
| 870 |
+
"gap50/g50/m10/t0.9": {
|
| 871 |
+
"decode": "gap50",
|
| 872 |
+
"gap": 50,
|
| 873 |
+
"minc": 10,
|
| 874 |
+
"threshold": 0.9,
|
| 875 |
+
"thr": {
|
| 876 |
+
"acl_val": 0.3889738030071446,
|
| 877 |
+
"vsdev": 0.3933529097704218
|
| 878 |
+
},
|
| 879 |
+
"objective": 0.3911633563887832
|
| 880 |
+
},
|
| 881 |
+
"gap50/g50/m10/t0.95": {
|
| 882 |
+
"decode": "gap50",
|
| 883 |
+
"gap": 50,
|
| 884 |
+
"minc": 10,
|
| 885 |
+
"threshold": 0.95,
|
| 886 |
+
"thr": {
|
| 887 |
+
"acl_val": 0.23679580906212128,
|
| 888 |
+
"vsdev": 0.2750666762776616
|
| 889 |
+
},
|
| 890 |
+
"objective": 0.2559312426698914
|
| 891 |
+
},
|
| 892 |
+
"gap100/g100/m10/t0.05": {
|
| 893 |
+
"decode": "gap100",
|
| 894 |
+
"gap": 100,
|
| 895 |
+
"minc": 10,
|
| 896 |
+
"threshold": 0.05,
|
| 897 |
+
"thr": {
|
| 898 |
+
"acl_val": 0.46425857548028043,
|
| 899 |
+
"vsdev": 0.5009319176561747
|
| 900 |
+
},
|
| 901 |
+
"objective": 0.4825952465682276
|
| 902 |
+
},
|
| 903 |
+
"gap100/g100/m10/t0.1": {
|
| 904 |
+
"decode": "gap100",
|
| 905 |
+
"gap": 100,
|
| 906 |
+
"minc": 10,
|
| 907 |
+
"threshold": 0.1,
|
| 908 |
+
"thr": {
|
| 909 |
+
"acl_val": 0.5132859631042567,
|
| 910 |
+
"vsdev": 0.5468346327076523
|
| 911 |
+
},
|
| 912 |
+
"objective": 0.5300602979059545
|
| 913 |
+
},
|
| 914 |
+
"gap100/g100/m10/t0.15": {
|
| 915 |
+
"decode": "gap100",
|
| 916 |
+
"gap": 100,
|
| 917 |
+
"minc": 10,
|
| 918 |
+
"threshold": 0.15,
|
| 919 |
+
"thr": {
|
| 920 |
+
"acl_val": 0.5509173406983626,
|
| 921 |
+
"vsdev": 0.5820498425638898
|
| 922 |
+
},
|
| 923 |
+
"objective": 0.5664835916311262
|
| 924 |
+
},
|
| 925 |
+
"gap100/g100/m10/t0.2": {
|
| 926 |
+
"decode": "gap100",
|
| 927 |
+
"gap": 100,
|
| 928 |
+
"minc": 10,
|
| 929 |
+
"threshold": 0.2,
|
| 930 |
+
"thr": {
|
| 931 |
+
"acl_val": 0.5806183123725955,
|
| 932 |
+
"vsdev": 0.6226646808288261
|
| 933 |
+
},
|
| 934 |
+
"objective": 0.6016414966007109
|
| 935 |
+
},
|
| 936 |
+
"gap100/g100/m10/t0.25": {
|
| 937 |
+
"decode": "gap100",
|
| 938 |
+
"gap": 100,
|
| 939 |
+
"minc": 10,
|
| 940 |
+
"threshold": 0.25,
|
| 941 |
+
"thr": {
|
| 942 |
+
"acl_val": 0.6068580360733608,
|
| 943 |
+
"vsdev": 0.6593039752969649
|
| 944 |
+
},
|
| 945 |
+
"objective": 0.6330810056851628
|
| 946 |
+
},
|
| 947 |
+
"gap100/g100/m10/t0.3": {
|
| 948 |
+
"decode": "gap100",
|
| 949 |
+
"gap": 100,
|
| 950 |
+
"minc": 10,
|
| 951 |
+
"threshold": 0.3,
|
| 952 |
+
"thr": {
|
| 953 |
+
"acl_val": 0.6327319798784184,
|
| 954 |
+
"vsdev": 0.6887896067250603
|
| 955 |
+
},
|
| 956 |
+
"objective": 0.6607607933017394
|
| 957 |
+
},
|
| 958 |
+
"gap100/g100/m10/t0.35": {
|
| 959 |
+
"decode": "gap100",
|
| 960 |
+
"gap": 100,
|
| 961 |
+
"minc": 10,
|
| 962 |
+
"threshold": 0.35,
|
| 963 |
+
"thr": {
|
| 964 |
+
"acl_val": 0.6537587220783779,
|
| 965 |
+
"vsdev": 0.7074736645265849
|
| 966 |
+
},
|
| 967 |
+
"objective": 0.6806161933024815
|
| 968 |
+
},
|
| 969 |
+
"gap100/g100/m10/t0.4": {
|
| 970 |
+
"decode": "gap100",
|
| 971 |
+
"gap": 100,
|
| 972 |
+
"minc": 10,
|
| 973 |
+
"threshold": 0.4,
|
| 974 |
+
"thr": {
|
| 975 |
+
"acl_val": 0.6689324421283185,
|
| 976 |
+
"vsdev": 0.727002673532033
|
| 977 |
+
},
|
| 978 |
+
"objective": 0.6979675578301757
|
| 979 |
+
},
|
| 980 |
+
"gap100/g100/m10/t0.45": {
|
| 981 |
+
"decode": "gap100",
|
| 982 |
+
"gap": 100,
|
| 983 |
+
"minc": 10,
|
| 984 |
+
"threshold": 0.45,
|
| 985 |
+
"thr": {
|
| 986 |
+
"acl_val": 0.6910210280611986,
|
| 987 |
+
"vsdev": 0.755001060295469
|
| 988 |
+
},
|
| 989 |
+
"objective": 0.7230110441783337
|
| 990 |
+
},
|
| 991 |
+
"gap100/g100/m10/t0.5": {
|
| 992 |
+
"decode": "gap100",
|
| 993 |
+
"gap": 100,
|
| 994 |
+
"minc": 10,
|
| 995 |
+
"threshold": 0.5,
|
| 996 |
+
"thr": {
|
| 997 |
+
"acl_val": 0.699096348895599,
|
| 998 |
+
"vsdev": 0.7594898498226548
|
| 999 |
+
},
|
| 1000 |
+
"objective": 0.7292930993591269
|
| 1001 |
+
},
|
| 1002 |
+
"gap100/g100/m10/t0.55": {
|
| 1003 |
+
"decode": "gap100",
|
| 1004 |
+
"gap": 100,
|
| 1005 |
+
"minc": 10,
|
| 1006 |
+
"threshold": 0.55,
|
| 1007 |
+
"thr": {
|
| 1008 |
+
"acl_val": 0.7034296115342243,
|
| 1009 |
+
"vsdev": 0.7519560160710509
|
| 1010 |
+
},
|
| 1011 |
+
"objective": 0.7276928138026376
|
| 1012 |
+
},
|
| 1013 |
+
"gap100/g100/m10/t0.6": {
|
| 1014 |
+
"decode": "gap100",
|
| 1015 |
+
"gap": 100,
|
| 1016 |
+
"minc": 10,
|
| 1017 |
+
"threshold": 0.6,
|
| 1018 |
+
"thr": {
|
| 1019 |
+
"acl_val": 0.6882425235559197,
|
| 1020 |
+
"vsdev": 0.7298877238563608
|
| 1021 |
+
},
|
| 1022 |
+
"objective": 0.7090651237061403
|
| 1023 |
+
},
|
| 1024 |
+
"gap100/g100/m10/t0.65": {
|
| 1025 |
+
"decode": "gap100",
|
| 1026 |
+
"gap": 100,
|
| 1027 |
+
"minc": 10,
|
| 1028 |
+
"threshold": 0.65,
|
| 1029 |
+
"thr": {
|
| 1030 |
+
"acl_val": 0.6582027267660773,
|
| 1031 |
+
"vsdev": 0.7029375462848678
|
| 1032 |
+
},
|
| 1033 |
+
"objective": 0.6805701365254726
|
| 1034 |
+
},
|
| 1035 |
+
"gap100/g100/m10/t0.7": {
|
| 1036 |
+
"decode": "gap100",
|
| 1037 |
+
"gap": 100,
|
| 1038 |
+
"minc": 10,
|
| 1039 |
+
"threshold": 0.7,
|
| 1040 |
+
"thr": {
|
| 1041 |
+
"acl_val": 0.6240045750232326,
|
| 1042 |
+
"vsdev": 0.6469497005353263
|
| 1043 |
+
},
|
| 1044 |
+
"objective": 0.6354771377792794
|
| 1045 |
+
},
|
| 1046 |
+
"gap100/g100/m10/t0.75": {
|
| 1047 |
+
"decode": "gap100",
|
| 1048 |
+
"gap": 100,
|
| 1049 |
+
"minc": 10,
|
| 1050 |
+
"threshold": 0.75,
|
| 1051 |
+
"thr": {
|
| 1052 |
+
"acl_val": 0.580807623872855,
|
| 1053 |
+
"vsdev": 0.5901804730411389
|
| 1054 |
+
},
|
| 1055 |
+
"objective": 0.5854940484569969
|
| 1056 |
+
},
|
| 1057 |
+
"gap100/g100/m10/t0.8": {
|
| 1058 |
+
"decode": "gap100",
|
| 1059 |
+
"gap": 100,
|
| 1060 |
+
"minc": 10,
|
| 1061 |
+
"threshold": 0.8,
|
| 1062 |
+
"thr": {
|
| 1063 |
+
"acl_val": 0.5367087007363863,
|
| 1064 |
+
"vsdev": 0.5291319857312723
|
| 1065 |
+
},
|
| 1066 |
+
"objective": 0.5329203432338293
|
| 1067 |
+
},
|
| 1068 |
+
"gap100/g100/m10/t0.85": {
|
| 1069 |
+
"decode": "gap100",
|
| 1070 |
+
"gap": 100,
|
| 1071 |
+
"minc": 10,
|
| 1072 |
+
"threshold": 0.85,
|
| 1073 |
+
"thr": {
|
| 1074 |
+
"acl_val": 0.4788474655221175,
|
| 1075 |
+
"vsdev": 0.4727973636759311
|
| 1076 |
+
},
|
| 1077 |
+
"objective": 0.4758224145990243
|
| 1078 |
+
},
|
| 1079 |
+
"gap100/g100/m10/t0.9": {
|
| 1080 |
+
"decode": "gap100",
|
| 1081 |
+
"gap": 100,
|
| 1082 |
+
"minc": 10,
|
| 1083 |
+
"threshold": 0.9,
|
| 1084 |
+
"thr": {
|
| 1085 |
+
"acl_val": 0.3925246901540807,
|
| 1086 |
+
"vsdev": 0.3967767714437933
|
| 1087 |
+
},
|
| 1088 |
+
"objective": 0.394650730798937
|
| 1089 |
+
},
|
| 1090 |
+
"gap100/g100/m10/t0.95": {
|
| 1091 |
+
"decode": "gap100",
|
| 1092 |
+
"gap": 100,
|
| 1093 |
+
"minc": 10,
|
| 1094 |
+
"threshold": 0.95,
|
| 1095 |
+
"thr": {
|
| 1096 |
+
"acl_val": 0.24355953655924448,
|
| 1097 |
+
"vsdev": 0.2750666762776616
|
| 1098 |
+
},
|
| 1099 |
+
"objective": 0.25931310641845307
|
| 1100 |
+
},
|
| 1101 |
+
"sent/g0/m0/t0.05": {
|
| 1102 |
+
"decode": "sent",
|
| 1103 |
+
"gap": 0,
|
| 1104 |
+
"minc": 0,
|
| 1105 |
+
"threshold": 0.05,
|
| 1106 |
+
"thr": {
|
| 1107 |
+
"acl_val": 0.5006147900930968,
|
| 1108 |
+
"vsdev": 0.5363546709700556
|
| 1109 |
+
},
|
| 1110 |
+
"objective": 0.5184847305315763
|
| 1111 |
+
},
|
| 1112 |
+
"sent/g0/m0/t0.1": {
|
| 1113 |
+
"decode": "sent",
|
| 1114 |
+
"gap": 0,
|
| 1115 |
+
"minc": 0,
|
| 1116 |
+
"threshold": 0.1,
|
| 1117 |
+
"thr": {
|
| 1118 |
+
"acl_val": 0.5442991235400432,
|
| 1119 |
+
"vsdev": 0.5812747937404171
|
| 1120 |
+
},
|
| 1121 |
+
"objective": 0.5627869586402301
|
| 1122 |
+
},
|
| 1123 |
+
"sent/g0/m0/t0.15": {
|
| 1124 |
+
"decode": "sent",
|
| 1125 |
+
"gap": 0,
|
| 1126 |
+
"minc": 0,
|
| 1127 |
+
"threshold": 0.15,
|
| 1128 |
+
"thr": {
|
| 1129 |
+
"acl_val": 0.5775545277983484,
|
| 1130 |
+
"vsdev": 0.6189166753138944
|
| 1131 |
+
},
|
| 1132 |
+
"objective": 0.5982356015561214
|
| 1133 |
+
},
|
| 1134 |
+
"sent/g0/m0/t0.2": {
|
| 1135 |
+
"decode": "sent",
|
| 1136 |
+
"gap": 0,
|
| 1137 |
+
"minc": 0,
|
| 1138 |
+
"threshold": 0.2,
|
| 1139 |
+
"thr": {
|
| 1140 |
+
"acl_val": 0.602438915308815,
|
| 1141 |
+
"vsdev": 0.6478565776189047
|
| 1142 |
+
},
|
| 1143 |
+
"objective": 0.6251477464638598
|
| 1144 |
+
},
|
| 1145 |
+
"sent/g0/m0/t0.25": {
|
| 1146 |
+
"decode": "sent",
|
| 1147 |
+
"gap": 0,
|
| 1148 |
+
"minc": 0,
|
| 1149 |
+
"threshold": 0.25,
|
| 1150 |
+
"thr": {
|
| 1151 |
+
"acl_val": 0.6253852664526741,
|
| 1152 |
+
"vsdev": 0.6814412436071542
|
| 1153 |
+
},
|
| 1154 |
+
"objective": 0.6534132550299141
|
| 1155 |
+
},
|
| 1156 |
+
"sent/g0/m0/t0.3": {
|
| 1157 |
+
"decode": "sent",
|
| 1158 |
+
"gap": 0,
|
| 1159 |
+
"minc": 0,
|
| 1160 |
+
"threshold": 0.3,
|
| 1161 |
+
"thr": {
|
| 1162 |
+
"acl_val": 0.6468015917162294,
|
| 1163 |
+
"vsdev": 0.7066528573169246
|
| 1164 |
+
},
|
| 1165 |
+
"objective": 0.676727224516577
|
| 1166 |
+
},
|
| 1167 |
+
"sent/g0/m0/t0.35": {
|
| 1168 |
+
"decode": "sent",
|
| 1169 |
+
"gap": 0,
|
| 1170 |
+
"minc": 0,
|
| 1171 |
+
"threshold": 0.35,
|
| 1172 |
+
"thr": {
|
| 1173 |
+
"acl_val": 0.6652333421772504,
|
| 1174 |
+
"vsdev": 0.7211334575595966
|
| 1175 |
+
},
|
| 1176 |
+
"objective": 0.6931833998684235
|
| 1177 |
+
},
|
| 1178 |
+
"sent/g0/m0/t0.4": {
|
| 1179 |
+
"decode": "sent",
|
| 1180 |
+
"gap": 0,
|
| 1181 |
+
"minc": 0,
|
| 1182 |
+
"threshold": 0.4,
|
| 1183 |
+
"thr": {
|
| 1184 |
+
"acl_val": 0.6799453231790665,
|
| 1185 |
+
"vsdev": 0.7448233444953188
|
| 1186 |
+
},
|
| 1187 |
+
"objective": 0.7123843338371927
|
| 1188 |
+
},
|
| 1189 |
+
"sent/g0/m0/t0.45": {
|
| 1190 |
+
"decode": "sent",
|
| 1191 |
+
"gap": 0,
|
| 1192 |
+
"minc": 0,
|
| 1193 |
+
"threshold": 0.45,
|
| 1194 |
+
"thr": {
|
| 1195 |
+
"acl_val": 0.6951572979033998,
|
| 1196 |
+
"vsdev": 0.7626867291207002
|
| 1197 |
+
},
|
| 1198 |
+
"objective": 0.72892201351205
|
| 1199 |
+
},
|
| 1200 |
+
"sent/g0/m0/t0.5": {
|
| 1201 |
+
"decode": "sent",
|
| 1202 |
+
"gap": 0,
|
| 1203 |
+
"minc": 0,
|
| 1204 |
+
"threshold": 0.5,
|
| 1205 |
+
"thr": {
|
| 1206 |
+
"acl_val": 0.7027774942442698,
|
| 1207 |
+
"vsdev": 0.7666041568995156
|
| 1208 |
+
},
|
| 1209 |
+
"objective": 0.7346908255718927
|
| 1210 |
+
},
|
| 1211 |
+
"sent/g0/m0/t0.55": {
|
| 1212 |
+
"decode": "sent",
|
| 1213 |
+
"gap": 0,
|
| 1214 |
+
"minc": 0,
|
| 1215 |
+
"threshold": 0.55,
|
| 1216 |
+
"thr": {
|
| 1217 |
+
"acl_val": 0.7048052122678988,
|
| 1218 |
+
"vsdev": 0.7554376196276318
|
| 1219 |
+
},
|
| 1220 |
+
"objective": 0.7301214159477654
|
| 1221 |
+
},
|
| 1222 |
+
"sent/g0/m0/t0.6": {
|
| 1223 |
+
"decode": "sent",
|
| 1224 |
+
"gap": 0,
|
| 1225 |
+
"minc": 0,
|
| 1226 |
+
"threshold": 0.6,
|
| 1227 |
+
"thr": {
|
| 1228 |
+
"acl_val": 0.6817494262466096,
|
| 1229 |
+
"vsdev": 0.728512960436562
|
| 1230 |
+
},
|
| 1231 |
+
"objective": 0.7051311933415858
|
| 1232 |
+
},
|
| 1233 |
+
"sent/g0/m0/t0.65": {
|
| 1234 |
+
"decode": "sent",
|
| 1235 |
+
"gap": 0,
|
| 1236 |
+
"minc": 0,
|
| 1237 |
+
"threshold": 0.65,
|
| 1238 |
+
"thr": {
|
| 1239 |
+
"acl_val": 0.6517662631332212,
|
| 1240 |
+
"vsdev": 0.6923541247484909
|
| 1241 |
+
},
|
| 1242 |
+
"objective": 0.672060193940856
|
| 1243 |
+
},
|
| 1244 |
+
"sent/g0/m0/t0.7": {
|
| 1245 |
+
"decode": "sent",
|
| 1246 |
+
"gap": 0,
|
| 1247 |
+
"minc": 0,
|
| 1248 |
+
"threshold": 0.7,
|
| 1249 |
+
"thr": {
|
| 1250 |
+
"acl_val": 0.611627196999433,
|
| 1251 |
+
"vsdev": 0.6402458751213199
|
| 1252 |
+
},
|
| 1253 |
+
"objective": 0.6259365360603765
|
| 1254 |
+
},
|
| 1255 |
+
"sent/g0/m0/t0.75": {
|
| 1256 |
+
"decode": "sent",
|
| 1257 |
+
"gap": 0,
|
| 1258 |
+
"minc": 0,
|
| 1259 |
+
"threshold": 0.75,
|
| 1260 |
+
"thr": {
|
| 1261 |
+
"acl_val": 0.5710671117411251,
|
| 1262 |
+
"vsdev": 0.5736779051856924
|
| 1263 |
+
},
|
| 1264 |
+
"objective": 0.5723725084634088
|
| 1265 |
+
},
|
| 1266 |
+
"sent/g0/m0/t0.8": {
|
| 1267 |
+
"decode": "sent",
|
| 1268 |
+
"gap": 0,
|
| 1269 |
+
"minc": 0,
|
| 1270 |
+
"threshold": 0.8,
|
| 1271 |
+
"thr": {
|
| 1272 |
+
"acl_val": 0.5246163938623017,
|
| 1273 |
+
"vsdev": 0.5115043178935926
|
| 1274 |
+
},
|
| 1275 |
+
"objective": 0.5180603558779472
|
| 1276 |
+
},
|
| 1277 |
+
"sent/g0/m0/t0.85": {
|
| 1278 |
+
"decode": "sent",
|
| 1279 |
+
"gap": 0,
|
| 1280 |
+
"minc": 0,
|
| 1281 |
+
"threshold": 0.85,
|
| 1282 |
+
"thr": {
|
| 1283 |
+
"acl_val": 0.46882081409987414,
|
| 1284 |
+
"vsdev": 0.4589088615578682
|
| 1285 |
+
},
|
| 1286 |
+
"objective": 0.46386483782887117
|
| 1287 |
+
},
|
| 1288 |
+
"sent/g0/m0/t0.9": {
|
| 1289 |
+
"decode": "sent",
|
| 1290 |
+
"gap": 0,
|
| 1291 |
+
"minc": 0,
|
| 1292 |
+
"threshold": 0.9,
|
| 1293 |
+
"thr": {
|
| 1294 |
+
"acl_val": 0.3768230193141506,
|
| 1295 |
+
"vsdev": 0.3725145407818206
|
| 1296 |
+
},
|
| 1297 |
+
"objective": 0.3746687800479856
|
| 1298 |
+
},
|
| 1299 |
+
"sent/g0/m0/t0.95": {
|
| 1300 |
+
"decode": "sent",
|
| 1301 |
+
"gap": 0,
|
| 1302 |
+
"minc": 0,
|
| 1303 |
+
"threshold": 0.95,
|
| 1304 |
+
"thr": {
|
| 1305 |
+
"acl_val": 0.21834630427053295,
|
| 1306 |
+
"vsdev": 0.2717030189224325
|
| 1307 |
+
},
|
| 1308 |
+
"objective": 0.24502466159648273
|
| 1309 |
+
},
|
| 1310 |
+
"sent/g1/m0/t0.05": {
|
| 1311 |
+
"decode": "sent",
|
| 1312 |
+
"gap": 1,
|
| 1313 |
+
"minc": 0,
|
| 1314 |
+
"threshold": 0.05,
|
| 1315 |
+
"thr": {
|
| 1316 |
+
"acl_val": 0.5006147900930968,
|
| 1317 |
+
"vsdev": 0.5363546709700556
|
| 1318 |
+
},
|
| 1319 |
+
"objective": 0.5184847305315763
|
| 1320 |
+
},
|
| 1321 |
+
"sent/g1/m0/t0.1": {
|
| 1322 |
+
"decode": "sent",
|
| 1323 |
+
"gap": 1,
|
| 1324 |
+
"minc": 0,
|
| 1325 |
+
"threshold": 0.1,
|
| 1326 |
+
"thr": {
|
| 1327 |
+
"acl_val": 0.5442991235400432,
|
| 1328 |
+
"vsdev": 0.5812747937404171
|
| 1329 |
+
},
|
| 1330 |
+
"objective": 0.5627869586402301
|
| 1331 |
+
},
|
| 1332 |
+
"sent/g1/m0/t0.15": {
|
| 1333 |
+
"decode": "sent",
|
| 1334 |
+
"gap": 1,
|
| 1335 |
+
"minc": 0,
|
| 1336 |
+
"threshold": 0.15,
|
| 1337 |
+
"thr": {
|
| 1338 |
+
"acl_val": 0.5775545277983484,
|
| 1339 |
+
"vsdev": 0.6189166753138944
|
| 1340 |
+
},
|
| 1341 |
+
"objective": 0.5982356015561214
|
| 1342 |
+
},
|
| 1343 |
+
"sent/g1/m0/t0.2": {
|
| 1344 |
+
"decode": "sent",
|
| 1345 |
+
"gap": 1,
|
| 1346 |
+
"minc": 0,
|
| 1347 |
+
"threshold": 0.2,
|
| 1348 |
+
"thr": {
|
| 1349 |
+
"acl_val": 0.602438915308815,
|
| 1350 |
+
"vsdev": 0.6478565776189047
|
| 1351 |
+
},
|
| 1352 |
+
"objective": 0.6251477464638598
|
| 1353 |
+
},
|
| 1354 |
+
"sent/g1/m0/t0.25": {
|
| 1355 |
+
"decode": "sent",
|
| 1356 |
+
"gap": 1,
|
| 1357 |
+
"minc": 0,
|
| 1358 |
+
"threshold": 0.25,
|
| 1359 |
+
"thr": {
|
| 1360 |
+
"acl_val": 0.6253852664526741,
|
| 1361 |
+
"vsdev": 0.6814412436071542
|
| 1362 |
+
},
|
| 1363 |
+
"objective": 0.6534132550299141
|
| 1364 |
+
},
|
| 1365 |
+
"sent/g1/m0/t0.3": {
|
| 1366 |
+
"decode": "sent",
|
| 1367 |
+
"gap": 1,
|
| 1368 |
+
"minc": 0,
|
| 1369 |
+
"threshold": 0.3,
|
| 1370 |
+
"thr": {
|
| 1371 |
+
"acl_val": 0.6468015917162294,
|
| 1372 |
+
"vsdev": 0.7066528573169246
|
| 1373 |
+
},
|
| 1374 |
+
"objective": 0.676727224516577
|
| 1375 |
+
},
|
| 1376 |
+
"sent/g1/m0/t0.35": {
|
| 1377 |
+
"decode": "sent",
|
| 1378 |
+
"gap": 1,
|
| 1379 |
+
"minc": 0,
|
| 1380 |
+
"threshold": 0.35,
|
| 1381 |
+
"thr": {
|
| 1382 |
+
"acl_val": 0.6652333421772504,
|
| 1383 |
+
"vsdev": 0.7211334575595966
|
| 1384 |
+
},
|
| 1385 |
+
"objective": 0.6931833998684235
|
| 1386 |
+
},
|
| 1387 |
+
"sent/g1/m0/t0.4": {
|
| 1388 |
+
"decode": "sent",
|
| 1389 |
+
"gap": 1,
|
| 1390 |
+
"minc": 0,
|
| 1391 |
+
"threshold": 0.4,
|
| 1392 |
+
"thr": {
|
| 1393 |
+
"acl_val": 0.6799453231790665,
|
| 1394 |
+
"vsdev": 0.7448233444953188
|
| 1395 |
+
},
|
| 1396 |
+
"objective": 0.7123843338371927
|
| 1397 |
+
},
|
| 1398 |
+
"sent/g1/m0/t0.45": {
|
| 1399 |
+
"decode": "sent",
|
| 1400 |
+
"gap": 1,
|
| 1401 |
+
"minc": 0,
|
| 1402 |
+
"threshold": 0.45,
|
| 1403 |
+
"thr": {
|
| 1404 |
+
"acl_val": 0.6951572979033998,
|
| 1405 |
+
"vsdev": 0.7626867291207002
|
| 1406 |
+
},
|
| 1407 |
+
"objective": 0.72892201351205
|
| 1408 |
+
},
|
| 1409 |
+
"sent/g1/m0/t0.5": {
|
| 1410 |
+
"decode": "sent",
|
| 1411 |
+
"gap": 1,
|
| 1412 |
+
"minc": 0,
|
| 1413 |
+
"threshold": 0.5,
|
| 1414 |
+
"thr": {
|
| 1415 |
+
"acl_val": 0.7027774942442698,
|
| 1416 |
+
"vsdev": 0.7666041568995156
|
| 1417 |
+
},
|
| 1418 |
+
"objective": 0.7346908255718927
|
| 1419 |
+
},
|
| 1420 |
+
"sent/g1/m0/t0.55": {
|
| 1421 |
+
"decode": "sent",
|
| 1422 |
+
"gap": 1,
|
| 1423 |
+
"minc": 0,
|
| 1424 |
+
"threshold": 0.55,
|
| 1425 |
+
"thr": {
|
| 1426 |
+
"acl_val": 0.7048052122678988,
|
| 1427 |
+
"vsdev": 0.7554376196276318
|
| 1428 |
+
},
|
| 1429 |
+
"objective": 0.7301214159477654
|
| 1430 |
+
},
|
| 1431 |
+
"sent/g1/m0/t0.6": {
|
| 1432 |
+
"decode": "sent",
|
| 1433 |
+
"gap": 1,
|
| 1434 |
+
"minc": 0,
|
| 1435 |
+
"threshold": 0.6,
|
| 1436 |
+
"thr": {
|
| 1437 |
+
"acl_val": 0.6817494262466096,
|
| 1438 |
+
"vsdev": 0.728512960436562
|
| 1439 |
+
},
|
| 1440 |
+
"objective": 0.7051311933415858
|
| 1441 |
+
},
|
| 1442 |
+
"sent/g1/m0/t0.65": {
|
| 1443 |
+
"decode": "sent",
|
| 1444 |
+
"gap": 1,
|
| 1445 |
+
"minc": 0,
|
| 1446 |
+
"threshold": 0.65,
|
| 1447 |
+
"thr": {
|
| 1448 |
+
"acl_val": 0.6517662631332212,
|
| 1449 |
+
"vsdev": 0.6923541247484909
|
| 1450 |
+
},
|
| 1451 |
+
"objective": 0.672060193940856
|
| 1452 |
+
},
|
| 1453 |
+
"sent/g1/m0/t0.7": {
|
| 1454 |
+
"decode": "sent",
|
| 1455 |
+
"gap": 1,
|
| 1456 |
+
"minc": 0,
|
| 1457 |
+
"threshold": 0.7,
|
| 1458 |
+
"thr": {
|
| 1459 |
+
"acl_val": 0.611627196999433,
|
| 1460 |
+
"vsdev": 0.6402458751213199
|
| 1461 |
+
},
|
| 1462 |
+
"objective": 0.6259365360603765
|
| 1463 |
+
},
|
| 1464 |
+
"sent/g1/m0/t0.75": {
|
| 1465 |
+
"decode": "sent",
|
| 1466 |
+
"gap": 1,
|
| 1467 |
+
"minc": 0,
|
| 1468 |
+
"threshold": 0.75,
|
| 1469 |
+
"thr": {
|
| 1470 |
+
"acl_val": 0.5710671117411251,
|
| 1471 |
+
"vsdev": 0.5736779051856924
|
| 1472 |
+
},
|
| 1473 |
+
"objective": 0.5723725084634088
|
| 1474 |
+
},
|
| 1475 |
+
"sent/g1/m0/t0.8": {
|
| 1476 |
+
"decode": "sent",
|
| 1477 |
+
"gap": 1,
|
| 1478 |
+
"minc": 0,
|
| 1479 |
+
"threshold": 0.8,
|
| 1480 |
+
"thr": {
|
| 1481 |
+
"acl_val": 0.5246163938623017,
|
| 1482 |
+
"vsdev": 0.5115043178935926
|
| 1483 |
+
},
|
| 1484 |
+
"objective": 0.5180603558779472
|
| 1485 |
+
},
|
| 1486 |
+
"sent/g1/m0/t0.85": {
|
| 1487 |
+
"decode": "sent",
|
| 1488 |
+
"gap": 1,
|
| 1489 |
+
"minc": 0,
|
| 1490 |
+
"threshold": 0.85,
|
| 1491 |
+
"thr": {
|
| 1492 |
+
"acl_val": 0.46882081409987414,
|
| 1493 |
+
"vsdev": 0.4589088615578682
|
| 1494 |
+
},
|
| 1495 |
+
"objective": 0.46386483782887117
|
| 1496 |
+
},
|
| 1497 |
+
"sent/g1/m0/t0.9": {
|
| 1498 |
+
"decode": "sent",
|
| 1499 |
+
"gap": 1,
|
| 1500 |
+
"minc": 0,
|
| 1501 |
+
"threshold": 0.9,
|
| 1502 |
+
"thr": {
|
| 1503 |
+
"acl_val": 0.3768230193141506,
|
| 1504 |
+
"vsdev": 0.3725145407818206
|
| 1505 |
+
},
|
| 1506 |
+
"objective": 0.3746687800479856
|
| 1507 |
+
},
|
| 1508 |
+
"sent/g1/m0/t0.95": {
|
| 1509 |
+
"decode": "sent",
|
| 1510 |
+
"gap": 1,
|
| 1511 |
+
"minc": 0,
|
| 1512 |
+
"threshold": 0.95,
|
| 1513 |
+
"thr": {
|
| 1514 |
+
"acl_val": 0.21834630427053295,
|
| 1515 |
+
"vsdev": 0.2717030189224325
|
| 1516 |
+
},
|
| 1517 |
+
"objective": 0.24502466159648273
|
| 1518 |
+
}
|
| 1519 |
+
}
|
| 1520 |
+
},
|
| 1521 |
+
"span_nb_router_reads": {
|
| 1522 |
+
"setting": {
|
| 1523 |
+
"ckpt": "ckpt-024000",
|
| 1524 |
+
"head": "final",
|
| 1525 |
+
"threshold_defs": 0.6,
|
| 1526 |
+
"threshold_names": 0.55
|
| 1527 |
+
},
|
| 1528 |
+
"sets": {
|
| 1529 |
+
"halu": {
|
| 1530 |
+
"threshold": 0.45,
|
| 1531 |
+
"arm": 0.9302558956347214,
|
| 1532 |
+
"B": 0.9098605577689242,
|
| 1533 |
+
"diff": 0.02039533786579728,
|
| 1534 |
+
"lo": 0.01211145360597379,
|
| 1535 |
+
"hi": 0.02884303966776603
|
| 1536 |
+
},
|
| 1537 |
+
"pii_short": {
|
| 1538 |
+
"threshold": 0.15,
|
| 1539 |
+
"arm": 1.0,
|
| 1540 |
+
"B": 1.0,
|
| 1541 |
+
"diff": 0.0,
|
| 1542 |
+
"lo": 0.0,
|
| 1543 |
+
"hi": 0.0
|
| 1544 |
+
},
|
| 1545 |
+
"pii_4096": {
|
| 1546 |
+
"threshold": 0.15,
|
| 1547 |
+
"arm": 0.9876044192939908,
|
| 1548 |
+
"B": 0.9458532623409439,
|
| 1549 |
+
"diff": 0.041751156953046875,
|
| 1550 |
+
"lo": 0.03169581355657979,
|
| 1551 |
+
"hi": 0.051694171532032
|
| 1552 |
+
},
|
| 1553 |
+
"pii_8192": {
|
| 1554 |
+
"threshold": 0.15,
|
| 1555 |
+
"arm": 0.9888612180212815,
|
| 1556 |
+
"B": 0.9409075322894045,
|
| 1557 |
+
"diff": 0.04795368573187697,
|
| 1558 |
+
"lo": 0.03853401618338624,
|
| 1559 |
+
"hi": 0.057281305609137155
|
| 1560 |
+
},
|
| 1561 |
+
"toxic": {
|
| 1562 |
+
"threshold": 0.2,
|
| 1563 |
+
"arm": 0.7154126435725006,
|
| 1564 |
+
"n": 2013,
|
| 1565 |
+
"at_060": 0.4198361119525234
|
| 1566 |
+
}
|
| 1567 |
+
}
|
| 1568 |
+
},
|
| 1569 |
+
"ner_pick": {
|
| 1570 |
+
"group": "N5",
|
| 1571 |
+
"rule": "max ner_shift eval-half exact micro-F1 over (tag, ckpt, mode); threshold from the thr half (FINE, ties lowest); head final; ties -> names < defs < ens, then earlier ckpt",
|
| 1572 |
+
"choice": {
|
| 1573 |
+
"tag": "N5",
|
| 1574 |
+
"ckpt": "ckpt-009000",
|
| 1575 |
+
"mode": "defs",
|
| 1576 |
+
"ckpt_dir": "N5/ckpt-009000",
|
| 1577 |
+
"threshold": 0.6,
|
| 1578 |
+
"thr_f1": 0.7715911049818996,
|
| 1579 |
+
"eval_f1": 0.7592236404965903,
|
| 1580 |
+
"rule_choice": {
|
| 1581 |
+
"tag": "N5",
|
| 1582 |
+
"ckpt": "ckpt-009000",
|
| 1583 |
+
"mode": "ens",
|
| 1584 |
+
"ckpt_dir": "N5/ckpt-009000",
|
| 1585 |
+
"threshold": 0.6,
|
| 1586 |
+
"thr_f1": 0.7715911049818996,
|
| 1587 |
+
"eval_f1": 0.7642020625764726
|
| 1588 |
+
}
|
| 1589 |
+
},
|
| 1590 |
+
"all": {
|
| 1591 |
+
"N5/ckpt-008000/names": {
|
| 1592 |
+
"threshold": 0.65,
|
| 1593 |
+
"thr_f1": 0.7464230305119807,
|
| 1594 |
+
"eval_f1": 0.7539668700959024
|
| 1595 |
+
},
|
| 1596 |
+
"N5/ckpt-008000/defs": {
|
| 1597 |
+
"threshold": 0.65,
|
| 1598 |
+
"thr_f1": 0.7635590216235377,
|
| 1599 |
+
"eval_f1": 0.7486153296408791
|
| 1600 |
+
},
|
| 1601 |
+
"N5/ckpt-008000/ens": {
|
| 1602 |
+
"threshold": 0.6,
|
| 1603 |
+
"thr_f1": 0.7638959714431413,
|
| 1604 |
+
"eval_f1": 0.757015742642026
|
| 1605 |
+
},
|
| 1606 |
+
"N5/ckpt-009000/names": {
|
| 1607 |
+
"threshold": 0.65,
|
| 1608 |
+
"thr_f1": 0.7550768144093237,
|
| 1609 |
+
"eval_f1": 0.7510175190231817
|
| 1610 |
+
},
|
| 1611 |
+
"N5/ckpt-009000/defs": {
|
| 1612 |
+
"threshold": 0.6,
|
| 1613 |
+
"thr_f1": 0.7699530516431925,
|
| 1614 |
+
"eval_f1": 0.7592236404965903
|
| 1615 |
+
},
|
| 1616 |
+
"N5/ckpt-009000/ens": {
|
| 1617 |
+
"threshold": 0.6,
|
| 1618 |
+
"thr_f1": 0.7715911049818996,
|
| 1619 |
+
"eval_f1": 0.7642020625764726
|
| 1620 |
+
},
|
| 1621 |
+
"N5/ckpt-010000/names": {
|
| 1622 |
+
"threshold": 0.65,
|
| 1623 |
+
"thr_f1": 0.7507864383082838,
|
| 1624 |
+
"eval_f1": 0.7534028637086795
|
| 1625 |
+
},
|
| 1626 |
+
"N5/ckpt-010000/defs": {
|
| 1627 |
+
"threshold": 0.6,
|
| 1628 |
+
"thr_f1": 0.7711511789181692,
|
| 1629 |
+
"eval_f1": 0.7592334494773519
|
| 1630 |
+
},
|
| 1631 |
+
"N5/ckpt-010000/ens": {
|
| 1632 |
+
"threshold": 0.6,
|
| 1633 |
+
"thr_f1": 0.7666209732693626,
|
| 1634 |
+
"eval_f1": 0.7629152896155853
|
| 1635 |
+
}
|
| 1636 |
+
}
|
| 1637 |
+
},
|
| 1638 |
+
"decision_pick": {
|
| 1639 |
+
"all": {
|
| 1640 |
+
"decision/ckpt-002000": {
|
| 1641 |
+
"domain": 0.9091596960319214,
|
| 1642 |
+
"factcheck": 1.0,
|
| 1643 |
+
"modality": 1.0,
|
| 1644 |
+
"feedback": 0.9321361983955841,
|
| 1645 |
+
"p_harm": 0.9593208581762928,
|
| 1646 |
+
"r_harm": 0.9707776717557252,
|
| 1647 |
+
"r_refusal": 0.9871599960339756,
|
| 1648 |
+
"attack": 0.9888497223845705,
|
| 1649 |
+
"massive_intent": 0.8157142857142857,
|
| 1650 |
+
"massive_scenario": 0.8785714285714286,
|
| 1651 |
+
"sgd_intent": 0.7328571428571429,
|
| 1652 |
+
"objective": 0.9249588181746297
|
| 1653 |
+
}
|
| 1654 |
+
},
|
| 1655 |
+
"choice": "decision/ckpt-002000"
|
| 1656 |
+
},
|
| 1657 |
+
"source": "Phase D final build on the 8x (d3-turbo); round-2 selection",
|
| 1658 |
+
"picks": {
|
| 1659 |
+
"decision": {
|
| 1660 |
+
"D": "B",
|
| 1661 |
+
"S": "B",
|
| 1662 |
+
"N": "B",
|
| 1663 |
+
"L": "B"
|
| 1664 |
+
},
|
| 1665 |
+
"why": {
|
| 1666 |
+
"rule": "port-first: the 27B round-2 adapters on frozen d3-turbo; only what failed the port is retrained",
|
| 1667 |
+
"approved": {
|
| 1668 |
+
"by": "lead (claude-review), 20 Oct 14:00Z",
|
| 1669 |
+
"note": "port-first: ported S (re-thresholded) + retrained L candidate (RE dev 40.65 vs 27B 37.05) + N-turbo ckpt-009000 (ens +1.1, hier +12.3 vs ported) + ported D with noncausal router pass (NB composite 86.32 = 27B, feedback 94.0 vs 92.9 prefix_bidir)",
|
| 1670 |
+
"N_src": "tbn",
|
| 1671 |
+
"N_ckpt": "ckpt-009000",
|
| 1672 |
+
"decisions_attention": "noncausal"
|
| 1673 |
+
}
|
| 1674 |
+
}
|
| 1675 |
+
},
|
| 1676 |
+
"router_gate": "not measured at this size (id gate)"
|
| 1677 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Qwen3_5Model"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoModel": "modeling_vela3.Vela3MiniModel"
|
| 7 |
+
},
|
| 8 |
+
"dtype": "bfloat16",
|
| 9 |
+
"image_token_id": 248056,
|
| 10 |
+
"language_model_only": false,
|
| 11 |
+
"model_type": "qwen3_5",
|
| 12 |
+
"text_config": {
|
| 13 |
+
"attention_bias": false,
|
| 14 |
+
"attention_dropout": 0.0,
|
| 15 |
+
"attn_output_gate": true,
|
| 16 |
+
"bos_token_id": 248044,
|
| 17 |
+
"dtype": "bfloat16",
|
| 18 |
+
"eos_token_id": 248044,
|
| 19 |
+
"full_attention_interval": 4,
|
| 20 |
+
"head_dim": 256,
|
| 21 |
+
"hidden_act": "silu",
|
| 22 |
+
"hidden_size": 5120,
|
| 23 |
+
"initializer_range": 0.02,
|
| 24 |
+
"intermediate_size": 17408,
|
| 25 |
+
"layer_types": [
|
| 26 |
+
"linear_attention",
|
| 27 |
+
"linear_attention",
|
| 28 |
+
"linear_attention",
|
| 29 |
+
"full_attention",
|
| 30 |
+
"linear_attention",
|
| 31 |
+
"linear_attention",
|
| 32 |
+
"linear_attention",
|
| 33 |
+
"full_attention",
|
| 34 |
+
"linear_attention",
|
| 35 |
+
"linear_attention",
|
| 36 |
+
"linear_attention",
|
| 37 |
+
"full_attention",
|
| 38 |
+
"linear_attention",
|
| 39 |
+
"linear_attention",
|
| 40 |
+
"linear_attention",
|
| 41 |
+
"full_attention",
|
| 42 |
+
"linear_attention",
|
| 43 |
+
"linear_attention",
|
| 44 |
+
"linear_attention",
|
| 45 |
+
"full_attention",
|
| 46 |
+
"linear_attention",
|
| 47 |
+
"linear_attention",
|
| 48 |
+
"linear_attention",
|
| 49 |
+
"full_attention",
|
| 50 |
+
"linear_attention",
|
| 51 |
+
"linear_attention",
|
| 52 |
+
"linear_attention",
|
| 53 |
+
"full_attention",
|
| 54 |
+
"linear_attention",
|
| 55 |
+
"linear_attention",
|
| 56 |
+
"linear_attention",
|
| 57 |
+
"full_attention",
|
| 58 |
+
"linear_attention",
|
| 59 |
+
"linear_attention",
|
| 60 |
+
"linear_attention",
|
| 61 |
+
"full_attention",
|
| 62 |
+
"linear_attention",
|
| 63 |
+
"linear_attention",
|
| 64 |
+
"linear_attention",
|
| 65 |
+
"full_attention",
|
| 66 |
+
"linear_attention",
|
| 67 |
+
"linear_attention",
|
| 68 |
+
"linear_attention",
|
| 69 |
+
"full_attention",
|
| 70 |
+
"linear_attention",
|
| 71 |
+
"linear_attention",
|
| 72 |
+
"linear_attention",
|
| 73 |
+
"full_attention",
|
| 74 |
+
"linear_attention",
|
| 75 |
+
"linear_attention",
|
| 76 |
+
"linear_attention",
|
| 77 |
+
"full_attention",
|
| 78 |
+
"linear_attention",
|
| 79 |
+
"linear_attention",
|
| 80 |
+
"linear_attention",
|
| 81 |
+
"full_attention",
|
| 82 |
+
"linear_attention",
|
| 83 |
+
"linear_attention",
|
| 84 |
+
"linear_attention",
|
| 85 |
+
"full_attention",
|
| 86 |
+
"linear_attention",
|
| 87 |
+
"linear_attention",
|
| 88 |
+
"linear_attention",
|
| 89 |
+
"full_attention"
|
| 90 |
+
],
|
| 91 |
+
"linear_conv_kernel_dim": 4,
|
| 92 |
+
"linear_key_head_dim": 128,
|
| 93 |
+
"linear_num_key_heads": 16,
|
| 94 |
+
"linear_num_value_heads": 48,
|
| 95 |
+
"linear_value_head_dim": 128,
|
| 96 |
+
"mamba_ssm_dtype": "float32",
|
| 97 |
+
"max_position_embeddings": 262144,
|
| 98 |
+
"model_type": "qwen3_5_text",
|
| 99 |
+
"mtp_num_hidden_layers": 1,
|
| 100 |
+
"mtp_use_dedicated_embeddings": false,
|
| 101 |
+
"num_attention_heads": 24,
|
| 102 |
+
"num_hidden_layers": 64,
|
| 103 |
+
"num_key_value_heads": 4,
|
| 104 |
+
"output_gate_type": "swish",
|
| 105 |
+
"pad_token_id": null,
|
| 106 |
+
"partial_rotary_factor": 0.25,
|
| 107 |
+
"rms_norm_eps": 1e-06,
|
| 108 |
+
"rope_parameters": {
|
| 109 |
+
"mrope_interleaved": true,
|
| 110 |
+
"mrope_section": [
|
| 111 |
+
11,
|
| 112 |
+
11,
|
| 113 |
+
10
|
| 114 |
+
],
|
| 115 |
+
"partial_rotary_factor": 0.25,
|
| 116 |
+
"rope_theta": 10000000,
|
| 117 |
+
"rope_type": "default"
|
| 118 |
+
},
|
| 119 |
+
"tie_word_embeddings": false,
|
| 120 |
+
"use_cache": true,
|
| 121 |
+
"vocab_size": 248320
|
| 122 |
+
},
|
| 123 |
+
"tie_word_embeddings": false,
|
| 124 |
+
"transformers_version": "5.17.0",
|
| 125 |
+
"video_token_id": 248057,
|
| 126 |
+
"vision_config": {
|
| 127 |
+
"deepstack_visual_indexes": [],
|
| 128 |
+
"depth": 27,
|
| 129 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 130 |
+
"hidden_size": 1152,
|
| 131 |
+
"in_channels": 3,
|
| 132 |
+
"initializer_range": 0.02,
|
| 133 |
+
"intermediate_size": 4304,
|
| 134 |
+
"model_type": "qwen3_5_vision",
|
| 135 |
+
"num_heads": 16,
|
| 136 |
+
"num_position_embeddings": 2304,
|
| 137 |
+
"out_hidden_size": 5120,
|
| 138 |
+
"patch_size": 16,
|
| 139 |
+
"rope_parameters": {
|
| 140 |
+
"rope_theta": 10000.0,
|
| 141 |
+
"rope_type": "axial"
|
| 142 |
+
},
|
| 143 |
+
"spatial_merge_size": 2,
|
| 144 |
+
"temporal_patch_size": 2
|
| 145 |
+
},
|
| 146 |
+
"vision_end_token_id": 248054,
|
| 147 |
+
"vision_start_token_id": 248053
|
| 148 |
+
}
|
decision_df_lora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05b8d0cf65b4292a9c2764b8316d53a66f813b8f6742b78d1f75c7c3aa36167e
|
| 3 |
+
size 870400216
|
decision_lora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6ceb0a0e0ea66d95db7d32ebd0a01631f2596870ebeb52d83300e2f2fef68b6
|
| 3 |
+
size 870400176
|
label_definitions.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "label-definition cache (looked up by label set, order-insensitive); ships empty: no benchmark label set is cached (17 Oct)",
|
| 3 |
+
"sets": [],
|
| 4 |
+
"removed": [
|
| 5 |
+
"OOD-7 CrossNER_AI",
|
| 6 |
+
"OOD-7 CrossNER_literature",
|
| 7 |
+
"OOD-7 CrossNER_music",
|
| 8 |
+
"OOD-7 CrossNER_politics",
|
| 9 |
+
"OOD-7 CrossNER_science",
|
| 10 |
+
"OOD-7 mit-movie",
|
| 11 |
+
"OOD-7 mit-restaurant"
|
| 12 |
+
]
|
| 13 |
+
}
|
modeling_vela3.py
ADDED
|
@@ -0,0 +1,471 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Vela on d3-turbo for Transformers (trust_remote_code=True).
|
| 2 |
+
|
| 3 |
+
m = AutoModel.from_pretrained("<this package>", trust_remote_code=True)
|
| 4 |
+
m.system_one(state=..., questions={...}, images=[...], videos=[...], assets={...}, long_context=False)
|
| 5 |
+
|
| 6 |
+
The base model is vllm-sr/d3-turbo at the revision pinned in vela3_config.json, loaded through d3-turbo's own runtime (prefix_bidir:
|
| 7 |
+
the state block is encoded once per request, then every question block reads it); its weights are not changed. Gated LoRA adapters
|
| 8 |
+
sit on its Linear layers (vela3_adapters.py):
|
| 9 |
+
* decision adapter: on only for the rows of the built-in router question ids (vela3_config.json "router_question_ids"; with
|
| 10 |
+
"router_gate": "id_and_wording" only when the question also uses the trained wording of router_questions.json). Every Choice /
|
| 11 |
+
Noul / Score question of a request first runs through d3-turbo's own state-once path with all adapters off, exactly as plain
|
| 12 |
+
d3-turbo answers that request (bit for bit). The router rows are then answered by one extra pass with the decision adapter on:
|
| 13 |
+
decisions.attention "prefix_bidir" = a second StateOnce pass (the state block once with the adapter on, the router question
|
| 14 |
+
blocks on it); "noncausal" = each router row as one full prompt under the noncausal full-attention mask (the d3 27B training mask).
|
| 15 |
+
* relation / span / NER adapters: span questions (type "span") as in Vela 3.0 (one extra forward pass per question, more for long
|
| 16 |
+
targets), under the single-block noncausal full-attention mask: d3-turbo's PromptEncoder without state blocks, which is
|
| 17 |
+
prefix_bidir with the whole prompt as one state block and is what the span adapters were trained on (vela3_span.py).
|
| 18 |
+
* Box / track questions, assets, videos and long_context requests are answered by d3-turbo itself (rt.system_one, adapters off);
|
| 19 |
+
span questions of such a request still run here. Requests with images take the base path for decisions (adapters off), as in
|
| 20 |
+
Vela 3.0; their span questions read the images in the same pass.
|
| 21 |
+
Open-label span questions get label definitions as vela3_config.json "definitions" says (writer "names": names only)."""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import importlib.util
|
| 25 |
+
import json
|
| 26 |
+
import os
|
| 27 |
+
import sys
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
from typing import Any
|
| 30 |
+
|
| 31 |
+
import numpy as np
|
| 32 |
+
import torch
|
| 33 |
+
from transformers import PreTrainedModel
|
| 34 |
+
from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5Config
|
| 35 |
+
|
| 36 |
+
HERE = Path(__file__).resolve().parent
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _load_local(name, path):
|
| 40 |
+
spec = importlib.util.spec_from_file_location(name, str(path)); mod = importlib.util.module_from_spec(spec)
|
| 41 |
+
sys.modules[name] = mod; spec.loader.exec_module(mod); return mod
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _resolve(name_or_path, revision=None, **hub):
|
| 45 |
+
p = Path(os.fspath(name_or_path)).expanduser()
|
| 46 |
+
if p.is_dir():
|
| 47 |
+
return p
|
| 48 |
+
from huggingface_hub import snapshot_download
|
| 49 |
+
return Path(snapshot_download(str(name_or_path), revision=revision, **{k: v for k, v in hub.items() if v is not None}))
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class Vela3MiniModel(PreTrainedModel): # class name kept: config.json auto_map points at it
|
| 53 |
+
config_class = Qwen3_5Config
|
| 54 |
+
base_model_prefix = "vela3"
|
| 55 |
+
_no_split_modules = []
|
| 56 |
+
|
| 57 |
+
def __init__(self, config):
|
| 58 |
+
super().__init__(config)
|
| 59 |
+
self.d3 = None; self.span = None; self.vela3_config = None; self.defs = None
|
| 60 |
+
self.post_init()
|
| 61 |
+
|
| 62 |
+
def _init_weights(self, module):
|
| 63 |
+
"""Nothing is initialised here: base weights come from d3-turbo, adapters and the span heads from this repository."""
|
| 64 |
+
|
| 65 |
+
@classmethod
|
| 66 |
+
def from_pretrained(cls, pretrained_model_name_or_path, *args, config=None, **kwargs):
|
| 67 |
+
"""kwargs: device / device_map, batch_size, verify (passed to d3); revision, token, cache_dir, local_files_only (this repo);
|
| 68 |
+
base_path: a local d3-turbo snapshot to use instead of downloading the pinned base revision."""
|
| 69 |
+
hub = {k: kwargs.pop(k) for k in ("token", "cache_dir", "local_files_only", "force_download") if k in kwargs}
|
| 70 |
+
revision = kwargs.pop("revision", None)
|
| 71 |
+
root = _resolve(pretrained_model_name_or_path, revision, **hub)
|
| 72 |
+
vc = json.loads((root / "vela3_config.json").read_text())
|
| 73 |
+
base_path = kwargs.pop("base_path", None)
|
| 74 |
+
d3_kw = {k: kwargs.pop(k) for k in ("device", "device_map", "batch_size", "verify") if k in kwargs}
|
| 75 |
+
for k in ("trust_remote_code", "_from_auto", "_from_pipeline", "adapter_kwargs", "code_revision", "_commit_hash", "dtype", "torch_dtype"):
|
| 76 |
+
kwargs.pop(k, None)
|
| 77 |
+
from transformers import AutoModel
|
| 78 |
+
base = base_path or vc["base"]["repo_id"]
|
| 79 |
+
d3 = AutoModel.from_pretrained(base, revision=None if base_path else vc["base"]["revision"], trust_remote_code=True, **d3_kw,
|
| 80 |
+
**{k: v for k, v in hub.items() if k != "force_download"})
|
| 81 |
+
rt = d3.runtime
|
| 82 |
+
d3_format = _load_local("vela3_d3_format", Path(rt.root) / "d3_format.py")
|
| 83 |
+
A = _load_local("vela3_adapters", root / "vela3_adapters.py")
|
| 84 |
+
vs = _load_local("vela3_span", root / "vela3_span.py")
|
| 85 |
+
ld = _load_local("vela3_labeldef", root / "vela3_labeldef.py")
|
| 86 |
+
dec = _load_local("vela3_span_decode", root / "span_decode_v2.py")
|
| 87 |
+
lm = rt.backbone.language_model
|
| 88 |
+
info = {name: A.attach(lm, name, root / spec["file"]) for name, spec in vc["adapters"].items()}
|
| 89 |
+
for name, spec in vc["adapters"].items():
|
| 90 |
+
if (info[name]["r"], info[name]["alpha"], info[name]["modules"]) != (spec["r"], spec["alpha"], spec["modules"]):
|
| 91 |
+
raise ValueError(f"adapter {name} does not match vela3_config.json: {info[name]}")
|
| 92 |
+
rt.backbone.eval()
|
| 93 |
+
from safetensors.torch import load_file
|
| 94 |
+
h = vs.SpanHeadV2(rt.backbone.config.text_config.hidden_size)
|
| 95 |
+
h.load_state_dict(load_file(str(root / vc["span"]["head_file"])), strict=True)
|
| 96 |
+
model = cls(config or Qwen3_5Config.from_pretrained(str(root)))
|
| 97 |
+
model.d3, model.vela3_config, model._decode, model._A = d3, vc, dec, A
|
| 98 |
+
model.span = vs.SpanEngine(rt, h.to(rt.device).eval(), A, d3_format)
|
| 99 |
+
model.span_ner = None
|
| 100 |
+
if vc["span"].get("ner"):
|
| 101 |
+
hn = vs.SpanHeadV2(rt.backbone.config.text_config.hidden_size)
|
| 102 |
+
hn.load_state_dict(load_file(str(root / vc["span"]["ner"]["head_file"])), strict=True)
|
| 103 |
+
model.span_ner = vs.SpanEngine(rt, hn.to(rt.device).eval(), A, d3_format, adapter="ner")
|
| 104 |
+
model.span_img = None # R5: separate image-halu adapter (span.image_halu.head_file + adapters.image_halu)
|
| 105 |
+
_ih = vc["span"].get("image_halu") or {}
|
| 106 |
+
if _ih.get("head_file") and "image_halu" in vc["adapters"]:
|
| 107 |
+
hi = vs.SpanHeadV2(rt.backbone.config.text_config.hidden_size)
|
| 108 |
+
hi.load_state_dict(load_file(str(root / _ih["head_file"])), strict=True)
|
| 109 |
+
model.span_img = vs.SpanEngine(rt, hi.to(rt.device).eval(), A, d3_format, adapter="image_halu")
|
| 110 |
+
model.span_rel = None # P5: relation adapter (span.relation.head_file + adapters.relation), relation questions only
|
| 111 |
+
_rl = vc["span"].get("relation") or {}
|
| 112 |
+
if _rl.get("head_file") and "relation" in vc["adapters"]:
|
| 113 |
+
hr = vs.SpanHeadV2(rt.backbone.config.text_config.hidden_size)
|
| 114 |
+
hr.load_state_dict(load_file(str(root / _rl["head_file"])), strict=True)
|
| 115 |
+
model.span_rel = vs.SpanEngine(rt, hr.to(rt.device).eval(), A, d3_format, adapter="relation")
|
| 116 |
+
model.defs = ld.LabelDefiner(rt, A, root / vc["definitions"]["cache_file"])
|
| 117 |
+
model.router_qids = frozenset(vc["router_question_ids"])
|
| 118 |
+
rq = json.loads((root / vc["router_questions_file"]).read_text())["trained_questions"]
|
| 119 |
+
model.router_wordings = {}
|
| 120 |
+
for qid, v in rq.items(): # trained instruction texts per id (intent: MASSIVE and SGD)
|
| 121 |
+
vs_ = [v] if "question" in v else list(v.values())
|
| 122 |
+
model.router_wordings[qid] = {" ".join(x["question"]["instructions"].split()) for x in vs_}
|
| 123 |
+
model.router_gate = vc.get("router_gate", "id")
|
| 124 |
+
model.name_or_path = str(pretrained_model_name_or_path)
|
| 125 |
+
model.eval()
|
| 126 |
+
return model
|
| 127 |
+
|
| 128 |
+
# ------------------------------------------------------------------ API
|
| 129 |
+
@property
|
| 130 |
+
def runtime(self):
|
| 131 |
+
return self.d3.runtime
|
| 132 |
+
|
| 133 |
+
# ---------------------------------------------------------------- decisions
|
| 134 |
+
def decide(self, state, questions, images=None, long_context=False):
|
| 135 |
+
"""d3-turbo's System One with the decision adapter on the router question rows (see the module docstring).
|
| 136 |
+
1. every runnable question through rt.run (state-once, adapters off): the answers plain d3-turbo gives for this request;
|
| 137 |
+
2. the gated rows (router ids; opt-in rows when an opt-in adapter ships) answered again with their adapter on, in one extra
|
| 138 |
+
pass per adapter (decisions.attention); their base answers from step 1 are replaced.
|
| 139 |
+
Requests with images (and every request when no row is gated) stop after step 1. usage.input_tokens = step 1's count plus
|
| 140 |
+
the extra pass's tokens (its state block once plus its question blocks, or the full router prompts)."""
|
| 141 |
+
rt = self.runtime
|
| 142 |
+
optin = {k for k, q in questions.items() if isinstance(q, dict) and q.get("decision_adapter") is True} if self.vela3_config.get("decision_opt_in") else set()
|
| 143 |
+
raw = questions # "decision_adapter" is ours, never passed to d3
|
| 144 |
+
questions = {k: ({a: b for a, b in q.items() if a != "decision_adapter"} if isinstance(q, dict) else q) for k, q in questions.items()}
|
| 145 |
+
saved = rt.max_length
|
| 146 |
+
rt.max_length = rt.question_cap(bool(long_context)) if hasattr(rt, "question_cap") else saved
|
| 147 |
+
try:
|
| 148 |
+
prepared = rt.prepare(state, questions, images)
|
| 149 |
+
probs, tokens = rt.run(prepared)
|
| 150 |
+
router = {k for k in prepared.runnable if self._router_row(k, raw.get(k))}
|
| 151 |
+
_ta = self.vela3_config.get("decision_task_adapters") or {} # RDF: domain / feedback rows -> their own adapter, own pass
|
| 152 |
+
task = {k: _ta[k] for k in router if k in _ta}
|
| 153 |
+
router = router - set(task)
|
| 154 |
+
oname = "decision_optin" if "decision_optin" in self.vela3_config["adapters"] else "decision"
|
| 155 |
+
opt = {k for k in prepared.runnable if k in optin and k not in router}
|
| 156 |
+
if oname == "decision":
|
| 157 |
+
router, opt = router | opt, set()
|
| 158 |
+
if prepared.images or getattr(prepared, "videos", None) or not (router or opt or task):
|
| 159 |
+
return rt.respond(prepared, probs, tokens)
|
| 160 |
+
probs = dict(probs)
|
| 161 |
+
for name, keys in (("decision", router), (oname, opt)) + tuple((n, {k for k, v in task.items() if v == n}) for n in sorted(set(task.values()))):
|
| 162 |
+
if keys:
|
| 163 |
+
got, extra = self._gated(prepared, [k for k in prepared.runnable if k in keys], name)
|
| 164 |
+
probs.update(got); tokens += extra
|
| 165 |
+
return rt.respond(prepared, probs, tokens)
|
| 166 |
+
finally:
|
| 167 |
+
rt.max_length = saved
|
| 168 |
+
|
| 169 |
+
def _subset(self, prepared, keys):
|
| 170 |
+
"""A Prepared holding only `keys` (same state, same tokens), for an extra pass over some rows of a request."""
|
| 171 |
+
import dataclasses
|
| 172 |
+
ks = set(keys)
|
| 173 |
+
cut = lambda d: {k: v for k, v in d.items() if k in ks} # noqa: E731
|
| 174 |
+
return dataclasses.replace(prepared, keys=[k for k in prepared.keys if k in ks], questions=cut(prepared.questions),
|
| 175 |
+
sequences=cut(prepared.sequences), errors={}, texts=cut(prepared.texts), lengths=cut(prepared.lengths),
|
| 176 |
+
reversed_sequences=cut(prepared.reversed_sequences), reversed_texts=cut(prepared.reversed_texts),
|
| 177 |
+
reversed_lengths=cut(prepared.reversed_lengths), states=cut(prepared.states),
|
| 178 |
+
reversed_states=cut(prepared.reversed_states))
|
| 179 |
+
|
| 180 |
+
def _gated(self, prepared, keys, name):
|
| 181 |
+
"""Probabilities of `keys` with adapter `name` on (every row of the pass), and the pass's input tokens."""
|
| 182 |
+
rt = self.runtime
|
| 183 |
+
mode = (self.vela3_config.get("decisions") or {}).get("attention", "prefix_bidir")
|
| 184 |
+
if mode == "prefix_bidir" and getattr(rt, "state_once", None) is not None:
|
| 185 |
+
with self._A.active(name):
|
| 186 |
+
return rt._run_state_once(self._subset(prepared, keys))
|
| 187 |
+
out = {} # noncausal (or a runtime without state-once): full prompts, request order, batch_size per pass, no state blocks
|
| 188 |
+
for start in range(0, len(keys), rt.batch_size):
|
| 189 |
+
chunk = keys[start:start + rt.batch_size]
|
| 190 |
+
seqs = [prepared.sequences[k] for k in chunk]; counts = [len(prepared.questions[k].keys) for k in chunk]
|
| 191 |
+
states = None if mode == "noncausal" else [prepared.states[k] for k in chunk]
|
| 192 |
+
with self._A.active(name):
|
| 193 |
+
lg = self._logits(seqs, counts, states)
|
| 194 |
+
rows = rt.torch.softmax(lg / rt.temperature, dim=-1).cpu().tolist()
|
| 195 |
+
out.update({k: r[:c] for k, r, c in zip(chunk, rows, counts)})
|
| 196 |
+
return out, sum(len(prepared.sequences[k]) for k in keys)
|
| 197 |
+
|
| 198 |
+
def _logits(self, seqs, counts, states):
|
| 199 |
+
"""rt.logits, with the noncausal mask when states is None on a prefix_bidir runtime (PromptEncoder without blocks)."""
|
| 200 |
+
rt = self.runtime
|
| 201 |
+
if states is not None or not getattr(rt, "blocked", False):
|
| 202 |
+
return rt.logits(seqs, counts, states)
|
| 203 |
+
torch = rt.torch
|
| 204 |
+
width = max(len(s) for s in seqs)
|
| 205 |
+
ids = torch.full((len(seqs), width), rt.pad_id, dtype=torch.long); mask = torch.zeros((len(seqs), width), dtype=torch.long)
|
| 206 |
+
for i, s in enumerate(seqs):
|
| 207 |
+
ids[i, width - len(s):] = torch.as_tensor(s, dtype=torch.long); mask[i, width - len(s):] = 1
|
| 208 |
+
ids, mask = ids.to(rt.device), mask.to(rt.device)
|
| 209 |
+
rt_mod = sys.modules.get(type(rt).__module__)
|
| 210 |
+
with torch.inference_mode(), rt_mod.sdpa_backends(rt.device):
|
| 211 |
+
hidden = rt.encoder(rt.backbone.get_input_embeddings()(ids), mask)[:, -1]
|
| 212 |
+
logits = hidden.float() @ rt.readout.T if rt.readout_dtype == "float32" else torch.nn.functional.linear(hidden, rt.readout).float()
|
| 213 |
+
invalid = torch.arange(logits.shape[-1], device=rt.device)[None] >= torch.as_tensor(list(counts), device=rt.device)[:, None]
|
| 214 |
+
return logits.masked_fill(invalid, float("-inf"))
|
| 215 |
+
|
| 216 |
+
def _router_row(self, qid, q):
|
| 217 |
+
"""Decision adapter on for this question? id gate, or id AND the trained wording (router_gate "id_and_wording")."""
|
| 218 |
+
if qid not in self.router_qids:
|
| 219 |
+
return False
|
| 220 |
+
if self.router_gate != "id_and_wording":
|
| 221 |
+
return True
|
| 222 |
+
ins = (q or {}).get("instructions") if isinstance(q, dict) else None
|
| 223 |
+
return ins is not None and " ".join(str(ins).split()) in self.router_wordings.get(qid, set())
|
| 224 |
+
|
| 225 |
+
# -------------------------------------------------------------------- spans
|
| 226 |
+
def _label_set(self, names):
|
| 227 |
+
sets = self.vela3_config["span"]["router_label_sets"]
|
| 228 |
+
hit = next((k for k, ls in sets.items() if names and set(names) <= set(ls)), None)
|
| 229 |
+
ev = self.vela3_config["span"].get("evidence")
|
| 230 |
+
if hit is None and ev and names and {n.strip().lower() for n in names} <= {x.lower() for x in ev["labels"]}:
|
| 231 |
+
return "evidence"
|
| 232 |
+
return hit
|
| 233 |
+
|
| 234 |
+
_REL = None
|
| 235 |
+
|
| 236 |
+
def _span_route(self, lset, names, instructions):
|
| 237 |
+
"""Which span adapter answers (vela3_config.json span.ner.route): "ner" (NER adapter) for open labels asked as an entity
|
| 238 |
+
question (no instructions, or instructions that name entities / types / categories and match no relation, answer or
|
| 239 |
+
evidence pattern); "span" (span adapter) for everything else: router label sets (PII, halu, toxic), evidence / answer
|
| 240 |
+
labels, relation questions ("Which spans are the <relation> of <head>?"), answer / evidence questions, other wordings.
|
| 241 |
+
Returns (engine, reason)."""
|
| 242 |
+
import re
|
| 243 |
+
if self.span_ner is None:
|
| 244 |
+
return "span", "span adapter"
|
| 245 |
+
if lset is not None:
|
| 246 |
+
return "span", f"{lset} label set: span adapter"
|
| 247 |
+
if {n.strip().lower() for n in names} & {"answer", "answers"}:
|
| 248 |
+
return "span", "answer label: span adapter"
|
| 249 |
+
if Vela3MiniModel._REL is None:
|
| 250 |
+
Vela3MiniModel._REL = (
|
| 251 |
+
re.compile(r"\bspans? (?:are|is) the .+ (?:of|for)\b|\brelat(?:ed|ion|ions|es|ing)?\b|\b(?:head|tail)\b|\bstands? in\b"
|
| 252 |
+
r"|\bwith respect to\b|\bcounterpart\b|\bsuch that\b|\brole of\b"),
|
| 253 |
+
re.compile(r"\b(?:answers?|evidence|support(?:s|ing|ed)?|cite[sd]?|citations?)\b"),
|
| 254 |
+
re.compile(r"\b(?:entit(?:y|ies)|types?|categor(?:y|ies)|named)\b"))
|
| 255 |
+
ins = " ".join(str(instructions or "").split()).lower()
|
| 256 |
+
if not ins:
|
| 257 |
+
return "ner", "open labels, default entity question: NER adapter"
|
| 258 |
+
if Vela3MiniModel._REL[0].search(ins):
|
| 259 |
+
if getattr(self, "span_rel", None) is not None:
|
| 260 |
+
return "rel", "relation question: relation adapter"
|
| 261 |
+
return "span", "relation question: span adapter"
|
| 262 |
+
if Vela3MiniModel._REL[1].search(ins):
|
| 263 |
+
return "span", "answer / evidence question: span adapter"
|
| 264 |
+
if Vela3MiniModel._REL[2].search(ins):
|
| 265 |
+
return "ner", "open labels, entity question: NER adapter"
|
| 266 |
+
return "span", "open labels, other question: span adapter"
|
| 267 |
+
|
| 268 |
+
def _threshold(self, lset, n_tok):
|
| 269 |
+
sp = self.vela3_config["span"]
|
| 270 |
+
if lset == "pii":
|
| 271 |
+
b = next(b for b in sp["pii_buckets"] if n_tok <= b["max_tokens"])
|
| 272 |
+
return float(b["threshold"]), f"pii label set, {b['name']} bucket ({n_tok} target tokens)"
|
| 273 |
+
if lset == "evidence":
|
| 274 |
+
return float(sp["evidence"]["threshold"]), "evidence label set (decoding: %s)" % sp["evidence"]["decode"]["name"]
|
| 275 |
+
if lset in sp["thresholds_by_set"]:
|
| 276 |
+
return float(sp["thresholds_by_set"][lset]), f"{lset} label set"
|
| 277 |
+
return float(sp["threshold"]), "open labels"
|
| 278 |
+
|
| 279 |
+
def span_answer(self, state, q, images=None):
|
| 280 |
+
if isinstance(state, str):
|
| 281 |
+
parts = [("user", state)]
|
| 282 |
+
elif isinstance(state, dict) and state and all(isinstance(v, str) for v in state.values()):
|
| 283 |
+
parts = list(state.items())
|
| 284 |
+
else:
|
| 285 |
+
raise ValueError("span questions need a text state or an object of text parts")
|
| 286 |
+
roles = [r for r, _ in parts]
|
| 287 |
+
target = q.get("target") or ("answer" if "answer" in roles else roles[-1])
|
| 288 |
+
if target not in roles:
|
| 289 |
+
raise ValueError(f"target {target!r} is not a part of the state")
|
| 290 |
+
crit = q.get("criteria") or q.get("labels")
|
| 291 |
+
if isinstance(crit, dict):
|
| 292 |
+
labels = [(str(k), "" if v is None else str(v)) for k, v in crit.items()]
|
| 293 |
+
elif isinstance(crit, (list, tuple)):
|
| 294 |
+
labels = [(str(k), "") for k in crit]
|
| 295 |
+
else:
|
| 296 |
+
raise ValueError("span questions need criteria: {label: description or null} or a list of labels")
|
| 297 |
+
if not 1 <= len(labels) <= 255:
|
| 298 |
+
raise ValueError("a span question needs 1 to 255 labels")
|
| 299 |
+
names = [n for n, _ in labels]
|
| 300 |
+
if len(set(names)) != len(names):
|
| 301 |
+
raise ValueError("span labels must be distinct")
|
| 302 |
+
sp = self.vela3_config["span"]; lset = self._label_set(names)
|
| 303 |
+
question = q.get("instructions")
|
| 304 |
+
route, route_why = self._span_route(lset, names, question)
|
| 305 |
+
img_halu = bool(images) and lset == "halu" and "image_halu" in sp
|
| 306 |
+
if lset in sp["schemas"]: # trained router schema: its question text and label descriptions fill what the request leaves empty
|
| 307 |
+
sch = sp["image_halu"] if img_halu else sp["schemas"][lset]
|
| 308 |
+
labels = [(n, d or sch["labels"].get(n, "")) for n, d in labels]
|
| 309 |
+
question = question or sch["question"]
|
| 310 |
+
if lset == "evidence":
|
| 311 |
+
question = question or sp["evidence"]["question"]
|
| 312 |
+
question = question or sp["default_question"]
|
| 313 |
+
# label definitions: "auto" (default for open labels) = the configured writer (vela3_labeldef.py) for labels without a description;
|
| 314 |
+
# a dict = given definitions; None / False = names only. Router label sets default to None.
|
| 315 |
+
mode = q.get("definitions", "auto" if lset is None else None)
|
| 316 |
+
used = {}
|
| 317 |
+
if mode == "auto":
|
| 318 |
+
empty = [n for n, d in labels if not d]
|
| 319 |
+
used = self.defs.define(names, only=set(empty)) if empty else {}
|
| 320 |
+
elif isinstance(mode, dict):
|
| 321 |
+
used = {str(k): str(v) for k, v in mode.items() if str(k) in names}
|
| 322 |
+
elif mode not in (None, False):
|
| 323 |
+
raise ValueError('definitions must be "auto", an object {label: definition} or null')
|
| 324 |
+
labels = [(n, used.get(n) or d) for n, d in labels]
|
| 325 |
+
if route == "ner" and not any(d for _, d in labels) and (sp.get("ner") or {}).get("names_only_route", "span") != "ner": # names only: span adapter unless span.ner.names_only_route is "ner" (X8 option B)
|
| 326 |
+
route, route_why = "span", "open labels, names only: span adapter"
|
| 327 |
+
eng = self.span_ner if route == "ner" else (self.span_rel if route == "rel" else self.span)
|
| 328 |
+
if img_halu and route != "ner" and getattr(self, "span_img", None) is not None: # R5: image-halu adapter
|
| 329 |
+
eng, route, route_why = self.span_img, "image_halu", route_why + "; image-halu adapter"
|
| 330 |
+
wo, lg, n_tok, n_win = eng.logits(parts, target, question, labels, images=images)
|
| 331 |
+
if route == "ner": # the NER adapter's dev picks: with definitions / descriptions, and names only
|
| 332 |
+
nr = sp["ner"]; described = any(d for _, d in labels)
|
| 333 |
+
thr0 = float(nr["threshold"] if described else nr["threshold_names_only"])
|
| 334 |
+
why = "open labels, " + ("with definitions" if described else "names only")
|
| 335 |
+
elif img_halu:
|
| 336 |
+
thr0, why = float(sp["image_halu"]["threshold"]), "halu label set with images (span.image_halu)"
|
| 337 |
+
else:
|
| 338 |
+
thr0, why = self._threshold(lset, n_tok)
|
| 339 |
+
# X8: relation / names-only thresholds (dev picks)
|
| 340 |
+
_rl = sp.get("relation") or {}; _pair = len(labels) == 1
|
| 341 |
+
_rt = _rl.get("threshold_dev_pair" if _pair else "threshold_dev_multi")
|
| 342 |
+
if route == "rel" and isinstance(_rt, (int, float)):
|
| 343 |
+
thr0, why = float(_rt), "relation adapter (%s; RE dev threshold)" % ("one relation" if _pair else "several relations")
|
| 344 |
+
elif route == "span" and lset is None and not any(d for _, d in labels) and sp.get("threshold_names_only") is not None:
|
| 345 |
+
thr0, why = float(sp["threshold_names_only"]), "open labels, names only"
|
| 346 |
+
thr = float(q.get("threshold", thr0))
|
| 347 |
+
text = dict(parts)[target]
|
| 348 |
+
if len(wo) == 0:
|
| 349 |
+
spans = []
|
| 350 |
+
else:
|
| 351 |
+
p = 1 / (1 + np.exp(-lg.astype(np.float64)))
|
| 352 |
+
if lset == "evidence":
|
| 353 |
+
spans = self._evidence_spans(p, wo, names, text, thr, sp["evidence"]["decode"])
|
| 354 |
+
else:
|
| 355 |
+
spans = self._decode.decode_v2(p, wo, names, text, thr, extend=False) if p.max() > thr else []
|
| 356 |
+
for s in spans:
|
| 357 |
+
k = names.index(s["label"]); m = (wo[:, 0] < s["end"]) & (wo[:, 1] > s["start"])
|
| 358 |
+
s["text"] = text[s["start"]:s["end"]]; s["score"] = float(p[m, k].mean()) if m.any() else 0.0
|
| 359 |
+
out = {"type": "span", "target": target, "spans": spans, "head": route, "head_reason": f"{route_why}; {why}", "threshold": thr}
|
| 360 |
+
if used:
|
| 361 |
+
out["definitions"] = used
|
| 362 |
+
return out
|
| 363 |
+
|
| 364 |
+
_SENT = None
|
| 365 |
+
|
| 366 |
+
@staticmethod
|
| 367 |
+
def _paras(text):
|
| 368 |
+
import re
|
| 369 |
+
out, pos = [], 0
|
| 370 |
+
for blk in re.split(r"(\n\s*\n|\n)", text):
|
| 371 |
+
if blk and not re.fullmatch(r"\s*", blk):
|
| 372 |
+
out.append((pos, pos + len(blk)))
|
| 373 |
+
pos += len(blk)
|
| 374 |
+
return out
|
| 375 |
+
|
| 376 |
+
def _para_spans(self, pk, wo, text, thr, name):
|
| 377 |
+
"""R5 evidence decodes: "para" keeps a paragraph when its mean word probability reaches thr; "sentfill" keeps sentences at thr and
|
| 378 |
+
fills a paragraph when at least half of its sentences are kept (r5_eval.decode2)."""
|
| 379 |
+
import re
|
| 380 |
+
if Vela3MiniModel._SENT is None:
|
| 381 |
+
Vela3MiniModel._SENT = re.compile(r"[^.!?\n]+(?:[.!?]+|\n|$)")
|
| 382 |
+
spans = []
|
| 383 |
+
if name == "para":
|
| 384 |
+
for a, b in self._paras(text):
|
| 385 |
+
k = (wo[:, 0] >= a) & (wo[:, 1] <= b)
|
| 386 |
+
if k.any() and pk[k].mean() >= thr:
|
| 387 |
+
spans.append([a, b])
|
| 388 |
+
return spans
|
| 389 |
+
sent = [(m.start(), m.end()) for m in Vela3MiniModel._SENT.finditer(text)]
|
| 390 |
+
keep = []
|
| 391 |
+
for a, b in sent:
|
| 392 |
+
k = (wo[:, 0] >= a) & (wo[:, 1] <= b)
|
| 393 |
+
keep.append(bool(k.any() and pk[k].mean() >= thr))
|
| 394 |
+
for a, b in self._paras(text):
|
| 395 |
+
inside = [i for i, (x, y) in enumerate(sent) if x >= a and y <= b]
|
| 396 |
+
if inside and sum(keep[i] for i in inside) * 2 >= len(inside):
|
| 397 |
+
spans.append([a, b])
|
| 398 |
+
else:
|
| 399 |
+
spans += [list(sent[i]) for i in inside if keep[i]]
|
| 400 |
+
return spans
|
| 401 |
+
|
| 402 |
+
def _evidence_spans(self, p, wo, names, text, thr, dc):
|
| 403 |
+
"""Evidence decoding picked on dev (ACL silver validation + RAGBench verbatim dev): "sent" keeps a sentence when the mean
|
| 404 |
+
word probability of its first label reaches the threshold; "plain" is the word decoder; then spans <= gap characters apart
|
| 405 |
+
are merged and spans shorter than min_chars dropped. One label (the first) carries the evidence spans."""
|
| 406 |
+
import re
|
| 407 |
+
pk = p[:, 0]
|
| 408 |
+
if dc["name"] in ("para", "sentfill"): # R5: paragraph-scale decodes (picked on the gold-like ACL dev)
|
| 409 |
+
spans = self._para_spans(pk, wo, text, thr, dc["name"])
|
| 410 |
+
elif dc["name"] == "sent":
|
| 411 |
+
if Vela3MiniModel._SENT is None:
|
| 412 |
+
Vela3MiniModel._SENT = re.compile(r"[^.!?\n]+(?:[.!?]+|\n|$)")
|
| 413 |
+
spans = []
|
| 414 |
+
for m in Vela3MiniModel._SENT.finditer(text):
|
| 415 |
+
a, b = m.start(), m.end(); k = (wo[:, 0] >= a) & (wo[:, 1] <= b)
|
| 416 |
+
if k.any() and pk[k].mean() >= thr:
|
| 417 |
+
spans.append([a, b])
|
| 418 |
+
else:
|
| 419 |
+
spans = [[s["start"], s["end"]] for s in self._decode.decode_v2(pk[:, None], wo, names[:1], text, thr, extend=False)] if pk.max() > thr else []
|
| 420 |
+
spans.sort(); out = []
|
| 421 |
+
for s in spans:
|
| 422 |
+
if out and s[0] - out[-1][1] <= dc.get("gap", 0):
|
| 423 |
+
out[-1][1] = max(out[-1][1], s[1])
|
| 424 |
+
else:
|
| 425 |
+
out.append(list(s))
|
| 426 |
+
return [dict(start=a, end=b, label=names[0]) for a, b in out if b - a >= dc.get("min_chars", 0)]
|
| 427 |
+
|
| 428 |
+
_BASE_KINDS = ("box", "track")
|
| 429 |
+
|
| 430 |
+
def system_one(self, *, state: Any, questions: dict[str, Any], images: list[Any] | None = None, videos: list[Any] | None = None,
|
| 431 |
+
assets: dict[str, Any] | None = None, long_context: bool = False) -> dict[str, Any]:
|
| 432 |
+
"""Choice / Noul / Score questions as d3-turbo (decision adapter on the router question ids), questions of type "span", and
|
| 433 |
+
everything else d3-turbo answers (box / track questions, assets, videos, long_context) passed to d3-turbo unchanged."""
|
| 434 |
+
span_q = {k: q for k, q in questions.items() if isinstance(q, dict) and q.get("type") == "span"}
|
| 435 |
+
rest = {k: q for k, q in questions.items() if k not in span_q}
|
| 436 |
+
imgs, img_err = None, None
|
| 437 |
+
if span_q and images: # decoded once, shared by every span question of the request
|
| 438 |
+
try:
|
| 439 |
+
imgs = self.runtime.load_images(images)
|
| 440 |
+
except ValueError as e:
|
| 441 |
+
img_err = str(e)
|
| 442 |
+
base_only = bool(videos) or bool(assets) or any(isinstance(q, dict) and q.get("type") in self._BASE_KINDS for q in rest.values())
|
| 443 |
+
if rest and base_only: # d3-turbo's own path (composed / video), adapters off
|
| 444 |
+
res = self.runtime.system_one(state=state, questions={k: ({a: b for a, b in q.items() if a != "decision_adapter"} if isinstance(q, dict) else q)
|
| 445 |
+
for k, q in rest.items()},
|
| 446 |
+
images=images, videos=videos, assets=assets, long_context=long_context)
|
| 447 |
+
elif rest:
|
| 448 |
+
res = self.decide(state, rest, images, long_context=long_context)
|
| 449 |
+
else:
|
| 450 |
+
res = {"model": self.d3.model_name, "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}
|
| 451 |
+
answers = dict(res["answers"])
|
| 452 |
+
for k, q in span_q.items():
|
| 453 |
+
try:
|
| 454 |
+
if img_err is not None:
|
| 455 |
+
raise ValueError(img_err)
|
| 456 |
+
if videos:
|
| 457 |
+
raise ValueError("span questions read text and images; this request has videos")
|
| 458 |
+
answers[k] = self.span_answer(state, q, imgs)
|
| 459 |
+
except ValueError as e:
|
| 460 |
+
answers[k] = {"type": "span", "error": "invalid_question", "reason": "question_malformed", "message": str(e)}
|
| 461 |
+
res["answers"] = {k: answers[k] for k in questions}
|
| 462 |
+
res["model"] = "vllm-sr/" + (self.vela3_config.get("size") or {}).get("name", "Vela-3.0-turbo")
|
| 463 |
+
return res
|
| 464 |
+
|
| 465 |
+
def forward(self, state=None, questions=None, images=None, videos=None, assets=None, long_context=False):
|
| 466 |
+
return self.system_one(state=state, questions=questions, images=images, videos=videos, assets=assets, long_context=long_context)
|
| 467 |
+
|
| 468 |
+
def train(self, mode: bool = True):
|
| 469 |
+
if mode:
|
| 470 |
+
raise RuntimeError("inference only")
|
| 471 |
+
return super().train(False)
|
ner_head.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8bc4c9848358782594e5f508bec047c1e49b9049429d13137672d68e1be6d87a
|
| 3 |
+
size 22387932
|
ner_lora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fd9d297b4cc07fb35e45636a3e926137f2e1d7b464abdb48ba088a50d3d6f65a
|
| 3 |
+
size 2611017504
|
relation_head.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3ec955d3a2144eb5b2359dc7ed3a3f53373ae7eb55cfcad974d81d8c6b02531
|
| 3 |
+
size 22387932
|
relation_lora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c61459f28b85f6c5c4fa25fcb850b501400312cefb4e0bba1125ab7c5635a64d
|
| 3 |
+
size 870359456
|
router_questions.json
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "The decision adapter switches on by question id only. These are the exact questions (instructions + criteria) it was trained on; other wording under the same id also gets the adapter.",
|
| 3 |
+
"question_ids": [
|
| 4 |
+
"attack",
|
| 5 |
+
"domain",
|
| 6 |
+
"factcheck",
|
| 7 |
+
"feedback",
|
| 8 |
+
"intent",
|
| 9 |
+
"modality",
|
| 10 |
+
"p_harm",
|
| 11 |
+
"r_harm",
|
| 12 |
+
"r_refusal",
|
| 13 |
+
"scenario"
|
| 14 |
+
],
|
| 15 |
+
"trained_questions": {
|
| 16 |
+
"domain": {
|
| 17 |
+
"question": {
|
| 18 |
+
"type": "choice",
|
| 19 |
+
"instructions": "Which subject area is this request about?",
|
| 20 |
+
"criteria": {
|
| 21 |
+
"biology": "living organisms, anatomy, genetics, medical genetics or viruses",
|
| 22 |
+
"business": "management, marketing, accounting, business ethics or public relations",
|
| 23 |
+
"chemistry": "chemical substances, reactions, elements or laboratory chemistry",
|
| 24 |
+
"computer science": "programming, algorithms, computer systems, security or machine learning",
|
| 25 |
+
"economics": "markets, macroeconomics, microeconomics or econometrics",
|
| 26 |
+
"engineering": "electrical or other engineering design and systems",
|
| 27 |
+
"health": "medicine, clinical practice, nutrition, ageing or sexual health",
|
| 28 |
+
"history": "past events, periods and historical societies",
|
| 29 |
+
"law": "legal rules, jurisprudence, courts or international law",
|
| 30 |
+
"math": "arithmetic, algebra, geometry, statistics or other mathematics",
|
| 31 |
+
"other": "a subject that fits none of the listed areas",
|
| 32 |
+
"philosophy": "philosophy, ethics, moral questions or formal logic",
|
| 33 |
+
"physics": "physical laws, mechanics, astronomy or physical phenomena",
|
| 34 |
+
"psychology": "mind, behaviour, mental processes or psychological practice",
|
| 35 |
+
"religion": "religions, religious texts, beliefs, practices and theology"
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
"state": "text"
|
| 39 |
+
},
|
| 40 |
+
"factcheck": {
|
| 41 |
+
"question": {
|
| 42 |
+
"type": "choice",
|
| 43 |
+
"instructions": "Does answering this request require checking facts?",
|
| 44 |
+
"criteria": {
|
| 45 |
+
"NO_FACT_CHECK_NEEDED": "the request can be handled without verifying facts about the world, e.g. translating, rewriting, formatting or writing fiction",
|
| 46 |
+
"FACT_CHECK_NEEDED": "answering the request relies on factual claims about the world that should be verified"
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
"state": "text"
|
| 50 |
+
},
|
| 51 |
+
"modality": {
|
| 52 |
+
"question": {
|
| 53 |
+
"type": "choice",
|
| 54 |
+
"instructions": "What kind of output does this request ask for?",
|
| 55 |
+
"criteria": {
|
| 56 |
+
"AR": "a text answer only, including code, analysis or describing an existing image",
|
| 57 |
+
"DIFFUSION": "a newly generated image only",
|
| 58 |
+
"BOTH": "a newly generated image together with a separate written explanation"
|
| 59 |
+
}
|
| 60 |
+
},
|
| 61 |
+
"state": "text"
|
| 62 |
+
},
|
| 63 |
+
"feedback": {
|
| 64 |
+
"question": {
|
| 65 |
+
"type": "choice",
|
| 66 |
+
"instructions": "What feedback does this user turn give about the previous answer?",
|
| 67 |
+
"criteria": {
|
| 68 |
+
"SAT": "the user is satisfied with the previous answer",
|
| 69 |
+
"NEED_CLARIFICATION": "the user asks for clarification or a more detailed explanation",
|
| 70 |
+
"WRONG_ANSWER": "the user says the previous answer was wrong or did not work",
|
| 71 |
+
"WANT_DIFFERENT": "the user wants a different answer, format, style or approach",
|
| 72 |
+
"NO_FEEDBACK": "the user turn gives no feedback on the previous answer: it continues the conversation, adds details or starts a new request"
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"state": "text"
|
| 76 |
+
},
|
| 77 |
+
"p_harm": {
|
| 78 |
+
"question": {
|
| 79 |
+
"type": "choice",
|
| 80 |
+
"instructions": "Is this request harmful?",
|
| 81 |
+
"criteria": {
|
| 82 |
+
"safe": "a benign request that does not violate any safety policy",
|
| 83 |
+
"unsafe": "a request that violates a safety policy or seeks harmful assistance"
|
| 84 |
+
}
|
| 85 |
+
},
|
| 86 |
+
"state": "text"
|
| 87 |
+
},
|
| 88 |
+
"r_harm": {
|
| 89 |
+
"question": {
|
| 90 |
+
"type": "choice",
|
| 91 |
+
"instructions": "Is this answer harmful?",
|
| 92 |
+
"criteria": {
|
| 93 |
+
"safe response": "an assistant answer that is itself harmless",
|
| 94 |
+
"unsafe response": "an assistant answer that is itself harmful, or that supplies the harmful assistance asked for"
|
| 95 |
+
}
|
| 96 |
+
},
|
| 97 |
+
"state": "parts: user, answer"
|
| 98 |
+
},
|
| 99 |
+
"r_refusal": {
|
| 100 |
+
"question": {
|
| 101 |
+
"type": "choice",
|
| 102 |
+
"instructions": "Did the assistant refuse?",
|
| 103 |
+
"criteria": {
|
| 104 |
+
"compliance": "an assistant answer that complies with the request and attempts the task",
|
| 105 |
+
"refusal": "an assistant answer that declines the request or refuses to help"
|
| 106 |
+
}
|
| 107 |
+
},
|
| 108 |
+
"state": "parts: user, answer"
|
| 109 |
+
},
|
| 110 |
+
"attack": {
|
| 111 |
+
"question": {
|
| 112 |
+
"type": "choice",
|
| 113 |
+
"instructions": "Is this a prompt injection or jailbreak attempt?",
|
| 114 |
+
"criteria": {
|
| 115 |
+
"benign": "a normal request or quoted content that does not try to override system instructions or bypass safety rules",
|
| 116 |
+
"jailbreak": "a prompt injection or jailbreak that attempts to override system instructions, hijack the task, or bypass safety rules"
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
"state": "text"
|
| 120 |
+
},
|
| 121 |
+
"scenario": {
|
| 122 |
+
"question": {
|
| 123 |
+
"type": "choice",
|
| 124 |
+
"instructions": "Which scenario does this request belong to?",
|
| 125 |
+
"criteria": {
|
| 126 |
+
"alarm": "setting, checking or removing alarms",
|
| 127 |
+
"audio": "device volume and audio settings",
|
| 128 |
+
"calendar": "calendar events, meetings and reminders",
|
| 129 |
+
"datetime": "the current date or time, and time-zone conversion",
|
| 130 |
+
"email": "reading, sending and managing e-mail and contacts",
|
| 131 |
+
"general": "greetings, jokes and small talk",
|
| 132 |
+
"lists": "to-do and shopping lists",
|
| 133 |
+
"music": "music preferences, information and player settings",
|
| 134 |
+
"news": "news headlines and updates",
|
| 135 |
+
"play": "playing music, radio, podcasts, audiobooks or games",
|
| 136 |
+
"qa": "general knowledge questions, definitions, maths, currency and stocks",
|
| 137 |
+
"recommendation": "recommendations for events, places or movies",
|
| 138 |
+
"social": "social media posts and updates",
|
| 139 |
+
"takeaway": "ordering takeaway food or checking an order",
|
| 140 |
+
"weather": "weather conditions and forecasts"
|
| 141 |
+
}
|
| 142 |
+
},
|
| 143 |
+
"state": "text",
|
| 144 |
+
"source": "MASSIVE (en)"
|
| 145 |
+
},
|
| 146 |
+
"intent": {
|
| 147 |
+
"MASSIVE": {
|
| 148 |
+
"question": {
|
| 149 |
+
"type": "choice",
|
| 150 |
+
"instructions": "What does the user want the assistant to do?",
|
| 151 |
+
"criteria": {
|
| 152 |
+
"alarm_query": "check which alarms are set",
|
| 153 |
+
"alarm_remove": "remove or cancel an alarm",
|
| 154 |
+
"alarm_set": "set an alarm for a time",
|
| 155 |
+
"audio_volume_down": "turn the volume down",
|
| 156 |
+
"audio_volume_mute": "mute the audio or make the device quiet",
|
| 157 |
+
"audio_volume_other": "change another audio setting",
|
| 158 |
+
"audio_volume_up": "turn the volume up",
|
| 159 |
+
"calendar_query": "ask about calendar events or reminders",
|
| 160 |
+
"calendar_remove": "delete a calendar event or reminder",
|
| 161 |
+
"calendar_set": "add a calendar event or reminder",
|
| 162 |
+
"datetime_convert": "convert a time between time zones",
|
| 163 |
+
"datetime_query": "ask for the current date or time",
|
| 164 |
+
"email_addcontact": "add an e-mail contact",
|
| 165 |
+
"email_query": "check or read e-mails",
|
| 166 |
+
"email_querycontact": "ask for a contact's details",
|
| 167 |
+
"email_sendemail": "send or reply to an e-mail",
|
| 168 |
+
"general_greet": "greet the assistant",
|
| 169 |
+
"general_joke": "ask for a joke",
|
| 170 |
+
"general_quirky": "chit-chat or an unusual request outside the other intents",
|
| 171 |
+
"lists_createoradd": "create a list or add an item to a list",
|
| 172 |
+
"lists_query": "ask what is on a list",
|
| 173 |
+
"lists_remove": "remove a list or an item from a list",
|
| 174 |
+
"music_dislikeness": "say the user dislikes a song or artist",
|
| 175 |
+
"music_likeness": "say the user likes a song or artist",
|
| 176 |
+
"music_query": "ask about the music that is playing",
|
| 177 |
+
"music_settings": "change music player settings such as shuffle or repeat",
|
| 178 |
+
"news_query": "ask for news",
|
| 179 |
+
"play_audiobook": "play or resume an audiobook",
|
| 180 |
+
"play_game": "start a game",
|
| 181 |
+
"play_music": "play a song, artist or playlist",
|
| 182 |
+
"play_podcasts": "play a podcast",
|
| 183 |
+
"play_radio": "play a radio station",
|
| 184 |
+
"qa_currency": "ask about currency exchange rates",
|
| 185 |
+
"qa_definition": "ask for the definition of a word",
|
| 186 |
+
"qa_factoid": "ask a factual general-knowledge question",
|
| 187 |
+
"qa_maths": "ask a maths question",
|
| 188 |
+
"qa_stock": "ask about stock prices",
|
| 189 |
+
"recommendation_events": "ask for events to go to",
|
| 190 |
+
"recommendation_locations": "ask for places such as shops or restaurants nearby",
|
| 191 |
+
"recommendation_movies": "ask for movie recommendations",
|
| 192 |
+
"social_post": "post on social media or send a complaint to a company",
|
| 193 |
+
"social_query": "check social media updates",
|
| 194 |
+
"takeaway_order": "order takeaway food",
|
| 195 |
+
"takeaway_query": "ask about a takeaway order or restaurant delivery",
|
| 196 |
+
"weather_query": "ask about the weather"
|
| 197 |
+
}
|
| 198 |
+
},
|
| 199 |
+
"state": "text",
|
| 200 |
+
"source": "MASSIVE (en)"
|
| 201 |
+
},
|
| 202 |
+
"SGD": {
|
| 203 |
+
"question": {
|
| 204 |
+
"type": "choice",
|
| 205 |
+
"instructions": "Using the dialogue history and service schema in state, choose the active intent for this service. Choose NONE when no service intent is active. Do not use future turns or hidden labels.",
|
| 206 |
+
"criteria": "{<intent>: <intent> for each intent of the service schema} + {NONE: NONE}"
|
| 207 |
+
},
|
| 208 |
+
"state": "object: history [{speaker, utterance}], service, schema (the SGD service schema JSON), task: 'SGD current-service intent'"
|
| 209 |
+
}
|
| 210 |
+
}
|
| 211 |
+
},
|
| 212 |
+
"contract_maps": {
|
| 213 |
+
"domain": {
|
| 214 |
+
"router-signal-suite": {
|
| 215 |
+
"religion": "philosophy"
|
| 216 |
+
}
|
| 217 |
+
},
|
| 218 |
+
"note": "the 14-option domain / 4-option feedback questions of the previous release are still trained and answered"
|
| 219 |
+
}
|
| 220 |
+
}
|
span_decode_v2.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Word-aware span decoding for the schema encoder's sigmoid span head (PII long-doc fix, 26 Sep).
|
| 2 |
+
|
| 3 |
+
The v1 decoder (span_data.decode) thresholds every token x label independently and turns each above-threshold
|
| 4 |
+
run into a span. With sub-word tokens (the Vela tokenizer splits digits one by one) a single below-threshold
|
| 5 |
+
piece fragments one entity into several false positives ('ois', '0', 'example'), and every label is decoded
|
| 6 |
+
separately, so overlapping labels each cost a false positive.
|
| 7 |
+
|
| 8 |
+
decode_v2(probs, offs, names, text, thr, ...) -> [dict(start, end, label)] (code-point offsets)
|
| 9 |
+
1. label resolution: each token takes its argmax label if that probability > thr, else O
|
| 10 |
+
(overlapping labels resolved by max score; upstream Vela PII argmax-BIO likewise gives one label per token).
|
| 11 |
+
2. word units from the text (GLiNER2 word splitter, see _UNIT): URL | run of letters/digits/@._-+ | any other
|
| 12 |
+
single non-space char; every CJK / kana / hangul / full-width character is its own unit (character level).
|
| 13 |
+
A token belongs to the unit that holds its first non-space character.
|
| 14 |
+
3. within a unit, one label: the label with the largest summed probability over the unit's labelled tokens
|
| 15 |
+
wins and the other labelled tokens of the unit are relabelled to it.
|
| 16 |
+
4. fill: unlabelled tokens BETWEEN two tokens of the same label inside one unit take that label;
|
| 17 |
+
extension (extend="unit", default): to the outermost letter/digit tokens of the unit, through inner
|
| 18 |
+
punctuation ('ois' -> 'lois123@x.example.org'), never onto a bare edge '[' / ']' / '.';
|
| 19 |
+
extend="alnum": only through directly adjacent letter/digit tokens; extend=False: off.
|
| 20 |
+
5. contiguous tokens of one label form a span (never joins different labels: upstream decode_entities rule);
|
| 21 |
+
whitespace is trimmed (upstream), and so are enclosing bracket / quote characters at the span edges.
|
| 22 |
+
6. optional: bridge=True joins two same-label spans separated by one unit whose probability for that
|
| 23 |
+
label is > thr * bridge_frac; min_chars={label: n} drops shorter spans.
|
| 24 |
+
Upstream semantics kept: exact code-point spans, whitespace trim, different types never joined. Not
|
| 25 |
+
reproducible from a sigmoid head: the B- tag that splits two adjacent same-type entities (it has no B/I).
|
| 26 |
+
"""
|
| 27 |
+
import re
|
| 28 |
+
import numpy as np
|
| 29 |
+
|
| 30 |
+
_CJK = ("⺀- -〿-ヿ-ㇿ㐀-䶿一-鿿ꀀ-"
|
| 31 |
+
"가-豈-︰-﹏-")
|
| 32 |
+
# Word units = GLiNER2's splitters (gliner2/processing/word_splitter.py, Apache-2.0): WhitespaceTokenSplitter's URL
|
| 33 |
+
# rule, CharLevelSplitter's [A-Za-z0-9@._\-+]+ runs (e-mails, IPs, phones, IDs stay one word), widened to Unicode
|
| 34 |
+
# letters/digits (Straße, título), and every CJK / kana / hangul / full-width character on its own (char level).
|
| 35 |
+
_UNIT = re.compile(rf"(?:https?://|www\.)[^\s\]\)\}}>「」『』\"'“”{_CJK}]+"
|
| 36 |
+
rf"|(?:(?![{_CJK}])[^\W_]|[@._\-+])+"
|
| 37 |
+
rf"|\S")
|
| 38 |
+
_ALNUM = re.compile(r"[^\W_]")
|
| 39 |
+
EDGE = set("[](){}<>「」『』【】《》〈〉\"'“”‘’«»‹›")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def units(text):
|
| 43 |
+
"""char index -> unit id (-1 for whitespace)."""
|
| 44 |
+
u = np.full(len(text) + 1, -1, dtype=np.int64)
|
| 45 |
+
for k, m in enumerate(_UNIT.finditer(text)):
|
| 46 |
+
u[m.start():m.end()] = k
|
| 47 |
+
return u
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _first_nonspace(text, a, b):
|
| 51 |
+
while a < b and text[a].isspace():
|
| 52 |
+
a += 1
|
| 53 |
+
return a
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def trim(text, s, e):
|
| 57 |
+
while s < e and (text[s].isspace() or text[s] in EDGE):
|
| 58 |
+
s += 1
|
| 59 |
+
while e > s and (text[e - 1].isspace() or text[e - 1] in EDGE):
|
| 60 |
+
e -= 1
|
| 61 |
+
return s, e
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def decode_v2(probs, offs, names, text, thr, fill=True, extend="unit", unit_vote=True, trim_edges=True,
|
| 65 |
+
bridge=False, bridge_frac=0.5, min_chars=None, u=None):
|
| 66 |
+
probs = np.asarray(probs, dtype=np.float32); offs = np.asarray(offs)
|
| 67 |
+
T = len(offs)
|
| 68 |
+
if T == 0:
|
| 69 |
+
return []
|
| 70 |
+
valid = offs[:, 1] > offs[:, 0]
|
| 71 |
+
k = probs.argmax(1); pk = probs[np.arange(T), k]
|
| 72 |
+
lab = np.where(valid & (pk > thr), k, -1)
|
| 73 |
+
u = units(text) if u is None else u
|
| 74 |
+
tu = np.full(T, -1, dtype=np.int64)
|
| 75 |
+
alnum = np.zeros(T, dtype=bool)
|
| 76 |
+
for t in range(T):
|
| 77 |
+
if not valid[t]:
|
| 78 |
+
continue
|
| 79 |
+
a, b = int(offs[t, 0]), int(offs[t, 1])
|
| 80 |
+
f = _first_nonspace(text, a, b)
|
| 81 |
+
tu[t] = u[f] if f < b and f < len(text) else -1
|
| 82 |
+
alnum[t] = bool(_ALNUM.search(text[a:b]))
|
| 83 |
+
# group tokens by unit (tokens are in text order, units contiguous)
|
| 84 |
+
if unit_vote or fill or extend:
|
| 85 |
+
t = 0
|
| 86 |
+
while t < T:
|
| 87 |
+
if tu[t] < 0:
|
| 88 |
+
t += 1; continue
|
| 89 |
+
t2 = t
|
| 90 |
+
while t2 + 1 < T and (tu[t2 + 1] == tu[t] or not valid[t2 + 1]):
|
| 91 |
+
t2 += 1
|
| 92 |
+
idx = [i for i in range(t, t2 + 1) if valid[i]]
|
| 93 |
+
L = [i for i in idx if lab[i] >= 0]
|
| 94 |
+
if L:
|
| 95 |
+
if unit_vote and len({int(lab[i]) for i in L}) > 1:
|
| 96 |
+
sc = {}
|
| 97 |
+
for i in L:
|
| 98 |
+
sc[int(lab[i])] = sc.get(int(lab[i]), 0.0) + float(pk[i])
|
| 99 |
+
w = max(sc, key=sc.get)
|
| 100 |
+
for i in L:
|
| 101 |
+
lab[i] = w
|
| 102 |
+
if fill:
|
| 103 |
+
for c in set(int(lab[i]) for i in L):
|
| 104 |
+
pos = [i for i in L if lab[i] == c]
|
| 105 |
+
for i in idx:
|
| 106 |
+
if pos[0] < i < pos[-1] and lab[i] < 0:
|
| 107 |
+
lab[i] = c
|
| 108 |
+
if extend:
|
| 109 |
+
for c in set(int(lab[i]) for i in idx if lab[i] >= 0):
|
| 110 |
+
pos = [idx.index(i) for i in idx if lab[i] == c]
|
| 111 |
+
if extend == "unit": # to the outermost letter/digit tokens of the unit, through inner punctuation
|
| 112 |
+
an = [j for j, i in enumerate(idx) if alnum[i]]
|
| 113 |
+
lo, hi = min([pos[0]] + an[:1]), max([pos[-1]] + an[-1:])
|
| 114 |
+
if all(lab[idx[j]] < 0 or lab[idx[j]] == c for j in range(lo, hi + 1)):
|
| 115 |
+
for j in range(lo, hi + 1):
|
| 116 |
+
lab[idx[j]] = c
|
| 117 |
+
else: # "alnum": only through adjacent letter/digit tokens
|
| 118 |
+
j = pos[0] - 1
|
| 119 |
+
while j >= 0 and lab[idx[j]] < 0 and alnum[idx[j]]:
|
| 120 |
+
lab[idx[j]] = c; j -= 1
|
| 121 |
+
j = pos[-1] + 1
|
| 122 |
+
while j < len(idx) and lab[idx[j]] < 0 and alnum[idx[j]]:
|
| 123 |
+
lab[idx[j]] = c; j += 1
|
| 124 |
+
t = t2 + 1
|
| 125 |
+
# contiguous runs of one label (skipping zero-width tokens)
|
| 126 |
+
spans = []
|
| 127 |
+
cur = None
|
| 128 |
+
for t in range(T):
|
| 129 |
+
if not valid[t]:
|
| 130 |
+
continue
|
| 131 |
+
c = int(lab[t])
|
| 132 |
+
if cur is not None and c == cur[0]:
|
| 133 |
+
cur[2] = int(offs[t, 1]); cur[3].append(t); continue
|
| 134 |
+
if cur is not None:
|
| 135 |
+
spans.append(cur)
|
| 136 |
+
cur = [c, int(offs[t, 0]), int(offs[t, 1]), [t]] if c >= 0 else None
|
| 137 |
+
if cur is not None:
|
| 138 |
+
spans.append(cur)
|
| 139 |
+
if bridge and len(spans) > 1:
|
| 140 |
+
out = [spans[0]]
|
| 141 |
+
for sp in spans[1:]:
|
| 142 |
+
pv = out[-1]
|
| 143 |
+
if sp[0] == pv[0]:
|
| 144 |
+
gap = [t for t in range(pv[3][-1] + 1, sp[3][0]) if valid[t]]
|
| 145 |
+
gu = {int(tu[t]) for t in gap if tu[t] >= 0}
|
| 146 |
+
if gap and len(gu) <= 1 and all(probs[t, sp[0]] > thr * bridge_frac for t in gap if tu[t] >= 0):
|
| 147 |
+
pv[2] = sp[2]; pv[3] = pv[3] + gap + sp[3]; continue
|
| 148 |
+
out.append(sp)
|
| 149 |
+
spans = out
|
| 150 |
+
res = []
|
| 151 |
+
for c, s, e, _ in spans:
|
| 152 |
+
s, e = trim(text, s, e) if trim_edges else trim_ws(text, s, e)
|
| 153 |
+
if e <= s:
|
| 154 |
+
continue
|
| 155 |
+
if min_chars and e - s < min_chars.get(names[c], 0):
|
| 156 |
+
continue
|
| 157 |
+
res.append(dict(start=s, end=e, label=names[c]))
|
| 158 |
+
return res
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def trim_ws(text, s, e):
|
| 162 |
+
while s < e and text[s].isspace():
|
| 163 |
+
s += 1
|
| 164 |
+
while e > s and text[e - 1].isspace():
|
| 165 |
+
e -= 1
|
| 166 |
+
return s, e
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def decode_v1_trimmed(probs, offs, names, text, thr):
|
| 170 |
+
"""The scorer's current rule (span_data.decode + whitespace trim), for before/after tables."""
|
| 171 |
+
from span_data import decode
|
| 172 |
+
out = []
|
| 173 |
+
for s in decode(probs, offs, names, thr):
|
| 174 |
+
a, b = trim_ws(text, s["start"], s["end"])
|
| 175 |
+
if b > a:
|
| 176 |
+
out.append(dict(start=a, end=b, label=s["label"]))
|
| 177 |
+
return out
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
# ---------------------------------------------------------------------------------------------------------------
|
| 181 |
+
# row-level API (rows = dicts with text, spans [{start,end,label}], names, offs, probs; see dump_span_probs_v2.py)
|
| 182 |
+
# ---------------------------------------------------------------------------------------------------------------
|
| 183 |
+
GRID = [round(x, 2) for x in np.arange(0.05, 0.96, 0.05)]
|
| 184 |
+
DECODERS = {
|
| 185 |
+
"v1": lambda r, thr: decode_v1_trimmed(r["probs"], r["offs"], r["names"], r["text"], thr),
|
| 186 |
+
"v2": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr),
|
| 187 |
+
"v2_noext": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr, extend=False),
|
| 188 |
+
"v2_alnum": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr, extend="alnum"),
|
| 189 |
+
"v2_notrim": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr, trim_edges=False),
|
| 190 |
+
"v2_bridge": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr, bridge=True),
|
| 191 |
+
"v2_argmax_only": lambda r, thr: decode_v2(r["probs"], r["offs"], r["names"], r["text"], thr, fill=False,
|
| 192 |
+
extend=False, unit_vote=False, trim_edges=False),
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def decode_rows(rows, thr, decoder="v2"):
|
| 197 |
+
f = DECODERS[decoder] if isinstance(decoder, str) else decoder
|
| 198 |
+
return [f(r, thr) for r in rows]
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _psd(span_data_path):
|
| 202 |
+
"""Load the upstream semantic-router span_data.py (src/training/model_classifier/pii_model_fine_tuning_lora/span_data.py)
|
| 203 |
+
from span_data_path. Eval helper only (dev threshold picks); inference never calls it."""
|
| 204 |
+
import importlib.util
|
| 205 |
+
from pathlib import Path
|
| 206 |
+
if span_data_path is None:
|
| 207 |
+
raise ValueError("pii_metrics needs span_data_path: the path of the upstream span_data.py")
|
| 208 |
+
spec = importlib.util.spec_from_file_location("pii_span_data_up", Path(span_data_path))
|
| 209 |
+
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
| 210 |
+
return m
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
PII_TYPES = ["AGE", "CREDIT_CARD", "DATE_TIME", "DOMAIN_NAME", "EMAIL_ADDRESS", "GPE", "IBAN_CODE", "IP_ADDRESS", "NRP",
|
| 214 |
+
"ORGANIZATION", "PERSON", "PHONE_NUMBER", "STREET_ADDRESS", "TITLE", "US_DRIVER_LICENSE", "US_SSN", "ZIP_CODE"]
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def pii_metrics(rows, preds, span_data_path):
|
| 218 |
+
"""Upstream Vela PII exact-entity metric (span_data.entity_metrics, whitespace-trimmed gold); span_data_path: see _psd."""
|
| 219 |
+
P = _psd(span_data_path)
|
| 220 |
+
recs = [dict(full_text=r["text"], spans=[dict(start_position=s["start"], end_position=s["end"], entity_type=s["label"],
|
| 221 |
+
entity_value=r["text"][s["start"]:s["end"]]) for s in r["spans"]]) for r in rows]
|
| 222 |
+
return P.entity_metrics(recs, [{(s["label"], s["start"], s["end"]) for s in p} for p in preds], PII_TYPES)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def toxic_score(rows, preds):
|
| 226 |
+
from span_data import char_f1
|
| 227 |
+
return float(np.mean([char_f1(p, r["spans"]) for r, p in zip(rows, preds)]))
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def halu_metrics(rows, preds):
|
| 231 |
+
from train_schema_span import ragtruth_metrics_spans
|
| 232 |
+
return ragtruth_metrics_spans([dict(text=r["text"], spans=r["spans"]) for r in rows], preds)
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def family_objective(family, span_data_path=None):
|
| 236 |
+
"""Dev objective per span label family (what the threshold is chosen to maximise); span_data_path: PII only (see _psd)."""
|
| 237 |
+
if family == "pii":
|
| 238 |
+
return lambda rows, preds: pii_metrics(rows, preds, span_data_path)["micro"]["f1"]
|
| 239 |
+
if family == "toxic":
|
| 240 |
+
return toxic_score
|
| 241 |
+
if family == "halu":
|
| 242 |
+
return lambda rows, preds: halu_metrics(rows, preds)["example_f1"]
|
| 243 |
+
raise KeyError(family)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def pick_threshold(dev_sets, family, decoder="v2", grid=GRID, span_data_path=None):
|
| 247 |
+
"""Choose one threshold per label family on DEV rows only.
|
| 248 |
+
dev_sets: list of row lists; the objective is the unweighted mean over the sets (for PII: dev short, dev 4096,
|
| 249 |
+
dev 8192 -> upstream's length-macro selection, so the dense long rows cannot dominate). Returns (thr, curve)."""
|
| 250 |
+
obj = family_objective(family, span_data_path)
|
| 251 |
+
curve = {t: float(np.mean([obj(rows, decode_rows(rows, t, decoder)) for rows in dev_sets])) for t in grid}
|
| 252 |
+
best = max(curve.values())
|
| 253 |
+
thr = min(t for t, v in curve.items() if v >= best - 1e-12) # ties -> lowest threshold
|
| 254 |
+
return thr, curve
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
if __name__ == "__main__": # self-test on the fragment cases seen in run C
|
| 258 |
+
text = "Le formulaire indique [pavel_200015@summit15.example.org] dans adresse courriel."
|
| 259 |
+
names = ["EMAIL_ADDRESS", "PERSON"]
|
| 260 |
+
import unicodedata # noqa
|
| 261 |
+
# fake token offsets: split at every non-alnum boundary
|
| 262 |
+
offs = [(m.start(), m.end()) for m in re.finditer(r"\s?\w+|\s?[^\w\s]", text)]
|
| 263 |
+
p = np.zeros((len(offs), 2), dtype=np.float32)
|
| 264 |
+
s0 = text.index("pavel"); e0 = text.index("]")
|
| 265 |
+
for i, (a, b) in enumerate(offs):
|
| 266 |
+
if a >= s0 - 1 and b <= e0 + 1:
|
| 267 |
+
p[i, 0] = 0.9 if i % 3 else 0.2 # fragmented
|
| 268 |
+
p[[i for i, (a, b) in enumerate(offs) if text[a:b].strip() == "["][0], 0] = 0.8 # bracket above threshold
|
| 269 |
+
out = decode_v2(p, np.array(offs), names, text, 0.5)
|
| 270 |
+
assert [(o["label"], text[o["start"]:o["end"]]) for o in out] == [("EMAIL_ADDRESS", "pavel_200015@summit15.example.org")], out
|
| 271 |
+
t2 = "表格在日期一栏填写了「2015-06-06」。"
|
| 272 |
+
print(units(t2)[:14], "ok")
|
span_head.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5b450a2b20d077f385b7c06042ff06c86797f05692f8b84755032920349f4dd1
|
| 3 |
+
size 22387892
|
span_lora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:94a02a79c37c1a7d53d3e9771b31515c94af19011e2a75c1c51ea37196b26383
|
| 3 |
+
size 870359416
|
vela3_adapters.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gated LoRA adapters of Vela 3.0.
|
| 2 |
+
|
| 3 |
+
The adapters (vela3_config.json "adapters") sit on the base's Linear layers; each adapter's layers, rank and alpha come from its
|
| 4 |
+
*_lora.safetensors. At most one adapter is on in any forward; with no adapter active, a wrapped layer returns base(x) itself, so every
|
| 5 |
+
other pass is the untouched base computation.
|
| 6 |
+
|
| 7 |
+
The adapter math is the training code's GatedLoRA in eval mode (dropout off):
|
| 8 |
+
y = base(x) + ((x.float() @ A.T) @ B.T * alpha / r).to(y.dtype)
|
| 9 |
+
A row mask applies the adapter to some rows of a batch only: torch.where keeps base(x) bit for bit on the other rows."""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import contextlib
|
| 13 |
+
import json
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
from torch import nn
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class _Gate:
|
| 20 |
+
name = None # active adapter name, or None (all adapters off)
|
| 21 |
+
rows = None # None = every row of the batch; else a bool tensor [B]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
GATE = _Gate()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@contextlib.contextmanager
|
| 28 |
+
def active(name, rows=None):
|
| 29 |
+
"""Activate one adapter for the forwards inside the block (rows: optional bool mask over the batch)."""
|
| 30 |
+
prev = (GATE.name, GATE.rows)
|
| 31 |
+
GATE.name, GATE.rows = name, rows
|
| 32 |
+
try:
|
| 33 |
+
yield
|
| 34 |
+
finally:
|
| 35 |
+
GATE.name, GATE.rows = prev
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class MultiLoRA(nn.Module):
|
| 39 |
+
def __init__(self, base: nn.Linear):
|
| 40 |
+
super().__init__()
|
| 41 |
+
self.base = base
|
| 42 |
+
self.ad = {} # name -> (A [r, in] fp32, B [out, r] fp32, scale)
|
| 43 |
+
|
| 44 |
+
def add(self, name, A, B, scale):
|
| 45 |
+
dev = self.base.weight.device
|
| 46 |
+
self.ad[name] = (A.to(dev, torch.float32), B.to(dev, torch.float32), float(scale))
|
| 47 |
+
|
| 48 |
+
def forward(self, x):
|
| 49 |
+
y = self.base(x)
|
| 50 |
+
name = GATE.name
|
| 51 |
+
if name is None or name not in self.ad:
|
| 52 |
+
return y
|
| 53 |
+
A, B, s = self.ad[name]
|
| 54 |
+
d = ((x.float() @ A.T) @ B.T * s).to(y.dtype)
|
| 55 |
+
rows = GATE.rows
|
| 56 |
+
if rows is None:
|
| 57 |
+
return y + d
|
| 58 |
+
if rows.shape[0] != y.shape[0]:
|
| 59 |
+
raise RuntimeError("row gate does not match the batch")
|
| 60 |
+
return torch.where(rows.view(-1, *([1] * (y.dim() - 1))), y + d, y)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def attach(lm, name, path):
|
| 64 |
+
"""Attach the adapter in `path` (safetensors: '<module path>.A' / '.B', metadata r / alpha / first / targets) to the text model."""
|
| 65 |
+
from safetensors import safe_open
|
| 66 |
+
with safe_open(str(path), framework="pt") as f:
|
| 67 |
+
meta = f.metadata() or {}
|
| 68 |
+
sd = {k: f.get_tensor(k) for k in f.keys()}
|
| 69 |
+
r, alpha = int(meta["r"]), int(meta["alpha"])
|
| 70 |
+
mods = sorted({k.rsplit(".", 1)[0] for k in sd})
|
| 71 |
+
for mp in mods:
|
| 72 |
+
parent_path, attr = mp.rsplit(".", 1)
|
| 73 |
+
parent = lm.get_submodule(parent_path)
|
| 74 |
+
cur = getattr(parent, attr)
|
| 75 |
+
if isinstance(cur, nn.Linear):
|
| 76 |
+
cur = MultiLoRA(cur)
|
| 77 |
+
setattr(parent, attr, cur)
|
| 78 |
+
elif not isinstance(cur, MultiLoRA):
|
| 79 |
+
raise ValueError(f"{mp} is not a Linear layer")
|
| 80 |
+
A, B = sd[mp + ".A"], sd[mp + ".B"]
|
| 81 |
+
if A.shape != (r, cur.base.in_features) or B.shape != (cur.base.out_features, r):
|
| 82 |
+
raise ValueError(f"adapter shape mismatch at {mp}")
|
| 83 |
+
cur.add(name, A, B, alpha / r)
|
| 84 |
+
if len(sd) != 2 * len(mods):
|
| 85 |
+
raise ValueError(f"adapter {name}: unpaired A / B tensors")
|
| 86 |
+
return dict(modules=len(mods), r=r, alpha=alpha, first=int(meta.get("first", -1)),
|
| 87 |
+
targets=json.loads(meta.get("targets", "[]")))
|
vela3_config.json
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "vela3/3",
|
| 3 |
+
"base": {
|
| 4 |
+
"repo_id": "vllm-sr/d3-turbo",
|
| 5 |
+
"revision": "da17b5d0c860f7e5be6e2afebe67228b4bd24d0c",
|
| 6 |
+
"snapshot": "vllm-sr/d3-turbo v1.0 (da17b5d0; the 7478169a pin of the build is not on the Hub, its 50 files are identical to da17b5d0)",
|
| 7 |
+
"runtime_note": "d3-turbo's own runtime (prefix_bidir, state-once); span passes through rt.encoder without state blocks (single-block noncausal mask, as the span adapters were trained)"
|
| 8 |
+
},
|
| 9 |
+
"adapters": {
|
| 10 |
+
"span": {
|
| 11 |
+
"file": "span_lora.safetensors",
|
| 12 |
+
"r": 64,
|
| 13 |
+
"alpha": 64,
|
| 14 |
+
"first_layer": 32,
|
| 15 |
+
"modules": 200,
|
| 16 |
+
"switched_on": "span passes of relation / evidence / answer questions and the router label sets (span.ner.route)",
|
| 17 |
+
"source": "Vela 3.0 d3 R4 span LoRA, ckpt-024000 (round 2, r4 --mix r6s (Phase B pools) --granfix + wordings, 24,000 steps; dev pick)"
|
| 18 |
+
},
|
| 19 |
+
"decision": {
|
| 20 |
+
"file": "decision_lora.safetensors",
|
| 21 |
+
"r": 32,
|
| 22 |
+
"alpha": 32,
|
| 23 |
+
"first_layer": 0,
|
| 24 |
+
"modules": 400,
|
| 25 |
+
"switched_on": "the rows of router_question_ids in a decision pass",
|
| 26 |
+
"source": "Vela 3.0 d3 decision LoRA, ckpt-002000 (--carve --sgd --dna, 3,000 steps; picked on the held-out sel carve)"
|
| 27 |
+
},
|
| 28 |
+
"ner": {
|
| 29 |
+
"file": "ner_lora.safetensors",
|
| 30 |
+
"r": 128,
|
| 31 |
+
"alpha": 128,
|
| 32 |
+
"first_layer": 16,
|
| 33 |
+
"modules": 300,
|
| 34 |
+
"switched_on": "span passes of open-label entity questions (span.ner.route)",
|
| 35 |
+
"source": "Vela 3.0 d3 N5 NER span LoRA, ckpt-009000 (round 2, n5 --mix n5b --ner-wordings-only, 24,000 steps; reviewed checkpoint (approved; the rule's pick was ckpt-009000))"
|
| 36 |
+
},
|
| 37 |
+
"relation": {
|
| 38 |
+
"file": "relation_lora.safetensors",
|
| 39 |
+
"r": 64,
|
| 40 |
+
"alpha": 64,
|
| 41 |
+
"first_layer": 32,
|
| 42 |
+
"modules": 200,
|
| 43 |
+
"switched_on": "span passes of relation questions only (modeling_vela3._span_route relation rule: 'Which spans are the <relation> of <head>?', 'Which spans are related to <head> by these relations?', ...)",
|
| 44 |
+
"source": "Vela 3.0 d3 R5S span LoRA, ckpt-008000 (8,000-step R5S recipe from this package's R4 ckpt-024000, r6s base mix + DocRED / CrossRE / FewRel train splits; relation questions only)"
|
| 45 |
+
},
|
| 46 |
+
"decision_df": {
|
| 47 |
+
"file": "decision_df_lora.safetensors",
|
| 48 |
+
"r": 32,
|
| 49 |
+
"alpha": 32,
|
| 50 |
+
"first_layer": 0,
|
| 51 |
+
"modules": 400,
|
| 52 |
+
"switched_on": "the rows of the domain and feedback router ids, in their own pass",
|
| 53 |
+
"source": "df_turbo/ckpt-002000 (init = this package's decision LoRA, Vela 2.0 router data + router-signal-suite train domain / feedback, decontaminated)"
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"router_question_ids": [
|
| 57 |
+
"attack",
|
| 58 |
+
"domain",
|
| 59 |
+
"factcheck",
|
| 60 |
+
"feedback",
|
| 61 |
+
"intent",
|
| 62 |
+
"modality",
|
| 63 |
+
"p_harm",
|
| 64 |
+
"r_harm",
|
| 65 |
+
"r_refusal",
|
| 66 |
+
"scenario"
|
| 67 |
+
],
|
| 68 |
+
"router_questions_file": "router_questions.json",
|
| 69 |
+
"span": {
|
| 70 |
+
"head_file": "span_head.safetensors",
|
| 71 |
+
"read": "final-norm states (last_hidden_state)",
|
| 72 |
+
"threshold": 0.6,
|
| 73 |
+
"thresholds_by_set": {
|
| 74 |
+
"halu": 0.45,
|
| 75 |
+
"toxic": 0.2
|
| 76 |
+
},
|
| 77 |
+
"pii_buckets": [
|
| 78 |
+
{
|
| 79 |
+
"name": "short",
|
| 80 |
+
"max_tokens": 1024,
|
| 81 |
+
"threshold": 0.15
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "4K",
|
| 85 |
+
"max_tokens": 6144,
|
| 86 |
+
"threshold": 0.15
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"name": "8K",
|
| 90 |
+
"max_tokens": 1000000000000,
|
| 91 |
+
"threshold": 0.15
|
| 92 |
+
}
|
| 93 |
+
],
|
| 94 |
+
"threshold_sources": {
|
| 95 |
+
"open": "R4 pick (ner_shift VALID: (ckpt, head) by eval F1 in defs mode; thresholds from the thr halves): ckpt-024000 / final, definitions 0.6, names-only 0.55",
|
| 96 |
+
"halu_pii_toxic": "NB-dev router-span reads of R4 at the pick (results/r4_spanread.json): halu, PII short / 4K / 8K, toxic dev carve",
|
| 97 |
+
"evidence": "evidence_pick.json: plain gap 0 min chars 10 at 0.5, mean thr-half word F1 over ACL silver validation + RAGBench verbatim dev",
|
| 98 |
+
"ner": "N5 ckpt-009000 head final (dev pick: mode defs, eval-half F1 0.7592); thresholds: that checkpoint's ner_shift thr-half picks (definitions 0.6, names only 0.65); mode: definitions, as the 4B",
|
| 99 |
+
"names_only": "open labels asked names only (span adapter): span.threshold_names_only 0.55, the span adapter's names-mode dev pick"
|
| 100 |
+
},
|
| 101 |
+
"router_label_sets": {
|
| 102 |
+
"pii": [
|
| 103 |
+
"AGE",
|
| 104 |
+
"CREDIT_CARD",
|
| 105 |
+
"DATE_TIME",
|
| 106 |
+
"DOMAIN_NAME",
|
| 107 |
+
"EMAIL_ADDRESS",
|
| 108 |
+
"GPE",
|
| 109 |
+
"IBAN_CODE",
|
| 110 |
+
"IP_ADDRESS",
|
| 111 |
+
"NRP",
|
| 112 |
+
"ORGANIZATION",
|
| 113 |
+
"PERSON",
|
| 114 |
+
"PHONE_NUMBER",
|
| 115 |
+
"STREET_ADDRESS",
|
| 116 |
+
"TITLE",
|
| 117 |
+
"US_DRIVER_LICENSE",
|
| 118 |
+
"US_SSN",
|
| 119 |
+
"ZIP_CODE"
|
| 120 |
+
],
|
| 121 |
+
"halu": [
|
| 122 |
+
"unsupported"
|
| 123 |
+
],
|
| 124 |
+
"toxic": [
|
| 125 |
+
"Criminal Planning/Confessions",
|
| 126 |
+
"Violence",
|
| 127 |
+
"Hate/Identity Hate",
|
| 128 |
+
"Harassment",
|
| 129 |
+
"Profanity",
|
| 130 |
+
"Sexual",
|
| 131 |
+
"Sexual (minor)",
|
| 132 |
+
"Controlled/Regulated Substances",
|
| 133 |
+
"PII/Privacy",
|
| 134 |
+
"Guns and Illegal Weapons",
|
| 135 |
+
"Suicide and Self Harm",
|
| 136 |
+
"Unauthorized Advice",
|
| 137 |
+
"Political/Misinformation/Conspiracy",
|
| 138 |
+
"Fraud/Deception",
|
| 139 |
+
"Illegal Activity",
|
| 140 |
+
"Threat",
|
| 141 |
+
"Copyright/Trademark/Plagiarism",
|
| 142 |
+
"High Risk Gov Decision Making",
|
| 143 |
+
"Immoral/Unethical",
|
| 144 |
+
"Needs Caution",
|
| 145 |
+
"Other",
|
| 146 |
+
"S1",
|
| 147 |
+
"S2",
|
| 148 |
+
"S3",
|
| 149 |
+
"S4",
|
| 150 |
+
"S5",
|
| 151 |
+
"S6",
|
| 152 |
+
"S7",
|
| 153 |
+
"S8",
|
| 154 |
+
"S9",
|
| 155 |
+
"S10",
|
| 156 |
+
"S11",
|
| 157 |
+
"S12",
|
| 158 |
+
"S14",
|
| 159 |
+
"others",
|
| 160 |
+
"social_stereotypes_and_unfair_discrimination",
|
| 161 |
+
"disseminating_false_or_misleading_information_encouraging_disinformation_campaigns",
|
| 162 |
+
"violence_and_physical_harm",
|
| 163 |
+
"private_information_individual",
|
| 164 |
+
"sensitive_information_organization_government",
|
| 165 |
+
"defamation_encouraging_unethical_or_unsafe_actions",
|
| 166 |
+
"toxic_language_hate_speech",
|
| 167 |
+
"fraud_assisting_illegal_activities",
|
| 168 |
+
"copyright_violations",
|
| 169 |
+
"causing_material_harm_by_disseminating_misinformation",
|
| 170 |
+
"sexual_content",
|
| 171 |
+
"mental_health_over-reliance_crisis"
|
| 172 |
+
]
|
| 173 |
+
},
|
| 174 |
+
"schemas": {
|
| 175 |
+
"pii": {
|
| 176 |
+
"question": "Which spans are personal information?",
|
| 177 |
+
"labels": {
|
| 178 |
+
"AGE": "a person's age",
|
| 179 |
+
"CREDIT_CARD": "a payment card number",
|
| 180 |
+
"DATE_TIME": "a date, time or date of birth",
|
| 181 |
+
"DOMAIN_NAME": "an internet domain name or website address",
|
| 182 |
+
"EMAIL_ADDRESS": "an e-mail address",
|
| 183 |
+
"GPE": "a country, city, state or other geopolitical place",
|
| 184 |
+
"IBAN_CODE": "an international bank account number (IBAN)",
|
| 185 |
+
"IP_ADDRESS": "an IPv4 or IPv6 address",
|
| 186 |
+
"NRP": "a nationality, religious or political group",
|
| 187 |
+
"ORGANIZATION": "the name of a company, institution or other organisation",
|
| 188 |
+
"PERSON": "a person's name",
|
| 189 |
+
"PHONE_NUMBER": "a telephone number",
|
| 190 |
+
"STREET_ADDRESS": "a street address or postal address",
|
| 191 |
+
"TITLE": "a personal or professional title such as Dr. or Mrs.",
|
| 192 |
+
"US_DRIVER_LICENSE": "a US driver's licence number",
|
| 193 |
+
"US_SSN": "a US social security number",
|
| 194 |
+
"ZIP_CODE": "a postal or ZIP code"
|
| 195 |
+
}
|
| 196 |
+
},
|
| 197 |
+
"halu": {
|
| 198 |
+
"question": "Which spans of the answer are not supported by the context?",
|
| 199 |
+
"labels": {
|
| 200 |
+
"unsupported": "a claim not supported by the context"
|
| 201 |
+
}
|
| 202 |
+
},
|
| 203 |
+
"evidence": {
|
| 204 |
+
"question": "Which spans of the context support the answer to the question?",
|
| 205 |
+
"labels": {
|
| 206 |
+
"evidence": "a sentence or passage of the context that supports the answer"
|
| 207 |
+
}
|
| 208 |
+
}
|
| 209 |
+
},
|
| 210 |
+
"default_question": "Which spans are named entities of these types?",
|
| 211 |
+
"rule": "routing: open-label entity questions run on the NER adapter (span.ner), everything else on the span adapter; on the span adapter, a label set within one router label set (PII types / unsupported / toxic labels) takes that set's threshold (PII by target length) and, for PII and halu, the trained question text and label descriptions where the request leaves them empty; a label set of evidence names (evidence and its training synonyms) takes the evidence threshold and decoding; other open labels take span.threshold. Open labels get d3-mini label definitions by default on both adapters",
|
| 212 |
+
"threshold_names_only": 0.55,
|
| 213 |
+
"evidence": {
|
| 214 |
+
"labels": [
|
| 215 |
+
"evidence",
|
| 216 |
+
"supporting evidence",
|
| 217 |
+
"supporting passage",
|
| 218 |
+
"relevant text",
|
| 219 |
+
"citation",
|
| 220 |
+
"evidence span",
|
| 221 |
+
"supporting sentence"
|
| 222 |
+
],
|
| 223 |
+
"question": "Which spans of the context support the answer to the question?",
|
| 224 |
+
"threshold": 0.5,
|
| 225 |
+
"decode": {
|
| 226 |
+
"name": "plain",
|
| 227 |
+
"gap": 0,
|
| 228 |
+
"min_chars": 10
|
| 229 |
+
}
|
| 230 |
+
},
|
| 231 |
+
"ner": {
|
| 232 |
+
"head_file": "ner_head.safetensors",
|
| 233 |
+
"definitions": "auto",
|
| 234 |
+
"threshold": 0.6,
|
| 235 |
+
"threshold_names_only": 0.65,
|
| 236 |
+
"route": "open labels (not a router label set, no evidence / answer label) asked with no instructions, or with instructions that name entities / types / categories and match no relation, answer or evidence pattern -> NER adapter + ner_head; everything else (router label sets PII / halu / toxic, evidence / answer labels, relation questions such as \"Which spans are the <relation> of <head>?\", answer / evidence questions, other wordings) -> span adapter + span_head (modeling_vela3._span_route)",
|
| 237 |
+
"source": "N5 ckpt-009000 head final (dev pick: mode defs, eval-half F1 0.7592); thresholds: that checkpoint's ner_shift thr-half picks (definitions 0.6, names only 0.65); mode: definitions, as the 4B"
|
| 238 |
+
},
|
| 239 |
+
"relation": {
|
| 240 |
+
"head_file": "relation_head.safetensors",
|
| 241 |
+
"threshold": "one relation asked (head-given): 0.65 (head-conditioned RE dev: CrossRE dev, FewRel dev); several relations at once: 0.6 (end-to-end RE dev: DocRED dev, CrossRE dev); both on the relation adapter",
|
| 242 |
+
"rule": "relation questions (_span_route relation pattern, no router label set) run on adapters.relation + this head; every other span question routes as before",
|
| 243 |
+
"source": "R5S ckpt-008000 head_final",
|
| 244 |
+
"threshold_dev_multi": 0.6,
|
| 245 |
+
"threshold_dev_pair": 0.65
|
| 246 |
+
}
|
| 247 |
+
},
|
| 248 |
+
"definitions": {
|
| 249 |
+
"cache_file": "label_definitions.json",
|
| 250 |
+
"default": {
|
| 251 |
+
"open labels": "auto",
|
| 252 |
+
"router label sets": null
|
| 253 |
+
},
|
| 254 |
+
"generation": "labels without a description run names only (no generation: this base writes degenerate or no definitions); option: definitions.labeldef_writer",
|
| 255 |
+
"writer": "names",
|
| 256 |
+
"writer_options": "base: the base writes (tied bases only); grafted_head: the backbone + lm_head_file; labeldef_writer: d3-mini@566da0ea (optional download); names: no generation, labels without a cached or given definition run names only. Env VELA3_DEF_WRITER overrides.",
|
| 257 |
+
"labeldef_writer": {
|
| 258 |
+
"repo_id": "vllm-sr/d3-mini",
|
| 259 |
+
"revision": "566da0ea398e27184472add700499245ed09af35",
|
| 260 |
+
"optional": true
|
| 261 |
+
},
|
| 262 |
+
"cache_note": "label_definitions.json ships empty (no benchmark label set is cached); definitions for open labels without a description are written at request time by definitions.writer"
|
| 263 |
+
},
|
| 264 |
+
"render": "d3 span rendering v2 (vela3_span.py)",
|
| 265 |
+
"attention": "prefix_bidir for decisions (state block once per request; router rows: decisions.attention); single-block noncausal full attention for span passes; causal for label definitions",
|
| 266 |
+
"router_gate": "id",
|
| 267 |
+
"decision_opt_in": false,
|
| 268 |
+
"size": {
|
| 269 |
+
"name": "Vela-3.0-27B",
|
| 270 |
+
"d3": "d3-turbo",
|
| 271 |
+
"x8_package": "final (Phase D: round-2 picks; d3 v3.1 runtime + span fix)"
|
| 272 |
+
},
|
| 273 |
+
"x8": {
|
| 274 |
+
"phase": "D (final)",
|
| 275 |
+
"picks": {
|
| 276 |
+
"D": "B",
|
| 277 |
+
"S": "B",
|
| 278 |
+
"N": "B",
|
| 279 |
+
"L": "B"
|
| 280 |
+
},
|
| 281 |
+
"decision": "decision LoRA ckpt-002000",
|
| 282 |
+
"span": "R4 ckpt-024000",
|
| 283 |
+
"ner": "N5 ckpt-009000",
|
| 284 |
+
"relation": "R5S ckpt-008000 (relation questions only)",
|
| 285 |
+
"image_halu": "none at this size yet (MM2 per size: Phase D step 4)",
|
| 286 |
+
"decision_optin": "none at this size (opt-in decision LoRA not trained)",
|
| 287 |
+
"turbo": {
|
| 288 |
+
"built_from": ".d3t_build",
|
| 289 |
+
"base_snapshot": "d3-turbo-v1.0",
|
| 290 |
+
"decisions_attention": "noncausal"
|
| 291 |
+
}
|
| 292 |
+
},
|
| 293 |
+
"decisions": {
|
| 294 |
+
"attention": "noncausal",
|
| 295 |
+
"validated_length": 32768,
|
| 296 |
+
"note": "decisions.attention prefix_bidir = the router rows' extra StateOnce pass with the decision adapter on; noncausal = each router row as one full prompt under the noncausal full-attention mask"
|
| 297 |
+
},
|
| 298 |
+
"decision_task_adapters": {
|
| 299 |
+
"domain": "decision_df",
|
| 300 |
+
"feedback": "decision_df"
|
| 301 |
+
}
|
| 302 |
+
}
|
vela3_labeldef.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Label definitions of Vela-3.0-4B.
|
| 2 |
+
|
| 3 |
+
d3-mini itself, with every adapter off and CAUSAL attention (d3's noncausal mask hook is lifted for the generation), writes a
|
| 4 |
+
one-line definition per label. The prompt and decoding are those of the training-time probe (labeldef.py):
|
| 5 |
+
user turn only, chat template with enable_thinking=False and the generation prompt;
|
| 6 |
+
context "The full label set is: a, b, c." when the label set has <= 60 labels, else the bare label;
|
| 7 |
+
greedy, <= 40 new tokens, first line kept, stripped.
|
| 8 |
+
Each span option is then rendered as `label: definition`. Definitions are cached per label set (in memory), and the file
|
| 9 |
+
label_definitions.json holds the definitions the OOD-7 read used (generated on GPU, batched), keyed by label set."""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import contextlib
|
| 13 |
+
import json
|
| 14 |
+
from collections import OrderedDict
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
CTX_MAX = 60
|
| 20 |
+
MAX_NEW = 40
|
| 21 |
+
PROMPT = ("You are writing annotation guidelines for named entity recognition.{ctx}\n"
|
| 22 |
+
"Define the entity type \"{label}\" in one short sentence: what kind of text span it covers. Reply with the definition only.")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def prompt(label, names):
|
| 26 |
+
ctx = f"\nThe full label set is: {', '.join(names)}." if len(names) <= CTX_MAX else ""
|
| 27 |
+
return PROMPT.format(ctx=ctx, label=label)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@contextlib.contextmanager
|
| 31 |
+
def causal(lm):
|
| 32 |
+
"""Lift d3's noncausal mask pre-hook (named mask_inputs) from the text model for the duration of the block."""
|
| 33 |
+
hooks = lm._forward_pre_hooks
|
| 34 |
+
saved = OrderedDict(hooks)
|
| 35 |
+
for k in [k for k, fn in hooks.items() if getattr(fn, "__name__", "") == "mask_inputs"]:
|
| 36 |
+
del hooks[k]
|
| 37 |
+
try:
|
| 38 |
+
yield
|
| 39 |
+
finally:
|
| 40 |
+
hooks.clear(); hooks.update(saved)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# X8 option A: definition-writer policy (patch_labeldef2.py)
|
| 44 |
+
class LabelDefiner:
|
| 45 |
+
def __init__(self, runtime, adapters, cache_file=None, batch=64):
|
| 46 |
+
self.rt, self.A, self.batch = runtime, adapters, batch
|
| 47 |
+
self.mem = {}
|
| 48 |
+
self.gen = None
|
| 49 |
+
self.def_cfg, self.root = None, None # X8 option A: definition-writer policy (definitions.* of the package's vela3_config.json)
|
| 50 |
+
if cache_file:
|
| 51 |
+
self.root = Path(cache_file).parent; vc = self.root / "vela3_config.json"
|
| 52 |
+
if vc.exists():
|
| 53 |
+
self.def_cfg = json.loads(vc.read_text()).get("definitions") or {}
|
| 54 |
+
if cache_file and Path(cache_file).exists():
|
| 55 |
+
for e in json.loads(Path(cache_file).read_text())["sets"]:
|
| 56 |
+
self.mem[self._key(e["labels"])] = dict(e["definitions"])
|
| 57 |
+
|
| 58 |
+
@staticmethod
|
| 59 |
+
def _key(names):
|
| 60 |
+
names = list(names)
|
| 61 |
+
return tuple(sorted(names)) if len(names) <= CTX_MAX else None
|
| 62 |
+
|
| 63 |
+
def _model(self):
|
| 64 |
+
"""X8 option A: definition-writer policy (vela3_config.json definitions.writer; env VELA3_DEF_WRITER overrides)."""
|
| 65 |
+
if self.gen is None:
|
| 66 |
+
import os, warnings
|
| 67 |
+
from transformers import GenerationConfig
|
| 68 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5ForConditionalGeneration
|
| 69 |
+
dcfg = self.def_cfg or {}; mode = os.environ.get("VELA3_DEF_WRITER") or dcfg.get("writer", "base")
|
| 70 |
+
bb = self.rt.backbone; cfg = bb.config; self.tok_w, self.writer = self.rt.tokenizer, mode
|
| 71 |
+
tied = getattr(cfg, "tie_word_embeddings", None)
|
| 72 |
+
if tied is None:
|
| 73 |
+
tied = getattr(getattr(cfg, "text_config", None), "tie_word_embeddings", True)
|
| 74 |
+
head = None
|
| 75 |
+
try:
|
| 76 |
+
if mode == "names":
|
| 77 |
+
self.gen = False; return None
|
| 78 |
+
if mode == "base":
|
| 79 |
+
assert tied, "refusing to write definitions with an untied base that has no lm_head"
|
| 80 |
+
head = bb.language_model.embed_tokens.weight
|
| 81 |
+
elif mode == "grafted_head":
|
| 82 |
+
from safetensors.torch import load_file
|
| 83 |
+
hf = Path(self.root) / dcfg["lm_head_file"]
|
| 84 |
+
head = load_file(str(hf))["lm_head.weight"].to(self.rt.device, bb.language_model.embed_tokens.weight.dtype)
|
| 85 |
+
assert head.shape == bb.language_model.embed_tokens.weight.shape, (tuple(head.shape), "grafted head does not fit the backbone")
|
| 86 |
+
elif mode == "labeldef_writer":
|
| 87 |
+
w = dcfg["labeldef_writer"]; path = os.environ.get("VELA3_LABELDEF_WRITER")
|
| 88 |
+
if not path:
|
| 89 |
+
from huggingface_hub import snapshot_download
|
| 90 |
+
path = snapshot_download(w["repo_id"], revision=w.get("revision"))
|
| 91 |
+
from transformers import AutoTokenizer
|
| 92 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model
|
| 93 |
+
bb = Qwen3_5Model.from_pretrained(path, dtype=torch.bfloat16, device_map={"": str(self.rt.device)}).eval(); cfg = bb.config
|
| 94 |
+
wt = getattr(cfg, "tie_word_embeddings", None)
|
| 95 |
+
wt = getattr(getattr(cfg, "text_config", None), "tie_word_embeddings", True) if wt is None else wt
|
| 96 |
+
assert wt, f"definition writer {path} is untied"
|
| 97 |
+
head = bb.language_model.embed_tokens.weight; self.tok_w = AutoTokenizer.from_pretrained(path); self.writer = f"labeldef_writer {path}"
|
| 98 |
+
else:
|
| 99 |
+
raise ValueError(f"unknown definitions.writer {mode!r}")
|
| 100 |
+
except Exception as e: # names only for uncached labels, never a definition from an untied head
|
| 101 |
+
warnings.warn(f"vela3_labeldef: no definition writer ({mode}: {type(e).__name__}: {e}); uncached labels run names only")
|
| 102 |
+
self.gen = False; return None
|
| 103 |
+
with torch.device("meta"):
|
| 104 |
+
g = Qwen3_5ForConditionalGeneration(cfg)
|
| 105 |
+
g.model = bb
|
| 106 |
+
g.lm_head = torch.nn.Linear(head.shape[1], head.shape[0], bias=False, device="meta")
|
| 107 |
+
g.lm_head.weight = head if isinstance(head, torch.nn.Parameter) else torch.nn.Parameter(head, requires_grad=False)
|
| 108 |
+
g.generation_config = GenerationConfig.from_model_config(cfg)
|
| 109 |
+
g.eval()
|
| 110 |
+
self.gen = g
|
| 111 |
+
return self.gen or None
|
| 112 |
+
|
| 113 |
+
@torch.no_grad()
|
| 114 |
+
def generate(self, prompts):
|
| 115 |
+
g = self._model(); out = []
|
| 116 |
+
if g is None:
|
| 117 |
+
return [None] * len(prompts)
|
| 118 |
+
tok = self.tok_w
|
| 119 |
+
lm = g.model.language_model
|
| 120 |
+
for i in range(0, len(prompts), self.batch):
|
| 121 |
+
bt = prompts[i:i + self.batch]
|
| 122 |
+
txt = [tok.apply_chat_template([{"role": "user", "content": p}], tokenize=False, add_generation_prompt=True, enable_thinking=False) for p in bt]
|
| 123 |
+
enc = tok(txt, return_tensors="pt", padding=True, add_special_tokens=False).to(self.rt.device)
|
| 124 |
+
with self.A.active(None), causal(lm):
|
| 125 |
+
o = g.generate(**enc, max_new_tokens=MAX_NEW, do_sample=False, pad_token_id=tok.pad_token_id)
|
| 126 |
+
out += [tok.decode(x, skip_special_tokens=True).strip().split("\n")[0].strip() for x in o[:, enc["input_ids"].shape[1]:]]
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
def define(self, names, only=None):
|
| 130 |
+
"""{label: definition} for the label set `names` (order as given); `only` restricts which labels are generated."""
|
| 131 |
+
names = list(names); want = [n for n in names if only is None or n in only]
|
| 132 |
+
key = self._key(names)
|
| 133 |
+
got = dict(self.mem.get(key, {})) if key is not None else {n: self.mem.get((None, n)) for n in want if (None, n) in self.mem}
|
| 134 |
+
miss = [n for n in want if not got.get(n)]
|
| 135 |
+
if miss:
|
| 136 |
+
for n, d in zip(miss, self.generate([prompt(n, names) for n in miss])):
|
| 137 |
+
if d is None: # no writer under this policy: names only for this label
|
| 138 |
+
continue
|
| 139 |
+
got[n] = d
|
| 140 |
+
if key is None:
|
| 141 |
+
self.mem[(None, n)] = d
|
| 142 |
+
if key is not None:
|
| 143 |
+
self.mem[key] = {**self.mem.get(key, {}), **{n: got[n] for n in miss if n in got}}
|
| 144 |
+
return {n: got.get(n) for n in want} # X8: a label without a definition runs names only
|
vela3_span.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Span pass of Vela-3.0-4B.
|
| 2 |
+
|
| 3 |
+
One span question = one forward pass of d3-mini (noncausal full attention, as shipped) with the SPAN LoRA switched on, over d3's
|
| 4 |
+
own prompt: system prompt, then `State:` (the request parts), `Question:` (the span question) and `Options:` (one line per label,
|
| 5 |
+
`<code>: <label>[: <description or definition>]`), through the chat template. The span head (Vela 2.0 SpanHeadV2: word x label
|
| 6 |
+
bilinear + MLP) reads the final-norm states and scores every word of the target part against every label; words above the
|
| 7 |
+
threshold are joined into spans (Vela 2.0 decoder, word units).
|
| 8 |
+
State rendering: one part = its text; several parts = "<role>:\\n<text>" blocks in order, blank-line separated.
|
| 9 |
+
Targets longer than 2,048 tokens are split into word-aligned windows (<= 1,800 tokens, stride 1,536); word logits are averaged.
|
| 10 |
+
This is the training-time eval path (r1_eval.Predictor) at batch 1.
|
| 11 |
+
With images (system_one images=[...]), the images enter the same pass in d3's image layout (SpanEngine._image_logits)."""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import contextlib
|
| 15 |
+
import math
|
| 16 |
+
import sys
|
| 17 |
+
import re as _re
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
import torch
|
| 21 |
+
import torch.nn.functional as F
|
| 22 |
+
from torch import nn
|
| 23 |
+
|
| 24 |
+
WIN, STRIDE, RMAX = 1800, 1536, 2048
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def text_states(rt, ids):
|
| 28 |
+
"""Final-norm states of one unpadded text prompt, as d3 runs a noncausal_full_attention checkpoint (what the adapters
|
| 29 |
+
were trained on). d3 <= 566da0ea puts a mask pre-hook on the text model, so the plain backbone call is noncausal; d3 v3.1
|
| 30 |
+
runs such checkpoints through rt.encoder (PromptEncoder) and leaves the stock backbone forward causal."""
|
| 31 |
+
mask = torch.ones_like(ids)
|
| 32 |
+
enc = getattr(rt, "encoder", None)
|
| 33 |
+
if enc is not None:
|
| 34 |
+
return enc(rt.backbone.get_input_embeddings()(ids), mask)
|
| 35 |
+
return rt.backbone(input_ids=ids, attention_mask=mask, use_cache=False).last_hidden_state
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def image_states(rt, inputs):
|
| 39 |
+
"""text_states for an image prompt (processor outputs): v3.1's image embeddings + mRoPE positions through rt.encoder."""
|
| 40 |
+
enc = getattr(rt, "encoder", None)
|
| 41 |
+
if enc is not None and hasattr(rt, "image_embeddings"):
|
| 42 |
+
embeds, mrope = rt.image_embeddings(inputs)
|
| 43 |
+
return enc(embeds, inputs["attention_mask"], mrope)
|
| 44 |
+
return rt.backbone(**inputs, use_cache=False).last_hidden_state
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class SpanHeadV2(nn.Module):
|
| 48 |
+
def __init__(self, hidden, d=256, slots=64):
|
| 49 |
+
super().__init__()
|
| 50 |
+
self.d = d
|
| 51 |
+
self.wn, self.ln = nn.LayerNorm(hidden), nn.LayerNorm(hidden)
|
| 52 |
+
self.slot = nn.Embedding(slots, hidden)
|
| 53 |
+
self.K, self.Q = nn.Linear(hidden, d, bias=False), nn.Linear(hidden, d, bias=False)
|
| 54 |
+
self.M, self.N = nn.Linear(hidden, d), nn.Linear(hidden, d, bias=False)
|
| 55 |
+
self.v = nn.Linear(d, 1, bias=False)
|
| 56 |
+
self.o = nn.Linear(hidden, 1)
|
| 57 |
+
|
| 58 |
+
def forward(self, words, labels):
|
| 59 |
+
with torch.autocast(device_type=words.device.type, enabled=False):
|
| 60 |
+
w = self.wn(words.float()); w = w - w.mean(0, keepdim=True)
|
| 61 |
+
L = labels.shape[0]
|
| 62 |
+
e = self.ln(labels.float()) + self.slot(torch.arange(L, device=labels.device).clamp(max=self.slot.num_embeddings - 1))
|
| 63 |
+
if L >= 2:
|
| 64 |
+
e = e - e.mean(0, keepdim=True)
|
| 65 |
+
k, q = self.K(w), self.Q(e)
|
| 66 |
+
bil = (k @ q.T) / math.sqrt(self.d)
|
| 67 |
+
mlp = self.v(F.gelu(self.M(w)[:, None, :] + self.N(e)[None, :, :])).squeeze(-1)
|
| 68 |
+
return bil + mlp, self.o(w).squeeze(-1), q
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
_CJK = ("⺀- -〿-ヿ-ㇿ㐀-䶿一-鿿ꀀ-"
|
| 72 |
+
"가-豈-︰-﹏-")
|
| 73 |
+
_TRAIL = ".,;:!?)\\]}>\"'”’»"
|
| 74 |
+
_WORD = _re.compile(rf"""(?:https?://|www\.)[^\s{_CJK}]+?(?=[{_TRAIL}]*(?![^\s{_CJK}]))
|
| 75 |
+
|[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{{2,}}
|
| 76 |
+
|@[a-z0-9_]+
|
| 77 |
+
|[{_CJK}]
|
| 78 |
+
|[^\W{_CJK}]+(?:[-_][^\W{_CJK}]+)*
|
| 79 |
+
|\S""", _re.VERBOSE | _re.IGNORECASE)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def word_first_tokens(text, offs):
|
| 83 |
+
w = [(m.start(), m.end()) for m in _WORD.finditer(text)]
|
| 84 |
+
if not w or offs is None or len(offs) == 0:
|
| 85 |
+
return np.zeros((0, 2), np.int32), np.zeros(0, np.int32)
|
| 86 |
+
offs = np.asarray(offs); st, en = offs[:, 0], offs[:, 1]; valid = en > st
|
| 87 |
+
cummax_end = np.maximum.accumulate(np.where(valid, en, -1)); wo, ft = [], []
|
| 88 |
+
for a, b in w:
|
| 89 |
+
t = int(np.searchsorted(cummax_end, a, side="right"))
|
| 90 |
+
while t < len(offs) and not valid[t]:
|
| 91 |
+
t += 1
|
| 92 |
+
if t < len(offs) and st[t] < b:
|
| 93 |
+
wo.append((a, b)); ft.append(t)
|
| 94 |
+
return np.asarray(wo, np.int32).reshape(-1, 2), np.asarray(ft, np.int32)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def windows(ft, n, win=WIN, stride=STRIDE):
|
| 98 |
+
starts = sorted(set(int(x) for x in ft))
|
| 99 |
+
if not starts:
|
| 100 |
+
return [(0, n)]
|
| 101 |
+
out, a = [], starts[0]
|
| 102 |
+
while True:
|
| 103 |
+
lim = a + win
|
| 104 |
+
nxt = [s for s in starts if s > a and s < lim]
|
| 105 |
+
b = n if lim >= n else (max(s for s in starts if s < lim and s > a) if nxt else min(n, lim))
|
| 106 |
+
out.append((a, b))
|
| 107 |
+
if b >= n:
|
| 108 |
+
break
|
| 109 |
+
cand = [s for s in starts if s >= a + stride and s < b]
|
| 110 |
+
a = cand[0] if cand else b
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class Render:
|
| 115 |
+
def __init__(self, tok, d3_format):
|
| 116 |
+
self.tok, self.F = tok, d3_format
|
| 117 |
+
self.codes, _ = d3_format.answer_codes(tok)
|
| 118 |
+
|
| 119 |
+
def state_of(self, parts, target, text):
|
| 120 |
+
if len(parts) == 1:
|
| 121 |
+
return text, 0
|
| 122 |
+
blocks, t0, pos = [], None, 0
|
| 123 |
+
for ro, t in parts:
|
| 124 |
+
hdr = f"{ro}:\n"; body = text if ro == target else t
|
| 125 |
+
if ro == target:
|
| 126 |
+
t0 = pos + len(hdr)
|
| 127 |
+
blocks.append(hdr + body); pos += len(hdr) + len(body) + 2
|
| 128 |
+
return "\n\n".join(blocks), t0
|
| 129 |
+
|
| 130 |
+
def __call__(self, parts, target, question, labels, text, n_images=0, processor=None):
|
| 131 |
+
F_ = self.F
|
| 132 |
+
state, t0 = self.state_of(parts, target, text)
|
| 133 |
+
opt_lines = [f"{c}: {n if not d else n + ': ' + d}" for c, (n, d) in zip(self.codes, labels)]
|
| 134 |
+
head = ["State:", state if state else "(empty)", "", "Question:", question, "", "Options:"]
|
| 135 |
+
user = "\n".join(head + opt_lines + ["", "Reply with only the code of the best option."])
|
| 136 |
+
if n_images: # d3's image layout: one placeholder per image before the user text, rendered by the checkpoint's processor
|
| 137 |
+
content = [{"type": "image"} for _ in range(n_images)] + [{"type": "text", "text": user}]
|
| 138 |
+
rendered = processor.apply_chat_template([{"role": "system", "content": F_.SYSTEM_PROMPT}, {"role": "user", "content": content}],
|
| 139 |
+
tokenize=False, add_generation_prompt=True, enable_thinking=False)
|
| 140 |
+
else:
|
| 141 |
+
rendered = self.tok.apply_chat_template([{"role": "system", "content": F_.SYSTEM_PROMPT}, {"role": "user", "content": user}],
|
| 142 |
+
tokenize=False, add_generation_prompt=True, enable_thinking=False)
|
| 143 |
+
u0 = rendered.index(user); s0 = u0 + len("State:\n") + t0
|
| 144 |
+
e = self.tok(rendered, add_special_tokens=False, return_offsets_mapping=True)
|
| 145 |
+
ids = list(e["input_ids"]); offs = np.asarray(e["offset_mapping"], dtype=np.int64).reshape(-1, 2)
|
| 146 |
+
wo, ft = word_first_tokens(text, offs - s0)
|
| 147 |
+
pos = u0 + len("\n".join(head)) + 1; ranges = []; st, en = offs[:, 0], offs[:, 1]
|
| 148 |
+
for ol in opt_lines:
|
| 149 |
+
ix = np.nonzero((st < pos + len(ol)) & (en > pos))[0]; ranges.append((int(ix[0]), int(ix[-1]))); pos += len(ol) + 1
|
| 150 |
+
rec = dict(ids=ids, w_pos=ft.astype(np.int64), w_offs=wo, ranges=ranges)
|
| 151 |
+
if n_images:
|
| 152 |
+
rec["rendered"] = rendered
|
| 153 |
+
return rec
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class SpanEngine:
|
| 157 |
+
"""One span head on the final-norm states; its adapter ("span" or "ner") is on for every forward of this engine."""
|
| 158 |
+
|
| 159 |
+
def __init__(self, runtime, head, adapters, d3_format, adapter="span"):
|
| 160 |
+
self.rt, self.head, self.A, self.adapter = runtime, head, adapters, adapter
|
| 161 |
+
self.R = Render(runtime.tokenizer, d3_format)
|
| 162 |
+
|
| 163 |
+
def n_tokens(self, text):
|
| 164 |
+
return len(self.rt.tokenizer(text, add_special_tokens=False)["input_ids"])
|
| 165 |
+
|
| 166 |
+
def _pieces(self, parts, target, question, labels):
|
| 167 |
+
text = dict(parts)[target]
|
| 168 |
+
e = self.rt.tokenizer(text, add_special_tokens=False, return_offsets_mapping=True)
|
| 169 |
+
ids = e["input_ids"]; offs = np.asarray(e["offset_mapping"], dtype=np.int64).reshape(-1, 2)
|
| 170 |
+
if len(ids) <= RMAX:
|
| 171 |
+
return [(self.R(parts, target, question, labels, text), 0)], len(ids)
|
| 172 |
+
wo, ft = word_first_tokens(text, offs); out = []
|
| 173 |
+
for a0, b0 in windows(ft, len(ids)):
|
| 174 |
+
cs, ce = int(offs[a0][0]), int(offs[b0 - 1][1])
|
| 175 |
+
out.append((self.R(parts, target, question, labels, text[cs:ce]), cs))
|
| 176 |
+
return out, len(ids)
|
| 177 |
+
|
| 178 |
+
@torch.no_grad()
|
| 179 |
+
def logits(self, parts, target, question, labels, images=None):
|
| 180 |
+
"""Word offsets (W, 2) into the target text, word x label logits (W, L), target tokens, windows."""
|
| 181 |
+
if images:
|
| 182 |
+
return self._image_logits(parts, target, question, labels, images)
|
| 183 |
+
acc = {}
|
| 184 |
+
pieces, n_tok = self._pieces(parts, target, question, labels)
|
| 185 |
+
for rec, cs in pieces:
|
| 186 |
+
if len(rec["w_offs"]) == 0:
|
| 187 |
+
continue
|
| 188 |
+
ids = torch.as_tensor([rec["ids"]], device=self.rt.device)
|
| 189 |
+
with self.A.active(self.adapter):
|
| 190 |
+
H = text_states(self.rt, ids)
|
| 191 |
+
w = H[0, torch.as_tensor(rec["w_pos"], device=H.device)]
|
| 192 |
+
lab = torch.stack([H[0, s:e + 1].mean(0) for s, e in rec["ranges"]])
|
| 193 |
+
lg = self.head(w, lab)[0].float().cpu().numpy()
|
| 194 |
+
for wi, (s, e) in enumerate(rec["w_offs"]):
|
| 195 |
+
v = acc.setdefault((int(s) + cs, int(e) + cs), [np.zeros(len(labels)), 0]); v[0] += lg[wi]; v[1] += 1
|
| 196 |
+
keys = sorted(acc)
|
| 197 |
+
wo = np.asarray(keys, dtype=np.int64).reshape(-1, 2)
|
| 198 |
+
lg = np.stack([acc[k][0] / acc[k][1] for k in keys]) if keys else np.zeros((0, len(labels)))
|
| 199 |
+
return wo, lg, n_tok, len(pieces)
|
| 200 |
+
|
| 201 |
+
@torch.no_grad()
|
| 202 |
+
def _image_logits(self, parts, target, question, labels, images):
|
| 203 |
+
"""The span pass with images (decoded RGB images): d3's image layout, the processor expands each placeholder into the
|
| 204 |
+
image's visual tokens and every tensor it returns goes to the backbone (its own mRoPE positions); word and label positions
|
| 205 |
+
shift by the visual tokens. One sequence: targets longer than RMAX tokens are refused."""
|
| 206 |
+
proc = self.rt.processor
|
| 207 |
+
if proc is None:
|
| 208 |
+
raise ValueError(f"image inputs are not available: {self.rt.image_unavailable}")
|
| 209 |
+
text = dict(parts)[target]
|
| 210 |
+
n_tok = len(self.rt.tokenizer(text, add_special_tokens=False)["input_ids"])
|
| 211 |
+
if n_tok > RMAX:
|
| 212 |
+
raise ValueError(f"span questions with images take targets of up to {RMAX} tokens")
|
| 213 |
+
rec = self.R(parts, target, question, labels, text, n_images=len(images), processor=proc)
|
| 214 |
+
if len(rec["w_offs"]) == 0:
|
| 215 |
+
return np.zeros((0, 2), np.int64), np.zeros((0, len(labels))), n_tok, 1
|
| 216 |
+
enc = proc(text=[rec["rendered"]], images=list(images), return_tensors="pt")
|
| 217 |
+
ids, got = rec["ids"], enc["input_ids"][0].tolist()
|
| 218 |
+
img = self.rt.backbone.config.image_token_id
|
| 219 |
+
k = [i for i, t in enumerate(ids) if t == img]
|
| 220 |
+
sh = len(got) - len(ids)
|
| 221 |
+
if (len(k) != len(images) or sh < 0 or got[:k[0]] != ids[:k[0]] or got[k[-1] + 1 + sh:] != ids[k[-1] + 1:]
|
| 222 |
+
or (rec["w_pos"] <= k[-1]).any() or any(a <= k[-1] for a, _ in rec["ranges"])):
|
| 223 |
+
raise RuntimeError("the processor's ids differ from the text ids with the image placeholders expanded")
|
| 224 |
+
rt_mod = sys.modules.get(type(self.rt).__module__)
|
| 225 |
+
backends = getattr(rt_mod, "sdpa_backends", None)
|
| 226 |
+
inputs = {n: v.to(self.rt.device) for n, v in enc.items()}
|
| 227 |
+
with self.A.active(self.adapter), (backends(self.rt.device) if backends else contextlib.nullcontext()):
|
| 228 |
+
H = image_states(self.rt, inputs)
|
| 229 |
+
w = H[0, torch.as_tensor(rec["w_pos"] + sh, device=H.device)]
|
| 230 |
+
lab = torch.stack([H[0, s + sh:e + sh + 1].mean(0) for s, e in rec["ranges"]])
|
| 231 |
+
lg = self.head(w, lab)[0].float().cpu().numpy()
|
| 232 |
+
return np.asarray(rec["w_offs"], dtype=np.int64).reshape(-1, 2), lg, n_tok, 1
|