ctxprune-small / eval /qa_v1-final-v1v3__deepseek_deepseek-v4-flash.json
darioooooo0o's picture
ctxprune-small v0.1: weights, ONNX fp32/int8, card, eval results
09d6bec verified
Raw History Blame Contribute Delete
3.99 kB
{
"original": {
"fineweb2": {
"em": 0.722,
"f1": 0.722,
"n": 18
},
"_all": {
"em": 0.918,
"f1": 0.88,
"n": 376
},
"fineweb_edu": {
"em": 0.966,
"f1": 0.942,
"n": 58
},
"github_code": {
"em": 0.938,
"f1": 0.923,
"n": 65
},
"oasst2": {
"em": 0.938,
"f1": 0.81,
"n": 16
},
"swe_rebench_assistant": {
"em": 0.946,
"f1": 0.949,
"n": 37
},
"swe_rebench_tool": {
"em": 0.896,
"f1": 0.802,
"n": 77
},
"toucan_mcp": {
"em": 0.926,
"f1": 0.908,
"n": 54
},
"swe_smith_tool": {
"em": 0.902,
"f1": 0.873,
"n": 51
},
"_char_ratio": 1.0,
"_llm_token_ratio": 1.0
},
"ours@0.5": {
"fineweb2": {
"em": 0.556,
"f1": 0.556,
"n": 18
},
"_all": {
"em": 0.688,
"f1": 0.684,
"n": 378
},
"fineweb_edu": {
"em": 0.733,
"f1": 0.789,
"n": 60
},
"github_code": {
"em": 0.723,
"f1": 0.708,
"n": 65
},
"oasst2": {
"em": 0.75,
"f1": 0.77,
"n": 16
},
"swe_rebench_assistant": {
"em": 0.703,
"f1": 0.786,
"n": 37
},
"swe_rebench_tool": {
"em": 0.597,
"f1": 0.543,
"n": 77
},
"toucan_mcp": {
"em": 0.593,
"f1": 0.617,
"n": 54
},
"swe_smith_tool": {
"em": 0.843,
"f1": 0.76,
"n": 51
},
"_char_ratio": 0.485,
"_llm_token_ratio": 0.492
},
"ours+ids@0.5": {
"fineweb2": {
"em": 0.556,
"f1": 0.556,
"n": 18
},
"_all": {
"em": 0.635,
"f1": 0.64,
"n": 378
},
"fineweb_edu": {
"em": 0.7,
"f1": 0.744,
"n": 60
},
"github_code": {
"em": 0.692,
"f1": 0.679,
"n": 65
},
"oasst2": {
"em": 0.75,
"f1": 0.77,
"n": 16
},
"swe_rebench_assistant": {
"em": 0.676,
"f1": 0.74,
"n": 37
},
"swe_rebench_tool": {
"em": 0.571,
"f1": 0.555,
"n": 77
},
"toucan_mcp": {
"em": 0.556,
"f1": 0.571,
"n": 54
},
"swe_smith_tool": {
"em": 0.627,
"f1": 0.586,
"n": 51
},
"_char_ratio": 0.498,
"_llm_token_ratio": 0.559
},
"ours@0.33": {
"fineweb2": {
"em": 0.389,
"f1": 0.452,
"n": 18
},
"_all": {
"em": 0.534,
"f1": 0.538,
"n": 378
},
"fineweb_edu": {
"em": 0.55,
"f1": 0.637,
"n": 60
},
"github_code": {
"em": 0.508,
"f1": 0.477,
"n": 65
},
"oasst2": {
"em": 0.688,
"f1": 0.648,
"n": 16
},
"swe_rebench_assistant": {
"em": 0.622,
"f1": 0.639,
"n": 37
},
"swe_rebench_tool": {
"em": 0.468,
"f1": 0.444,
"n": 77
},
"toucan_mcp": {
"em": 0.481,
"f1": 0.478,
"n": 54
},
"swe_smith_tool": {
"em": 0.647,
"f1": 0.626,
"n": 51
},
"_char_ratio": 0.31,
"_llm_token_ratio": 0.322
},
"ours+ids@0.33": {
"fineweb2": {
"em": 0.444,
"f1": 0.444,
"n": 18
},
"_all": {
"em": 0.529,
"f1": 0.541,
"n": 378
},
"fineweb_edu": {
"em": 0.533,
"f1": 0.615,
"n": 60
},
"github_code": {
"em": 0.585,
"f1": 0.551,
"n": 65
},
"oasst2": {
"em": 0.688,
"f1": 0.648,
"n": 16
},
"swe_rebench_assistant": {
"em": 0.432,
"f1": 0.509,
"n": 37
},
"swe_rebench_tool": {
"em": 0.558,
"f1": 0.536,
"n": 77
},
"toucan_mcp": {
"em": 0.481,
"f1": 0.528,
"n": 54
},
"swe_smith_tool": {
"em": 0.51,
"f1": 0.49,
"n": 51
},
"_char_ratio": 0.382,
"_llm_token_ratio": 0.459
}
}