ctxprune-small / eval /v1-final-v1v3.json
darioooooo0o's picture
ctxprune-small v0.1: weights, ONNX fp32/int8, card, eval results
09d6bec verified
Raw History Blame Contribute Delete
16.5 kB
{
"model": "/mnt/ssd/ctxprune-data/runs/v1-final-v1v3/final",
"split": "test",
"methods": {
"ours@0.5": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 167,
"lost_rate": 0.5758620689655173,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.46790665518158064,
"mean_llm_token_ratio": 0.454551076188952
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 116,
"lost_rate": 0.5550239234449761,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5251933817728502,
"mean_llm_token_ratio": 0.49892634141769937
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 80,
"lost_rate": 0.2888086642599278,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4900923855979181,
"mean_llm_token_ratio": 0.49410920530451385
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5233399079552926,
"mean_llm_token_ratio": 0.4981949458483754
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 7,
"lost_rate": 0.0875,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4762202894804961,
"mean_llm_token_ratio": 0.4943262326551133
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 5,
"lost_rate": 0.13157894736842105,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.46665221854276134,
"mean_llm_token_ratio": 0.4980696352415177
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 38,
"lost_rate": 0.4367816091954023,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.531668937503145,
"mean_llm_token_ratio": 0.4917416835051349
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 6,
"lost_rate": 0.21428571428571427,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.34696757073863704,
"mean_llm_token_ratio": 0.4981844242713808
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 419,
"lost_rate": 0.4136229022704837,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.4807325967331789,
"mean_llm_token_ratio": 0.48781368873878034
},
"_seconds": 5.2,
"_ms_per_chunk": 87.8
},
"ours(chars)@0.5": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 165,
"lost_rate": 0.5689655172413793,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5031079237938365,
"mean_llm_token_ratio": 0.49231885120417124
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 133,
"lost_rate": 0.6363636363636364,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4381255138088153,
"mean_llm_token_ratio": 0.42866471440857196
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 81,
"lost_rate": 0.2924187725631769,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4916920386002186,
"mean_llm_token_ratio": 0.5075418238279651
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5016436554898094,
"mean_llm_token_ratio": 0.47653429602888087
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 5,
"lost_rate": 0.0625,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5067393191897068,
"mean_llm_token_ratio": 0.5236473269669588
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 4,
"lost_rate": 0.10526315789473684,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5362221881998749,
"mean_llm_token_ratio": 0.5748280755051555
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 45,
"lost_rate": 0.5172413793103449,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.4654126478489207,
"mean_llm_token_ratio": 0.4294138779235846
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 4,
"lost_rate": 0.14285714285714285,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5077163620401056,
"mean_llm_token_ratio": 0.6181099697404046
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 437,
"lost_rate": 0.43139190523198423,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.4954730832799744,
"mean_llm_token_ratio": 0.5064165971236685
},
"_seconds": 5.2,
"_ms_per_chunk": 88.9
},
"ours+ids@0.5": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5848708565017198,
"mean_llm_token_ratio": 0.6607594850156845
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.43344472653878213,
"mean_llm_token_ratio": 0.5282570081147547
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.49802537688259274,
"mean_llm_token_ratio": 0.5062940001561812
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5233399079552926,
"mean_llm_token_ratio": 0.4981949458483754
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4647579279628507,
"mean_llm_token_ratio": 0.4987588567685885
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.4646904859777982,
"mean_llm_token_ratio": 0.4980696352415177
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.4868151814862343,
"mean_llm_token_ratio": 0.5379034319648817
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.33618142678510043,
"mean_llm_token_ratio": 0.4981844242713808
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.48817782525844866,
"mean_llm_token_ratio": 0.5369469744482747
},
"_seconds": 5.7,
"_ms_per_chunk": 96.0
},
"ours@0.33": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 209,
"lost_rate": 0.7206896551724138,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.327296715791273,
"mean_llm_token_ratio": 0.31459708951298865
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 151,
"lost_rate": 0.722488038277512,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3492269554168825,
"mean_llm_token_ratio": 0.3270323309890272
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 134,
"lost_rate": 0.48375451263537905,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3234588557857195,
"mean_llm_token_ratio": 0.32284723564881385
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.33596318211702825,
"mean_llm_token_ratio": 0.3285198555956679
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 18,
"lost_rate": 0.225,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.2877951029262656,
"mean_llm_token_ratio": 0.3277054998143117
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 12,
"lost_rate": 0.3157894736842105,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.30053077932409866,
"mean_llm_token_ratio": 0.3264086608045815
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 52,
"lost_rate": 0.5977011494252874,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.34743765737380344,
"mean_llm_token_ratio": 0.2995299597179061
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 10,
"lost_rate": 0.35714285714285715,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.18068367830835266,
"mean_llm_token_ratio": 0.3247372193024367
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 586,
"lost_rate": 0.5784797630799605,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.3118052335847517,
"mean_llm_token_ratio": 0.32136154585011445
},
"_seconds": 5.6,
"_ms_per_chunk": 94.8
},
"ours(chars)@0.33": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 210,
"lost_rate": 0.7241379310344828,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3439364522856813,
"mean_llm_token_ratio": 0.3314479793413248
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 162,
"lost_rate": 0.7751196172248804,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.28580805596060765,
"mean_llm_token_ratio": 0.27243181669152455
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 138,
"lost_rate": 0.4981949458483754,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.31775308471956165,
"mean_llm_token_ratio": 0.31525592307107037
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3333333333333333,
"mean_llm_token_ratio": 0.3249097472924188
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 10,
"lost_rate": 0.125,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3365119094824531,
"mean_llm_token_ratio": 0.37627862799399553
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 11,
"lost_rate": 0.2894736842105263,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3573804579329673,
"mean_llm_token_ratio": 0.3856092246572405
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 54,
"lost_rate": 0.6206896551724138,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.30707900248035375,
"mean_llm_token_ratio": 0.27636029308647636
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 6,
"lost_rate": 0.21428571428571427,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3374447399733389,
"mean_llm_token_ratio": 0.49111124382863514
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 591,
"lost_rate": 0.5834155972359328,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.3282637488467005,
"mean_llm_token_ratio": 0.3404873669260685
},
"_seconds": 5.0,
"_ms_per_chunk": 84.4
},
"ours+ids@0.33": {
"swe_rebench_tool": {
"n": 11,
"protected": 290,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.5750178444782263,
"mean_llm_token_ratio": 0.6529765472791863
},
"swe_smith_tool": {
"n": 6,
"protected": 209,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3742236154767515,
"mean_llm_token_ratio": 0.47296373915503437
},
"github_code": {
"n": 14,
"protected": 277,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.3753987414580236,
"mean_llm_token_ratio": 0.38185252441714607
},
"oasst2": {
"n": 1,
"protected": 4,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.33596318211702825,
"mean_llm_token_ratio": 0.3285198555956679
},
"fineweb_edu": {
"n": 12,
"protected": 80,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.27070723428966886,
"mean_llm_token_ratio": 0.3296230704501357
},
"fineweb2": {
"n": 7,
"protected": 38,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.2918360105579523,
"mean_llm_token_ratio": 0.3264086608045815
},
"toucan_mcp": {
"n": 5,
"protected": 87,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.41575461058811414,
"mean_llm_token_ratio": 0.48518356616090375
},
"swe_rebench_assistant": {
"n": 3,
"protected": 28,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 0,
"json_still_valid": 0,
"mean_char_ratio": 0.17654440314872719,
"mean_llm_token_ratio": 0.3341654722089505
},
"_all": {
"n": 59,
"protected": 1013,
"lost": 0,
"lost_rate": 0.0,
"json_inputs": 1,
"json_still_valid": 0,
"mean_char_ratio": 0.3739292998715051,
"mean_llm_token_ratio": 0.42989375204918706
},
"_seconds": 5.7,
"_ms_per_chunk": 97.0
}
},
"agreement": {
"swe_rebench_tool": {
"keep_f1": 0.789,
"acc": 0.697
},
"swe_smith_tool": {
"keep_f1": 0.834,
"acc": 0.746
},
"github_code": {
"keep_f1": 0.932,
"acc": 0.921
},
"oasst2": {
"keep_f1": 0.802,
"acc": 0.847
},
"fineweb_edu": {
"keep_f1": 0.873,
"acc": 0.795
},
"fineweb2": {
"keep_f1": 0.762,
"acc": 0.624
},
"toucan_mcp": {
"keep_f1": 0.684,
"acc": 0.592
},
"swe_rebench_assistant": {
"keep_f1": 0.833,
"acc": 0.731
}
}
}