{ "model": "/mnt/ssd/ctxprune-data/runs/v1-final-v1v3/final", "split": "test", "methods": { "ours@0.5": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 167, "lost_rate": 0.5758620689655173, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.46790665518158064, "mean_llm_token_ratio": 0.454551076188952 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 116, "lost_rate": 0.5550239234449761, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5251933817728502, "mean_llm_token_ratio": 0.49892634141769937 }, "github_code": { "n": 14, "protected": 277, "lost": 80, "lost_rate": 0.2888086642599278, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4900923855979181, "mean_llm_token_ratio": 0.49410920530451385 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5233399079552926, "mean_llm_token_ratio": 0.4981949458483754 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 7, "lost_rate": 0.0875, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4762202894804961, "mean_llm_token_ratio": 0.4943262326551133 }, "fineweb2": { "n": 7, "protected": 38, "lost": 5, "lost_rate": 0.13157894736842105, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.46665221854276134, "mean_llm_token_ratio": 0.4980696352415177 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 38, "lost_rate": 0.4367816091954023, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.531668937503145, "mean_llm_token_ratio": 0.4917416835051349 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 6, "lost_rate": 0.21428571428571427, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.34696757073863704, "mean_llm_token_ratio": 0.4981844242713808 }, "_all": { "n": 59, "protected": 1013, "lost": 419, "lost_rate": 0.4136229022704837, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.4807325967331789, "mean_llm_token_ratio": 0.48781368873878034 }, "_seconds": 5.2, "_ms_per_chunk": 87.8 }, "ours(chars)@0.5": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 165, "lost_rate": 0.5689655172413793, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5031079237938365, "mean_llm_token_ratio": 0.49231885120417124 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 133, "lost_rate": 0.6363636363636364, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4381255138088153, "mean_llm_token_ratio": 0.42866471440857196 }, "github_code": { "n": 14, "protected": 277, "lost": 81, "lost_rate": 0.2924187725631769, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4916920386002186, "mean_llm_token_ratio": 0.5075418238279651 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5016436554898094, "mean_llm_token_ratio": 0.47653429602888087 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 5, "lost_rate": 0.0625, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5067393191897068, "mean_llm_token_ratio": 0.5236473269669588 }, "fineweb2": { "n": 7, "protected": 38, "lost": 4, "lost_rate": 0.10526315789473684, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5362221881998749, "mean_llm_token_ratio": 0.5748280755051555 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 45, "lost_rate": 0.5172413793103449, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.4654126478489207, "mean_llm_token_ratio": 0.4294138779235846 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 4, "lost_rate": 0.14285714285714285, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5077163620401056, "mean_llm_token_ratio": 0.6181099697404046 }, "_all": { "n": 59, "protected": 1013, "lost": 437, "lost_rate": 0.43139190523198423, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.4954730832799744, "mean_llm_token_ratio": 0.5064165971236685 }, "_seconds": 5.2, "_ms_per_chunk": 88.9 }, "ours+ids@0.5": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5848708565017198, "mean_llm_token_ratio": 0.6607594850156845 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.43344472653878213, "mean_llm_token_ratio": 0.5282570081147547 }, "github_code": { "n": 14, "protected": 277, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.49802537688259274, "mean_llm_token_ratio": 0.5062940001561812 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5233399079552926, "mean_llm_token_ratio": 0.4981949458483754 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4647579279628507, "mean_llm_token_ratio": 0.4987588567685885 }, "fineweb2": { "n": 7, "protected": 38, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.4646904859777982, "mean_llm_token_ratio": 0.4980696352415177 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 0, "lost_rate": 0.0, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.4868151814862343, "mean_llm_token_ratio": 0.5379034319648817 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.33618142678510043, "mean_llm_token_ratio": 0.4981844242713808 }, "_all": { "n": 59, "protected": 1013, "lost": 0, "lost_rate": 0.0, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.48817782525844866, "mean_llm_token_ratio": 0.5369469744482747 }, "_seconds": 5.7, "_ms_per_chunk": 96.0 }, "ours@0.33": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 209, "lost_rate": 0.7206896551724138, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.327296715791273, "mean_llm_token_ratio": 0.31459708951298865 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 151, "lost_rate": 0.722488038277512, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3492269554168825, "mean_llm_token_ratio": 0.3270323309890272 }, "github_code": { "n": 14, "protected": 277, "lost": 134, "lost_rate": 0.48375451263537905, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3234588557857195, "mean_llm_token_ratio": 0.32284723564881385 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.33596318211702825, "mean_llm_token_ratio": 0.3285198555956679 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 18, "lost_rate": 0.225, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.2877951029262656, "mean_llm_token_ratio": 0.3277054998143117 }, "fineweb2": { "n": 7, "protected": 38, "lost": 12, "lost_rate": 0.3157894736842105, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.30053077932409866, "mean_llm_token_ratio": 0.3264086608045815 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 52, "lost_rate": 0.5977011494252874, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.34743765737380344, "mean_llm_token_ratio": 0.2995299597179061 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 10, "lost_rate": 0.35714285714285715, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.18068367830835266, "mean_llm_token_ratio": 0.3247372193024367 }, "_all": { "n": 59, "protected": 1013, "lost": 586, "lost_rate": 0.5784797630799605, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.3118052335847517, "mean_llm_token_ratio": 0.32136154585011445 }, "_seconds": 5.6, "_ms_per_chunk": 94.8 }, "ours(chars)@0.33": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 210, "lost_rate": 0.7241379310344828, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3439364522856813, "mean_llm_token_ratio": 0.3314479793413248 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 162, "lost_rate": 0.7751196172248804, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.28580805596060765, "mean_llm_token_ratio": 0.27243181669152455 }, "github_code": { "n": 14, "protected": 277, "lost": 138, "lost_rate": 0.4981949458483754, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.31775308471956165, "mean_llm_token_ratio": 0.31525592307107037 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3333333333333333, "mean_llm_token_ratio": 0.3249097472924188 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 10, "lost_rate": 0.125, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3365119094824531, "mean_llm_token_ratio": 0.37627862799399553 }, "fineweb2": { "n": 7, "protected": 38, "lost": 11, "lost_rate": 0.2894736842105263, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3573804579329673, "mean_llm_token_ratio": 0.3856092246572405 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 54, "lost_rate": 0.6206896551724138, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.30707900248035375, "mean_llm_token_ratio": 0.27636029308647636 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 6, "lost_rate": 0.21428571428571427, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3374447399733389, "mean_llm_token_ratio": 0.49111124382863514 }, "_all": { "n": 59, "protected": 1013, "lost": 591, "lost_rate": 0.5834155972359328, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.3282637488467005, "mean_llm_token_ratio": 0.3404873669260685 }, "_seconds": 5.0, "_ms_per_chunk": 84.4 }, "ours+ids@0.33": { "swe_rebench_tool": { "n": 11, "protected": 290, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.5750178444782263, "mean_llm_token_ratio": 0.6529765472791863 }, "swe_smith_tool": { "n": 6, "protected": 209, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3742236154767515, "mean_llm_token_ratio": 0.47296373915503437 }, "github_code": { "n": 14, "protected": 277, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.3753987414580236, "mean_llm_token_ratio": 0.38185252441714607 }, "oasst2": { "n": 1, "protected": 4, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.33596318211702825, "mean_llm_token_ratio": 0.3285198555956679 }, "fineweb_edu": { "n": 12, "protected": 80, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.27070723428966886, "mean_llm_token_ratio": 0.3296230704501357 }, "fineweb2": { "n": 7, "protected": 38, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.2918360105579523, "mean_llm_token_ratio": 0.3264086608045815 }, "toucan_mcp": { "n": 5, "protected": 87, "lost": 0, "lost_rate": 0.0, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.41575461058811414, "mean_llm_token_ratio": 0.48518356616090375 }, "swe_rebench_assistant": { "n": 3, "protected": 28, "lost": 0, "lost_rate": 0.0, "json_inputs": 0, "json_still_valid": 0, "mean_char_ratio": 0.17654440314872719, "mean_llm_token_ratio": 0.3341654722089505 }, "_all": { "n": 59, "protected": 1013, "lost": 0, "lost_rate": 0.0, "json_inputs": 1, "json_still_valid": 0, "mean_char_ratio": 0.3739292998715051, "mean_llm_token_ratio": 0.42989375204918706 }, "_seconds": 5.7, "_ms_per_chunk": 97.0 } }, "agreement": { "swe_rebench_tool": { "keep_f1": 0.789, "acc": 0.697 }, "swe_smith_tool": { "keep_f1": 0.834, "acc": 0.746 }, "github_code": { "keep_f1": 0.932, "acc": 0.921 }, "oasst2": { "keep_f1": 0.802, "acc": 0.847 }, "fineweb_edu": { "keep_f1": 0.873, "acc": 0.795 }, "fineweb2": { "keep_f1": 0.762, "acc": 0.624 }, "toucan_mcp": { "keep_f1": 0.684, "acc": 0.592 }, "swe_rebench_assistant": { "keep_f1": 0.833, "acc": 0.731 } } }