{ "cells": { "by_column": { "agent_name": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "structural", "OpenHands/DeepSeek-V4-Pro": "structural", "OpenHands/GLM-4.7": "structural", "OpenHands/GLM-5": "structural", "OpenHands/GLM-5.1": "structural", "OpenHands/GPT-5.2": "structural", "OpenHands/GPT-5.2-Codex": "structural", "OpenHands/GPT-5.4": "structural", "OpenHands/GPT-5.5": "structural", "OpenHands/Gemini-3-Flash": "structural", "OpenHands/Gemini-3-Pro": "structural", "OpenHands/Gemini-3.1-Pro": "structural", "OpenHands/Gemini-3.5-Flash": "structural", "OpenHands/Kimi-K2-Thinking": "structural", "OpenHands/Kimi-K2.5": "structural", "OpenHands/Kimi-K2.6": "structural", "OpenHands/MiniMax-M2.1": "structural", "OpenHands/MiniMax-M2.5": "structural", "OpenHands/MiniMax-M2.7": "structural", "OpenHands/MiniMax-M3": "structural", "OpenHands/Nemotron-3-Nano": "structural", "OpenHands/Nemotron-3-Super": "structural", "OpenHands/Qwen3-Coder-480B": "structural", "OpenHands/Qwen3-Coder-Next": "structural", "OpenHands/Qwen3.5-Flash": "structural", "OpenHands/Qwen3.6-Plus": "structural", "OpenHands/Trinity-Large-Thinking": "structural", "OpenHands/claude-fable-5": "structural", "OpenHands/claude-opus-4-5": "structural", "OpenHands/claude-opus-4-6": "structural", "OpenHands/claude-opus-4-7": "structural", "OpenHands/claude-opus-4-8": "structural", "OpenHands/claude-sonnet-4-5": "structural", "OpenHands/claude-sonnet-4-6": "structural" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 0, "structural": 34 } }, "agent_type": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "structural", "OpenHands/DeepSeek-V4-Pro": "structural", "OpenHands/GLM-4.7": "structural", "OpenHands/GLM-5": "structural", "OpenHands/GLM-5.1": "structural", "OpenHands/GPT-5.2": "structural", "OpenHands/GPT-5.2-Codex": "structural", "OpenHands/GPT-5.4": "structural", "OpenHands/GPT-5.5": "structural", "OpenHands/Gemini-3-Flash": "structural", "OpenHands/Gemini-3-Pro": "structural", "OpenHands/Gemini-3.1-Pro": "structural", "OpenHands/Gemini-3.5-Flash": "structural", "OpenHands/Kimi-K2-Thinking": "structural", "OpenHands/Kimi-K2.5": "structural", "OpenHands/Kimi-K2.6": "structural", "OpenHands/MiniMax-M2.1": "structural", "OpenHands/MiniMax-M2.5": "structural", "OpenHands/MiniMax-M2.7": "structural", "OpenHands/MiniMax-M3": "structural", "OpenHands/Nemotron-3-Nano": "structural", "OpenHands/Nemotron-3-Super": "structural", "OpenHands/Qwen3-Coder-480B": "structural", "OpenHands/Qwen3-Coder-Next": "structural", "OpenHands/Qwen3.5-Flash": "structural", "OpenHands/Qwen3.6-Plus": "structural", "OpenHands/Trinity-Large-Thinking": "structural", "OpenHands/claude-fable-5": "structural", "OpenHands/claude-opus-4-5": "structural", "OpenHands/claude-opus-4-6": "structural", "OpenHands/claude-opus-4-7": "structural", "OpenHands/claude-opus-4-8": "structural", "OpenHands/claude-sonnet-4-5": "structural", "OpenHands/claude-sonnet-4-6": "structural" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 0, "structural": 34 } }, "average_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T2:AC_macro_raw", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:AC_macro_raw", "OpenHands/GLM-4.7": "reconciled_T2:AC_macro_raw", "OpenHands/GLM-5": "divergent", "OpenHands/GLM-5.1": "divergent", "OpenHands/GPT-5.2": "reconciled_T1:AC_macro_raw", "OpenHands/GPT-5.2-Codex": "divergent", "OpenHands/GPT-5.4": "divergent", "OpenHands/GPT-5.5": "divergent", "OpenHands/Gemini-3-Flash": "reconciled_T2:AC_macro_raw", "OpenHands/Gemini-3-Pro": "reconciled_T2:AC_macro_raw", "OpenHands/Gemini-3.1-Pro": "divergent", "OpenHands/Gemini-3.5-Flash": "divergent", "OpenHands/Kimi-K2-Thinking": "reconciled_T1:AC_macro_raw", "OpenHands/Kimi-K2.5": "reconciled_T2:AC_macro_raw", "OpenHands/Kimi-K2.6": "divergent", "OpenHands/MiniMax-M2.1": "divergent", "OpenHands/MiniMax-M2.5": "reconciled_T2:AC_macro_raw", "OpenHands/MiniMax-M2.7": "reconciled_T0:AC_macro_raw", "OpenHands/MiniMax-M3": "divergent", "OpenHands/Nemotron-3-Nano": "reconciled_T1:AC_macro_raw", "OpenHands/Nemotron-3-Super": "reconciled_T1:AC_macro_raw", "OpenHands/Qwen3-Coder-480B": "reconciled_T2:AC_macro_raw", "OpenHands/Qwen3-Coder-Next": "reconciled_T2:AC_macro_raw", "OpenHands/Qwen3.5-Flash": "reconciled_T0:AC_macro_raw", "OpenHands/Qwen3.6-Plus": "reconciled_T0:AC_macro_raw", "OpenHands/Trinity-Large-Thinking": "divergent", "OpenHands/claude-fable-5": "divergent", "OpenHands/claude-opus-4-5": "divergent", "OpenHands/claude-opus-4-6": "divergent", "OpenHands/claude-opus-4-7": "divergent", "OpenHands/claude-opus-4-8": "divergent", "OpenHands/claude-sonnet-4-5": "reconciled_T2:AC_macro_raw", "OpenHands/claude-sonnet-4-6": "divergent" }, "counts": { "divergent": 17, "no_antecedent": 0, "reconciled": 17, "structural": 0 } }, "average_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "average_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "divergent", "OpenHands/DeepSeek-V4-Pro": "divergent", "OpenHands/GLM-4.7": "divergent", "OpenHands/GLM-5": "divergent", "OpenHands/GLM-5.1": "reconciled_T2:AS_macro_raw_Dall", "OpenHands/GPT-5.2": "divergent", "OpenHands/GPT-5.2-Codex": "divergent", "OpenHands/GPT-5.4": "reconciled_T0:AS_macro_pub1dp", "OpenHands/GPT-5.5": "reconciled_T0:AS_macro_pub1dp", "OpenHands/Gemini-3-Flash": "reconciled_T2:AS_macro_raw_Dobs", "OpenHands/Gemini-3-Pro": "divergent", "OpenHands/Gemini-3.1-Pro": "divergent", "OpenHands/Gemini-3.5-Flash": "divergent", "OpenHands/Kimi-K2-Thinking": "divergent", "OpenHands/Kimi-K2.5": "divergent", "OpenHands/Kimi-K2.6": "reconciled_T0:AS_macro_pub1dp", "OpenHands/MiniMax-M2.1": "reconciled_T2:AS_macro_raw_Dobs", "OpenHands/MiniMax-M2.5": "divergent", "OpenHands/MiniMax-M2.7": "divergent", "OpenHands/MiniMax-M3": "reconciled_T2:AS_macro_raw_Dall", "OpenHands/Nemotron-3-Nano": "divergent", "OpenHands/Nemotron-3-Super": "divergent", "OpenHands/Qwen3-Coder-480B": "divergent", "OpenHands/Qwen3-Coder-Next": "divergent", "OpenHands/Qwen3.5-Flash": "reconciled_T2:AS_macro_raw_Dall", "OpenHands/Qwen3.6-Plus": "reconciled_T2:AS_macro_raw_Dall", "OpenHands/Trinity-Large-Thinking": "reconciled_T1:AS_macro_raw_Dall", "OpenHands/claude-fable-5": "reconciled_T0:AS_macro_pub1dp", "OpenHands/claude-opus-4-5": "divergent", "OpenHands/claude-opus-4-6": "divergent", "OpenHands/claude-opus-4-7": "reconciled_T0:AS_macro_pub1dp", "OpenHands/claude-opus-4-8": "reconciled_T0:AS_macro_raw_Dall", "OpenHands/claude-sonnet-4-5": "divergent", "OpenHands/claude-sonnet-4-6": "divergent" }, "counts": { "divergent": 21, "no_antecedent": 0, "reconciled": 13, "structural": 0 } }, "categories_completed": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "structural", "OpenHands/DeepSeek-V4-Pro": "structural", "OpenHands/GLM-4.7": "structural", "OpenHands/GLM-5": "structural", "OpenHands/GLM-5.1": "structural", "OpenHands/GPT-5.2": "structural", "OpenHands/GPT-5.2-Codex": "structural", "OpenHands/GPT-5.4": "structural", "OpenHands/GPT-5.5": "structural", "OpenHands/Gemini-3-Flash": "structural", "OpenHands/Gemini-3-Pro": "structural", "OpenHands/Gemini-3.1-Pro": "structural", "OpenHands/Gemini-3.5-Flash": "structural", "OpenHands/Kimi-K2-Thinking": "structural", "OpenHands/Kimi-K2.5": "structural", "OpenHands/Kimi-K2.6": "structural", "OpenHands/MiniMax-M2.1": "structural", "OpenHands/MiniMax-M2.5": "structural", "OpenHands/MiniMax-M2.7": "structural", "OpenHands/MiniMax-M3": "structural", "OpenHands/Nemotron-3-Nano": "structural", "OpenHands/Nemotron-3-Super": "structural", "OpenHands/Qwen3-Coder-480B": "structural", "OpenHands/Qwen3-Coder-Next": "structural", "OpenHands/Qwen3.5-Flash": "structural", "OpenHands/Qwen3.6-Plus": "structural", "OpenHands/Trinity-Large-Thinking": "structural", "OpenHands/claude-fable-5": "structural", "OpenHands/claude-opus-4-5": "structural", "OpenHands/claude-opus-4-6": "structural", "OpenHands/claude-opus-4-7": "structural", "OpenHands/claude-opus-4-8": "structural", "OpenHands/claude-sonnet-4-5": "structural", "OpenHands/claude-sonnet-4-6": "structural" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 0, "structural": 34 } }, "country": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "frontend_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T2:C_mean_excl", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:C_mean_excl", "OpenHands/GLM-4.7": "reconciled_T1:C_mean_excl", "OpenHands/GLM-5": "reconciled_T1:C_mean_zero", "OpenHands/GLM-5.1": "divergent", "OpenHands/GPT-5.2": "divergent", "OpenHands/GPT-5.2-Codex": "divergent", "OpenHands/GPT-5.4": "divergent", "OpenHands/GPT-5.5": "divergent", "OpenHands/Gemini-3-Flash": "divergent", "OpenHands/Gemini-3-Pro": "reconciled_T2:C_mean_excl", "OpenHands/Gemini-3.1-Pro": "reconciled_T2:C_mean_zero", "OpenHands/Gemini-3.5-Flash": "divergent", "OpenHands/Kimi-K2-Thinking": "divergent", "OpenHands/Kimi-K2.5": "reconciled_T1:C_mean_excl", "OpenHands/Kimi-K2.6": "divergent", "OpenHands/MiniMax-M2.1": "divergent", "OpenHands/MiniMax-M2.5": "reconciled_T1:C_mean_excl", "OpenHands/MiniMax-M2.7": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M3": "divergent", "OpenHands/Nemotron-3-Nano": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Super": "reconciled_T1:C_mean_excl", "OpenHands/Qwen3-Coder-480B": "divergent", "OpenHands/Qwen3-Coder-Next": "reconciled_T2:C_median_excl", "OpenHands/Qwen3.5-Flash": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3.6-Plus": "reconciled_T0:C_mean_excl", "OpenHands/Trinity-Large-Thinking": "divergent", "OpenHands/claude-fable-5": "divergent", "OpenHands/claude-opus-4-5": "divergent", "OpenHands/claude-opus-4-6": "reconciled_T1:C_mean_zero", "OpenHands/claude-opus-4-7": "divergent", "OpenHands/claude-opus-4-8": "divergent", "OpenHands/claude-sonnet-4-5": "divergent", "OpenHands/claude-sonnet-4-6": "reconciled_T1:C_mean_zero" }, "counts": { "divergent": 18, "no_antecedent": 0, "reconciled": 16, "structural": 0 } }, "frontend_logs_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "frontend_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "frontend_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "divergent", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-4.7": "divergent", "OpenHands/GLM-5": "divergent", "OpenHands/GLM-5.1": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2": "divergent", "OpenHands/GPT-5.2-Codex": "divergent", "OpenHands/GPT-5.4": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Flash": "divergent", "OpenHands/Gemini-3-Pro": "divergent", "OpenHands/Gemini-3.1-Pro": "divergent", "OpenHands/Gemini-3.5-Flash": "divergent", "OpenHands/Kimi-K2-Thinking": "divergent", "OpenHands/Kimi-K2.5": "divergent", "OpenHands/Kimi-K2.6": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.1": "divergent", "OpenHands/MiniMax-M2.5": "divergent", "OpenHands/MiniMax-M2.7": "divergent", "OpenHands/MiniMax-M3": "divergent", "OpenHands/Nemotron-3-Nano": "divergent", "OpenHands/Nemotron-3-Super": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-480B": "divergent", "OpenHands/Qwen3-Coder-Next": "divergent", "OpenHands/Qwen3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.6-Plus": "reconciled_T0:S1_pct_Dall", "OpenHands/Trinity-Large-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-fable-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-5": "divergent", "OpenHands/claude-opus-4-6": "divergent", "OpenHands/claude-opus-4-7": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-8": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-5": "divergent", "OpenHands/claude-sonnet-4-6": "divergent" }, "counts": { "divergent": 22, "no_antecedent": 0, "reconciled": 12, "structural": 0 } }, "frontend_visualization_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "greenfield_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T1:C_mean_zero", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:C_mean_excl", "OpenHands/GLM-4.7": "reconciled_T1:C_mean_zero", "OpenHands/GLM-5": "reconciled_T1:C_mean_zero", "OpenHands/GLM-5.1": "reconciled_T1:C_mean_zero", "OpenHands/GPT-5.2": "reconciled_T1:C_mean_zero", "OpenHands/GPT-5.2-Codex": "reconciled_T1:C_mean_zero", "OpenHands/GPT-5.4": "reconciled_T1:C_mean_excl", "OpenHands/GPT-5.5": "reconciled_T1:C_mean_excl", "OpenHands/Gemini-3-Flash": "reconciled_T1:C_mean_zero", "OpenHands/Gemini-3-Pro": "reconciled_T1:C_mean_excl", "OpenHands/Gemini-3.1-Pro": "reconciled_T1:C_mean_zero", "OpenHands/Gemini-3.5-Flash": "reconciled_T1:C_mean_zero", "OpenHands/Kimi-K2-Thinking": "reconciled_T1:C_mean_zero", "OpenHands/Kimi-K2.5": "reconciled_T1:C_mean_zero", "OpenHands/Kimi-K2.6": "reconciled_T1:C_mean_excl", "OpenHands/MiniMax-M2.1": "reconciled_T1:C_mean_excl", "OpenHands/MiniMax-M2.5": "reconciled_T1:C_mean_zero", "OpenHands/MiniMax-M2.7": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M3": "reconciled_T1:C_mean_zero", "OpenHands/Nemotron-3-Nano": "reconciled_T1:C_mean_zero", "OpenHands/Nemotron-3-Super": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3-Coder-480B": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3-Coder-Next": "reconciled_T1:C_mean_excl", "OpenHands/Qwen3.5-Flash": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3.6-Plus": "reconciled_T0:C_mean_excl", "OpenHands/Trinity-Large-Thinking": "reconciled_T1:C_mean_zero", "OpenHands/claude-fable-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-5": "reconciled_T1:C_mean_zero", "OpenHands/claude-opus-4-6": "reconciled_T1:C_mean_zero", "OpenHands/claude-opus-4-7": "reconciled_T1:C_mean_zero", "OpenHands/claude-opus-4-8": "reconciled_T1:C_mean_zero", "OpenHands/claude-sonnet-4-5": "reconciled_T1:C_mean_zero", "OpenHands/claude-sonnet-4-6": "reconciled_T1:C_mean_zero" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 34, "structural": 0 } }, "greenfield_logs_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "greenfield_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "greenfield_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T0:S1_pct_Dall", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-4.7": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5.1": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2-Codex": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.4": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3.1-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.6": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.1": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.7": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M3": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Nano": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Super": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-480B": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-Next": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.6-Plus": "reconciled_T0:S1_pct_Dall", "OpenHands/Trinity-Large-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-fable-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-6": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-7": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-8": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-6": "reconciled_T0:S1_pct_Dall" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 34, "structural": 0 } }, "greenfield_visualization_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "id": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "structural", "OpenHands/DeepSeek-V4-Pro": "structural", "OpenHands/GLM-4.7": "structural", "OpenHands/GLM-5": "structural", "OpenHands/GLM-5.1": "structural", "OpenHands/GPT-5.2": "structural", "OpenHands/GPT-5.2-Codex": "structural", "OpenHands/GPT-5.4": "structural", "OpenHands/GPT-5.5": "structural", "OpenHands/Gemini-3-Flash": "structural", "OpenHands/Gemini-3-Pro": "structural", "OpenHands/Gemini-3.1-Pro": "structural", "OpenHands/Gemini-3.5-Flash": "structural", "OpenHands/Kimi-K2-Thinking": "structural", "OpenHands/Kimi-K2.5": "structural", "OpenHands/Kimi-K2.6": "structural", "OpenHands/MiniMax-M2.1": "structural", "OpenHands/MiniMax-M2.5": "structural", "OpenHands/MiniMax-M2.7": "structural", "OpenHands/MiniMax-M3": "structural", "OpenHands/Nemotron-3-Nano": "structural", "OpenHands/Nemotron-3-Super": "structural", "OpenHands/Qwen3-Coder-480B": "structural", "OpenHands/Qwen3-Coder-Next": "structural", "OpenHands/Qwen3.5-Flash": "structural", "OpenHands/Qwen3.6-Plus": "structural", "OpenHands/Trinity-Large-Thinking": "structural", "OpenHands/claude-fable-5": "structural", "OpenHands/claude-opus-4-5": "structural", "OpenHands/claude-opus-4-6": "structural", "OpenHands/claude-opus-4-7": "structural", "OpenHands/claude-opus-4-8": "structural", "OpenHands/claude-sonnet-4-5": "structural", "OpenHands/claude-sonnet-4-6": "structural" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 0, "structural": 34 } }, "information_gathering_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T1:C_mean_zero", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:C_mean_excl", "OpenHands/GLM-4.7": "reconciled_T2:C_mean_excl", "OpenHands/GLM-5": "reconciled_T1:C_mean_zero", "OpenHands/GLM-5.1": "reconciled_T2:C_mean_excl", "OpenHands/GPT-5.2": "reconciled_T1:C_mean_zero", "OpenHands/GPT-5.2-Codex": "divergent", "OpenHands/GPT-5.4": "divergent", "OpenHands/GPT-5.5": "reconciled_T2:C_sum_over_resolved", "OpenHands/Gemini-3-Flash": "reconciled_T1:C_mean_zero", "OpenHands/Gemini-3-Pro": "divergent", "OpenHands/Gemini-3.1-Pro": "divergent", "OpenHands/Gemini-3.5-Flash": "reconciled_T2:C_mean_zero", "OpenHands/Kimi-K2-Thinking": "reconciled_T2:C_mean_excl", "OpenHands/Kimi-K2.5": "reconciled_T0:C_mean_excl", "OpenHands/Kimi-K2.6": "reconciled_T2:C_mean_excl", "OpenHands/MiniMax-M2.1": "reconciled_T2:C_mean_excl", "OpenHands/MiniMax-M2.5": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M2.7": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M3": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Nano": "reconciled_T1:C_mean_excl", "OpenHands/Nemotron-3-Super": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3-Coder-480B": "reconciled_T2:C_mean_excl", "OpenHands/Qwen3-Coder-Next": "reconciled_T2:C_median_excl", "OpenHands/Qwen3.5-Flash": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3.6-Plus": "reconciled_T0:C_mean_excl", "OpenHands/Trinity-Large-Thinking": "divergent", "OpenHands/claude-fable-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-6": "reconciled_T2:C_mean_excl", "OpenHands/claude-opus-4-7": "reconciled_T2:C_sum_over_resolved", "OpenHands/claude-opus-4-8": "reconciled_T2:C_mean_excl", "OpenHands/claude-sonnet-4-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-sonnet-4-6": "reconciled_T1:C_mean_excl" }, "counts": { "divergent": 5, "no_antecedent": 0, "reconciled": 29, "structural": 0 } }, "information_gathering_logs_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "information_gathering_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "information_gathering_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T0:S1_pct_Dall", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-4.7": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5.1": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2-Codex": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.4": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3.1-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3.5-Flash": "divergent", "OpenHands/Kimi-K2-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.6": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.1": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.7": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M3": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Nano": "divergent", "OpenHands/Nemotron-3-Super": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-480B": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-Next": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.6-Plus": "reconciled_T0:S1_pct_Dall", "OpenHands/Trinity-Large-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-fable-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-6": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-7": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-8": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-6": "reconciled_T0:S1_pct_Dall" }, "counts": { "divergent": 2, "no_antecedent": 0, "reconciled": 32, "structural": 0 } }, "information_gathering_visualization_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "issue_resolution_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T2:C_mean_excl", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:C_mean_excl", "OpenHands/GLM-4.7": "reconciled_T2:C_mean_excl", "OpenHands/GLM-5": "reconciled_T2:C_mean_excl", "OpenHands/GLM-5.1": "reconciled_T1:C_mean_excl", "OpenHands/GPT-5.2": "reconciled_T2:C_mean_excl", "OpenHands/GPT-5.2-Codex": "reconciled_T2:C_mean_excl", "OpenHands/GPT-5.4": "reconciled_T1:C_mean_excl", "OpenHands/GPT-5.5": "reconciled_T2:C_mean_excl", "OpenHands/Gemini-3-Flash": "reconciled_T2:C_mean_excl", "OpenHands/Gemini-3-Pro": "reconciled_T1:C_mean_zero", "OpenHands/Gemini-3.1-Pro": "reconciled_T2:C_mean_excl", "OpenHands/Gemini-3.5-Flash": "reconciled_T2:C_mean_excl", "OpenHands/Kimi-K2-Thinking": "divergent", "OpenHands/Kimi-K2.5": "reconciled_T1:C_mean_excl", "OpenHands/Kimi-K2.6": "reconciled_T2:C_mean_excl", "OpenHands/MiniMax-M2.1": "reconciled_T2:C_mean_excl", "OpenHands/MiniMax-M2.5": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M2.7": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M3": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Nano": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Super": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3-Coder-480B": "reconciled_T2:C_mean_excl", "OpenHands/Qwen3-Coder-Next": "reconciled_T1:C_mean_excl", "OpenHands/Qwen3.5-Flash": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3.6-Plus": "reconciled_T0:C_mean_excl", "OpenHands/Trinity-Large-Thinking": "reconciled_T1:C_mean_excl", "OpenHands/claude-fable-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-5": "reconciled_T2:C_mean_excl", "OpenHands/claude-opus-4-6": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-7": "divergent", "OpenHands/claude-opus-4-8": "reconciled_T2:C_mean_excl", "OpenHands/claude-sonnet-4-5": "reconciled_T2:C_mean_excl", "OpenHands/claude-sonnet-4-6": "reconciled_T1:C_mean_excl" }, "counts": { "divergent": 2, "no_antecedent": 0, "reconciled": 32, "structural": 0 } }, "issue_resolution_logs_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "issue_resolution_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "issue_resolution_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T0:S1_pct_Dall", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-4.7": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5": "reconciled_T0:S1_pct_Dall", "OpenHands/GLM-5.1": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.2-Codex": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.4": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Pro": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3.1-Pro": "reconciled_T2:S1_pct_Dall", "OpenHands/Gemini-3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.6": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.1": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.7": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M3": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Nano": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Super": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-480B": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3-Coder-Next": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.5-Flash": "reconciled_T0:S1_pct_Dall", "OpenHands/Qwen3.6-Plus": "reconciled_T0:S1_pct_Dall", "OpenHands/Trinity-Large-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-fable-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-6": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-7": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-8": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-6": "reconciled_T0:S1_pct_Dall" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 34, "structural": 0 } }, "issue_resolution_visualization_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "language_model": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T0:U_unique", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:U_unique", "OpenHands/GLM-4.7": "reconciled_T0:U_unique", "OpenHands/GLM-5": "reconciled_T0:U_unique", "OpenHands/GLM-5.1": "reconciled_T0:U_unique", "OpenHands/GPT-5.2": "reconciled_T0:U_unique", "OpenHands/GPT-5.2-Codex": "reconciled_T0:U_unique", "OpenHands/GPT-5.4": "reconciled_T0:U_unique", "OpenHands/GPT-5.5": "reconciled_T0:U_unique", "OpenHands/Gemini-3-Flash": "reconciled_T0:U_unique", "OpenHands/Gemini-3-Pro": "reconciled_T0:U_unique", "OpenHands/Gemini-3.1-Pro": "reconciled_T0:U_unique", "OpenHands/Gemini-3.5-Flash": "reconciled_T0:U_unique", "OpenHands/Kimi-K2-Thinking": "reconciled_T0:U_unique", "OpenHands/Kimi-K2.5": "reconciled_T0:U_unique", "OpenHands/Kimi-K2.6": "reconciled_T0:U_unique", "OpenHands/MiniMax-M2.1": "reconciled_T0:U_unique", "OpenHands/MiniMax-M2.5": "reconciled_T0:U_unique", "OpenHands/MiniMax-M2.7": "reconciled_T0:U_unique", "OpenHands/MiniMax-M3": "reconciled_T0:U_unique", "OpenHands/Nemotron-3-Nano": "reconciled_T0:U_unique", "OpenHands/Nemotron-3-Super": "reconciled_T0:U_unique", "OpenHands/Qwen3-Coder-480B": "reconciled_T0:U_unique", "OpenHands/Qwen3-Coder-Next": "reconciled_T0:U_unique", "OpenHands/Qwen3.5-Flash": "reconciled_T0:U_unique", "OpenHands/Qwen3.6-Plus": "reconciled_T0:U_unique", "OpenHands/Trinity-Large-Thinking": "reconciled_T0:U_unique", "OpenHands/claude-fable-5": "reconciled_T0:U_unique", "OpenHands/claude-opus-4-5": "reconciled_T0:U_unique", "OpenHands/claude-opus-4-6": "reconciled_T0:U_unique", "OpenHands/claude-opus-4-7": "reconciled_T0:U_unique", "OpenHands/claude-opus-4-8": "reconciled_T0:U_unique", "OpenHands/claude-sonnet-4-5": "reconciled_T0:U_unique", "OpenHands/claude-sonnet-4-6": "reconciled_T0:U_unique" }, "counts": { "divergent": 0, "no_antecedent": 0, "reconciled": 34, "structural": 0 } }, "openness": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "release_date": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "sdk_version": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "supports_vision": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "testing_cost": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "reconciled_T2:C_mean_excl", "OpenHands/DeepSeek-V4-Pro": "reconciled_T0:C_mean_excl", "OpenHands/GLM-4.7": "reconciled_T1:C_mean_excl", "OpenHands/GLM-5": "reconciled_T1:C_mean_excl", "OpenHands/GLM-5.1": "reconciled_T1:C_mean_excl", "OpenHands/GPT-5.2": "reconciled_T2:C_mean_excl", "OpenHands/GPT-5.2-Codex": "reconciled_T2:C_mean_excl", "OpenHands/GPT-5.4": "reconciled_T1:C_mean_excl", "OpenHands/GPT-5.5": "reconciled_T1:C_mean_excl", "OpenHands/Gemini-3-Flash": "reconciled_T1:C_mean_excl", "OpenHands/Gemini-3-Pro": "divergent", "OpenHands/Gemini-3.1-Pro": "reconciled_T2:C_mean_excl", "OpenHands/Gemini-3.5-Flash": "reconciled_T1:C_mean_excl", "OpenHands/Kimi-K2-Thinking": "divergent", "OpenHands/Kimi-K2.5": "reconciled_T1:C_mean_excl", "OpenHands/Kimi-K2.6": "reconciled_T1:C_mean_excl", "OpenHands/MiniMax-M2.1": "reconciled_T2:C_mean_excl", "OpenHands/MiniMax-M2.5": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M2.7": "reconciled_T0:C_mean_excl", "OpenHands/MiniMax-M3": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Nano": "reconciled_T2:C_mean_excl", "OpenHands/Nemotron-3-Super": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3-Coder-480B": "divergent", "OpenHands/Qwen3-Coder-Next": "reconciled_T2:C_mean_excl", "OpenHands/Qwen3.5-Flash": "reconciled_T0:C_mean_excl", "OpenHands/Qwen3.6-Plus": "reconciled_T0:C_mean_excl", "OpenHands/Trinity-Large-Thinking": "reconciled_T2:C_mean_excl", "OpenHands/claude-fable-5": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-5": "reconciled_T2:C_mean_excl", "OpenHands/claude-opus-4-6": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-7": "reconciled_T1:C_mean_excl", "OpenHands/claude-opus-4-8": "reconciled_T1:C_mean_excl", "OpenHands/claude-sonnet-4-5": "reconciled_T2:C_mean_excl", "OpenHands/claude-sonnet-4-6": "reconciled_T2:C_mean_excl" }, "counts": { "divergent": 3, "no_antecedent": 0, "reconciled": 31, "structural": 0 } }, "testing_logs_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "testing_runtime": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } }, "testing_score": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "divergent", "OpenHands/DeepSeek-V4-Pro": "divergent", "OpenHands/GLM-4.7": "divergent", "OpenHands/GLM-5": "divergent", "OpenHands/GLM-5.1": "reconciled_T2:S1_pct_Dall", "OpenHands/GPT-5.2": "divergent", "OpenHands/GPT-5.2-Codex": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.4": "reconciled_T0:S1_pct_Dall", "OpenHands/GPT-5.5": "reconciled_T0:S1_pct_Dall", "OpenHands/Gemini-3-Flash": "reconciled_T2:S1_pct_Dall", "OpenHands/Gemini-3-Pro": "reconciled_T2:S1_pct_Dall", "OpenHands/Gemini-3.1-Pro": "reconciled_T2:S1_pct_Dall", "OpenHands/Gemini-3.5-Flash": "reconciled_T2:S1_pct_Dall", "OpenHands/Kimi-K2-Thinking": "reconciled_T0:S1_pct_Dall", "OpenHands/Kimi-K2.5": "reconciled_T2:S1_pct_Dall", "OpenHands/Kimi-K2.6": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.1": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.5": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M2.7": "reconciled_T0:S1_pct_Dall", "OpenHands/MiniMax-M3": "reconciled_T0:S1_pct_Dall", "OpenHands/Nemotron-3-Nano": "divergent", "OpenHands/Nemotron-3-Super": "divergent", "OpenHands/Qwen3-Coder-480B": "divergent", "OpenHands/Qwen3-Coder-Next": "divergent", "OpenHands/Qwen3.5-Flash": "divergent", "OpenHands/Qwen3.6-Plus": "divergent", "OpenHands/Trinity-Large-Thinking": "reconciled_T2:S1_pct_Dall", "OpenHands/claude-fable-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-5": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-6": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-7": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-opus-4-8": "reconciled_T0:S1_pct_Dall", "OpenHands/claude-sonnet-4-5": "divergent", "OpenHands/claude-sonnet-4-6": "reconciled_T0:S1_pct_Dall" }, "counts": { "divergent": 12, "no_antecedent": 0, "reconciled": 22, "structural": 0 } }, "testing_visualization_url": { "cells": { "OpenHands/DeepSeek-V3.2-Reasoner": "no_antecedent", "OpenHands/DeepSeek-V4-Pro": "no_antecedent", "OpenHands/GLM-4.7": "no_antecedent", "OpenHands/GLM-5": "no_antecedent", "OpenHands/GLM-5.1": "no_antecedent", "OpenHands/GPT-5.2": "no_antecedent", "OpenHands/GPT-5.2-Codex": "no_antecedent", "OpenHands/GPT-5.4": "no_antecedent", "OpenHands/GPT-5.5": "no_antecedent", "OpenHands/Gemini-3-Flash": "no_antecedent", "OpenHands/Gemini-3-Pro": "no_antecedent", "OpenHands/Gemini-3.1-Pro": "no_antecedent", "OpenHands/Gemini-3.5-Flash": "no_antecedent", "OpenHands/Kimi-K2-Thinking": "no_antecedent", "OpenHands/Kimi-K2.5": "no_antecedent", "OpenHands/Kimi-K2.6": "no_antecedent", "OpenHands/MiniMax-M2.1": "no_antecedent", "OpenHands/MiniMax-M2.5": "no_antecedent", "OpenHands/MiniMax-M2.7": "no_antecedent", "OpenHands/MiniMax-M3": "no_antecedent", "OpenHands/Nemotron-3-Nano": "no_antecedent", "OpenHands/Nemotron-3-Super": "no_antecedent", "OpenHands/Qwen3-Coder-480B": "no_antecedent", "OpenHands/Qwen3-Coder-Next": "no_antecedent", "OpenHands/Qwen3.5-Flash": "no_antecedent", "OpenHands/Qwen3.6-Plus": "no_antecedent", "OpenHands/Trinity-Large-Thinking": "no_antecedent", "OpenHands/claude-fable-5": "no_antecedent", "OpenHands/claude-opus-4-5": "no_antecedent", "OpenHands/claude-opus-4-6": "no_antecedent", "OpenHands/claude-opus-4-7": "no_antecedent", "OpenHands/claude-opus-4-8": "no_antecedent", "OpenHands/claude-sonnet-4-5": "no_antecedent", "OpenHands/claude-sonnet-4-6": "no_antecedent" }, "counts": { "divergent": 0, "no_antecedent": 34, "reconciled": 0, "structural": 0 } } }, "totals": { "divergent": 102, "no_antecedent": 714, "reconciled": 340, "reconciled_by_tier": { "T0": 199, "T1": 71, "T2": 70 }, "structural": 136, "total": 1292 } }, "colmap": { "C1_ambiguous_benchmarks": [], "C1_pairs": [ [ "commit0", "Greenfield" ], [ "gaia", "Information Gathering" ], [ "swe-bench", "Issue Resolution" ], [ "swe-bench-multimodal", "Frontend" ], [ "swt-bench", "Testing" ] ], "agrees_with_C2_README_and_C3_code": { "commit0": true, "gaia": true, "swe-bench": true, "swe-bench-multimodal": true, "swt-bench": true }, "mapping_used": { "commit0": "greenfield", "gaia": "information_gathering", "swe-bench": "issue_resolution", "swe-bench-multimodal": "frontend", "swt-bench": "testing" } }, "columns": { "agent_name": { "dp": null, "family": "categorical", "max_abs_delta": null, "max_abs_rows": [], "policies": { "U_major": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "OpenHands", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "OpenHands", null, 0 ], "OpenHands/GLM-4.7": [ "OpenHands", null, 0 ], "OpenHands/GLM-5": [ "OpenHands", null, 0 ], "OpenHands/GLM-5.1": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.4": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.5": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Flash": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.5": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.6": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.1": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.5": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.7": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M3": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Super": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "OpenHands", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "OpenHands", null, 0 ], "OpenHands/claude-fable-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-6": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-7": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-8": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "OpenHands", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 }, "U_unique": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "OpenHands", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "OpenHands", null, 0 ], "OpenHands/GLM-4.7": [ "OpenHands", null, 0 ], "OpenHands/GLM-5": [ "OpenHands", null, 0 ], "OpenHands/GLM-5.1": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.4": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.5": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Flash": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.5": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.6": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.1": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.5": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.7": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M3": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Super": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "OpenHands", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "OpenHands", null, 0 ], "OpenHands/claude-fable-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-6": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-7": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-8": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "OpenHands", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "U_unique", "published_range": null, "zero_variance": true }, "agent_type": { "dp": null, "family": "categorical", "max_abs_delta": null, "max_abs_rows": [], "policies": { "U_major": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "OpenHands", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "OpenHands", null, 0 ], "OpenHands/GLM-4.7": [ "OpenHands", null, 0 ], "OpenHands/GLM-5": [ "OpenHands", null, 0 ], "OpenHands/GLM-5.1": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.4": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.5": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Flash": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.5": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.6": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.1": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.5": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.7": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M3": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Super": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "OpenHands", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "OpenHands", null, 0 ], "OpenHands/claude-fable-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-6": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-7": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-8": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "OpenHands", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 }, "U_unique": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "OpenHands", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "OpenHands", null, 0 ], "OpenHands/GLM-4.7": [ "OpenHands", null, 0 ], "OpenHands/GLM-5": [ "OpenHands", null, 0 ], "OpenHands/GLM-5.1": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.4": [ "OpenHands", null, 0 ], "OpenHands/GPT-5.5": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Flash": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "OpenHands", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.5": [ "OpenHands", null, 0 ], "OpenHands/Kimi-K2.6": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.1": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.5": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M2.7": [ "OpenHands", null, 0 ], "OpenHands/MiniMax-M3": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "OpenHands", null, 0 ], "OpenHands/Nemotron-3-Super": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "OpenHands", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "OpenHands", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "OpenHands", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "OpenHands", null, 0 ], "OpenHands/claude-fable-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-6": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-7": [ "OpenHands", null, 0 ], "OpenHands/claude-opus-4-8": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "OpenHands", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "OpenHands", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "U_unique", "published_range": null, "zero_variance": true }, "average_cost": { "dp": 4, "family": "cost", "max_abs_delta": 0.594564538, "max_abs_rows": [ "OpenHands/Gemini-3.5-Flash" ], "policies": { "AC_macro_4dp": { "div": 17, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.2326, 0.0126, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.15331999999999998, 2e-05, 0 ], "OpenHands/GLM-4.7": [ 0.7196400000000001, 0.04374, 2 ], "OpenHands/GLM-5": [ 1.2727400000000002, 0.24474, "div" ], "OpenHands/GLM-5.1": [ 2.5793799999999996, -0.42062, "div" ], "OpenHands/GPT-5.2": [ 1.8770400000000003, 0.00304, 1 ], "OpenHands/GPT-5.2-Codex": [ 2.21146, 0.08746, "div" ], "OpenHands/GPT-5.4": [ 1.5271000000000001, 0.0871, "div" ], "OpenHands/GPT-5.5": [ 2.52418, 0.21418, "div" ], "OpenHands/Gemini-3-Flash": [ 0.93362, 0.02962, 2 ], "OpenHands/Gemini-3-Pro": [ 1.43364, 0.01364, 2 ], "OpenHands/Gemini-3.1-Pro": [ 2.01804, 0.47604, "div" ], "OpenHands/Gemini-3.5-Flash": [ 3.6125599999999998, 0.59456, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.3733600000000001, -0.00464, 1 ], "OpenHands/Kimi-K2.5": [ 0.83454, 0.01644, 2 ], "OpenHands/Kimi-K2.6": [ 0.7777000000000001, 0.0617, "div" ], "OpenHands/MiniMax-M2.1": [ 0.53244, -0.12506, "div" ], "OpenHands/MiniMax-M2.5": [ 0.34158, 0.00688, 2 ], "OpenHands/MiniMax-M2.7": [ 0.36094, 4e-05, 0 ], "OpenHands/MiniMax-M3": [ 0.40506000000000003, 0.08506, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.18292, 0.00292, 1 ], "OpenHands/Nemotron-3-Super": [ 0.5811999999999999, 0.0005, 1 ], "OpenHands/Qwen3-Coder-480B": [ 0.8754199999999999, -0.04658, 2 ], "OpenHands/Qwen3-Coder-Next": [ 1.0595999999999999, 0.0296, 2 ], "OpenHands/Qwen3.5-Flash": [ 2.2339, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.7540000000000004, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.8869999999999999, 0.131, "div" ], "OpenHands/claude-fable-5": [ 5.97616, 0.43816, "div" ], "OpenHands/claude-opus-4-5": [ 3.45414, 0.37414, "div" ], "OpenHands/claude-opus-4-6": [ 2.5734, 0.2334, "div" ], "OpenHands/claude-opus-4-7": [ 2.80414, 0.49214, "div" ], "OpenHands/claude-opus-4-8": [ 2.7394600000000002, 0.28146, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.64788, 0.01588, 2 ], "OpenHands/claude-sonnet-4-6": [ 2.5330400000000006, 0.32704, "div" ] }, "t0": 4, "t1": 4, "t2": 9, "undef": 0 }, "AC_macro_all5_zero": { "div": 17, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.23260729260753393, 0.012607293, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.15332, 2e-05, 0 ], "OpenHands/GLM-4.7": [ 0.7196376865488274, 0.043737687, 2 ], "OpenHands/GLM-5": [ 1.2727509060928428, 0.244750906, "div" ], "OpenHands/GLM-5.1": [ 2.579355005211931, -0.420644995, "div" ], "OpenHands/GPT-5.2": [ 1.8770355727779449, 0.003035573, 1 ], "OpenHands/GPT-5.2-Codex": [ 2.2114878410533607, 0.087487841, "div" ], "OpenHands/GPT-5.4": [ 1.5271248250284257, 0.087124825, "div" ], "OpenHands/GPT-5.5": [ 2.524188913771051, 0.214188914, "div" ], "OpenHands/Gemini-3-Flash": [ 0.9336452223133591, 0.029645222, 2 ], "OpenHands/Gemini-3-Pro": [ 1.433646679688666, 0.01364668, 2 ], "OpenHands/Gemini-3.1-Pro": [ 2.0180364021662767, 0.476036402, "div" ], "OpenHands/Gemini-3.5-Flash": [ 3.61256453849512, 0.594564538, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.3733695564614161, -0.004630444, 1 ], "OpenHands/Kimi-K2.5": [ 0.8345180260360359, 0.016418026, 2 ], "OpenHands/Kimi-K2.6": [ 0.777681277298955, 0.061681277, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5324477935277362, -0.125052206, "div" ], "OpenHands/MiniMax-M2.5": [ 0.3415845361866956, 0.006884536, 2 ], "OpenHands/MiniMax-M2.7": [ 0.36094, 4e-05, 0 ], "OpenHands/MiniMax-M3": [ 0.40506288951455627, 0.08506289, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.1829109890410207, 0.002910989, 1 ], "OpenHands/Nemotron-3-Super": [ 0.5812001438930102, 0.000500144, 1 ], "OpenHands/Qwen3-Coder-480B": [ 0.8754111083793029, -0.046588892, 2 ], "OpenHands/Qwen3-Coder-Next": [ 1.0595994668184352, 0.029599467, 2 ], "OpenHands/Qwen3.5-Flash": [ 2.2339000000000007, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.7540000000000004, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.8870050712285986, 0.131005071, "div" ], "OpenHands/claude-fable-5": [ 5.976169196456131, 0.438169196, "div" ], "OpenHands/claude-opus-4-5": [ 3.4541518242762543, 0.374151824, "div" ], "OpenHands/claude-opus-4-6": [ 2.573416101246537, 0.233416101, "div" ], "OpenHands/claude-opus-4-7": [ 2.804163178894942, 0.492163179, "div" ], "OpenHands/claude-opus-4-8": [ 2.7394598432790627, 0.281459843, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.6478899133062406, 0.015889913, 2 ], "OpenHands/claude-sonnet-4-6": [ 2.533023689322259, 0.327023689, "div" ] }, "t0": 4, "t1": 4, "t2": 9, "undef": 0 }, "AC_macro_raw": { "div": 17, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.23260729260753393, 0.012607293, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.15332, 2e-05, 0 ], "OpenHands/GLM-4.7": [ 0.7196376865488274, 0.043737687, 2 ], "OpenHands/GLM-5": [ 1.2727509060928428, 0.244750906, "div" ], "OpenHands/GLM-5.1": [ 2.579355005211931, -0.420644995, "div" ], "OpenHands/GPT-5.2": [ 1.8770355727779449, 0.003035573, 1 ], "OpenHands/GPT-5.2-Codex": [ 2.2114878410533607, 0.087487841, "div" ], "OpenHands/GPT-5.4": [ 1.5271248250284257, 0.087124825, "div" ], "OpenHands/GPT-5.5": [ 2.524188913771051, 0.214188914, "div" ], "OpenHands/Gemini-3-Flash": [ 0.9336452223133591, 0.029645222, 2 ], "OpenHands/Gemini-3-Pro": [ 1.433646679688666, 0.01364668, 2 ], "OpenHands/Gemini-3.1-Pro": [ 2.0180364021662767, 0.476036402, "div" ], "OpenHands/Gemini-3.5-Flash": [ 3.61256453849512, 0.594564538, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.3733695564614161, -0.004630444, 1 ], "OpenHands/Kimi-K2.5": [ 0.8345180260360359, 0.016418026, 2 ], "OpenHands/Kimi-K2.6": [ 0.777681277298955, 0.061681277, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5324477935277362, -0.125052206, "div" ], "OpenHands/MiniMax-M2.5": [ 0.3415845361866956, 0.006884536, 2 ], "OpenHands/MiniMax-M2.7": [ 0.36094, 4e-05, 0 ], "OpenHands/MiniMax-M3": [ 0.40506288951455627, 0.08506289, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.1829109890410207, 0.002910989, 1 ], "OpenHands/Nemotron-3-Super": [ 0.5812001438930102, 0.000500144, 1 ], "OpenHands/Qwen3-Coder-480B": [ 0.8754111083793029, -0.046588892, 2 ], "OpenHands/Qwen3-Coder-Next": [ 1.0595994668184352, 0.029599467, 2 ], "OpenHands/Qwen3.5-Flash": [ 2.2339000000000007, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.7540000000000004, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.8870050712285986, 0.131005071, "div" ], "OpenHands/claude-fable-5": [ 5.976169196456131, 0.438169196, "div" ], "OpenHands/claude-opus-4-5": [ 3.4541518242762543, 0.374151824, "div" ], "OpenHands/claude-opus-4-6": [ 2.573416101246537, 0.233416101, "div" ], "OpenHands/claude-opus-4-7": [ 2.804163178894942, 0.492163179, "div" ], "OpenHands/claude-opus-4-8": [ 2.7394598432790627, 0.281459843, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.6478899133062406, 0.015889913, 2 ], "OpenHands/claude-sonnet-4-6": [ 2.533023689322259, 0.327023689, "div" ] }, "t0": 4, "t1": 4, "t2": 9, "undef": 0 }, "AC_micro_excl": { "div": 33, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.13524433690192791, -0.084755663, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.07561353637901863, -0.077686464, "div" ], "OpenHands/GLM-4.7": [ 0.44621924407064817, -0.229680756, "div" ], "OpenHands/GLM-5": [ 0.9073649562152126, -0.120635044, "div" ], "OpenHands/GLM-5.1": [ 1.2832388604555591, -1.71676114, "div" ], "OpenHands/GPT-5.2": [ 0.8775043439519464, -0.996495656, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.9883237664191422, -1.135676234, "div" ], "OpenHands/GPT-5.4": [ 0.6833131614406798, -0.756686839, "div" ], "OpenHands/GPT-5.5": [ 1.3734319020442947, -0.936568098, "div" ], "OpenHands/Gemini-3-Flash": [ 0.41227299121848743, -0.491727009, "div" ], "OpenHands/Gemini-3-Pro": [ 0.9697258350498337, -0.450274165, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.7909197913043462, 0.248919791, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.6911799626833466, -1.326820037, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.5082832945240077, 0.130283295, "div" ], "OpenHands/Kimi-K2.5": [ 0.5200513171940305, -0.298048683, "div" ], "OpenHands/Kimi-K2.6": [ 0.5360499501526715, -0.17995005, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5700080395555547, -0.08749196, "div" ], "OpenHands/MiniMax-M2.5": [ 0.2703463851582014, -0.064353615, "div" ], "OpenHands/MiniMax-M2.7": [ 0.19082557755775578, -0.170074422, "div" ], "OpenHands/MiniMax-M3": [ 0.1854160555269923, -0.134583944, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.20891781402985107, 0.028917814, 2 ], "OpenHands/Nemotron-3-Super": [ 0.39580623318373437, -0.184893767, "div" ], "OpenHands/Qwen3-Coder-480B": [ 1.0564183069158077, 0.134418307, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.1316265795862066, 0.10162658, "div" ], "OpenHands/Qwen3.5-Flash": [ 1.798788794567064, -0.435111205, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.68219541206457, -1.071804588, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.7037222973788921, -0.052277703, "div" ], "OpenHands/claude-fable-5": [ 2.794489766920472, -2.743510233, "div" ], "OpenHands/claude-opus-4-5": [ 1.6156165295587, -1.46438347, "div" ], "OpenHands/claude-opus-4-6": [ 0.8366087586920534, -1.503391241, "div" ], "OpenHands/claude-opus-4-7": [ 1.2867689944444454, -1.025231006, "div" ], "OpenHands/claude-opus-4-8": [ 1.0068540947033897, -1.451145905, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.1215646478476824, -0.510435352, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.070291526845638, -1.135708473, "div" ] }, "t0": 0, "t1": 0, "t2": 1, "undef": 0 }, "AC_micro_zero": { "div": 32, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.13345450283209262, -0.086545497, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.07561353637901863, -0.077686464, "div" ], "OpenHands/GLM-4.7": [ 0.44029434124481387, -0.235605659, "div" ], "OpenHands/GLM-5": [ 0.8756843534467317, -0.152315647, "div" ], "OpenHands/GLM-5.1": [ 1.2604402004982267, -1.7395598, "div" ], "OpenHands/GPT-5.2": [ 0.8753287133471068, -0.998671287, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.9850727013980266, -1.138927299, "div" ], "OpenHands/GPT-5.4": [ 0.6821569631979714, -0.757843037, "div" ], "OpenHands/GPT-5.5": [ 1.364136254653132, -0.945863745, "div" ], "OpenHands/Gemini-3-Flash": [ 0.4041226190691928, -0.499877381, "div" ], "OpenHands/Gemini-3-Pro": [ 0.9617379780889619, -0.458262022, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.761463873684209, 0.219463874, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.6709953765984642, -1.347004623, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.4723127225328925, 0.094312723, "div" ], "OpenHands/Kimi-K2.5": [ 0.5187682958522689, -0.299331704, "div" ], "OpenHands/Kimi-K2.6": [ 0.5346894172842637, -0.181310583, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5695392829440781, -0.087960717, "div" ], "OpenHands/MiniMax-M2.5": [ 0.2701240608283016, -0.064575939, "div" ], "OpenHands/MiniMax-M2.7": [ 0.19019786184210527, -0.170702138, "div" ], "OpenHands/MiniMax-M3": [ 0.18321806672311602, -0.136781933, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.20691946972173947, 0.02691947, 2 ], "OpenHands/Nemotron-3-Super": [ 0.3948016488355523, -0.185898351, "div" ], "OpenHands/Qwen3-Coder-480B": [ 1.0264364851836396, 0.104436485, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.0795121976315787, 0.049512198, 2 ], "OpenHands/Qwen3.5-Flash": [ 1.7927015228426406, -0.441198477, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.6750795262267333, -1.078920474, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.6882427882994918, -0.067757212, "div" ], "OpenHands/claude-fable-5": [ 2.794489766920472, -2.743510233, "div" ], "OpenHands/claude-opus-4-5": [ 1.5970003720164598, -1.482999628, "div" ], "OpenHands/claude-opus-4-6": [ 0.8311047537006583, -1.508895246, "div" ], "OpenHands/claude-opus-4-7": [ 1.2737053498307962, -1.03829465, "div" ], "OpenHands/claude-opus-4-8": [ 1.0051504498730963, -1.45284955, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.1169415454245675, -0.515058455, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.0491673519736846, -1.156832648, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 0 } }, "primary": "AC_macro_raw", "published_range": [ 0.1533, 5.538 ], "zero_variance": false }, "average_score": { "dp": 2, "family": "score", "max_abs_delta": 2.752574296, "max_abs_rows": [ "OpenHands/Gemini-3.1-Pro" ], "policies": { "AS_macro_all5_zero": { "div": 23, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 43.998115036027215, -1.681884964, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 39.232391102987485, -1.427608897, "div" ], "OpenHands/GLM-4.7": [ 41.05123535325375, -1.208764647, "div" ], "OpenHands/GLM-5": [ 50.49956674885523, 1.059566749, "div" ], "OpenHands/GLM-5.1": [ 58.13903125810475, -0.100968742, 2 ], "OpenHands/GPT-5.2": [ 56.97633132537665, -1.863668675, "div" ], "OpenHands/GPT-5.2-Codex": [ 55.596541271978026, -2.683458728, "div" ], "OpenHands/GPT-5.4": [ 64.29554947656158, 0.015549477, 1 ], "OpenHands/GPT-5.5": [ 65.92354493172454, -0.016455068, 1 ], "OpenHands/Gemini-3-Flash": [ 47.56828819104102, -1.431711809, "div" ], "OpenHands/Gemini-3-Pro": [ 46.6689798979898, -2.371020102, "div" ], "OpenHands/Gemini-3.1-Pro": [ 57.807425703852815, -2.752574296, "div" ], "OpenHands/Gemini-3.5-Flash": [ 63.51950458677079, 0.879504587, "div" ], "OpenHands/Kimi-K2-Thinking": [ 38.8498203883695, -2.150179612, "div" ], "OpenHands/Kimi-K2.5": [ 46.916524794874306, -2.263475205, "div" ], "OpenHands/Kimi-K2.6": [ 57.152144841980466, 0.012144842, 1 ], "OpenHands/MiniMax-M2.1": [ 40.07444900025936, -1.085551, "div" ], "OpenHands/MiniMax-M2.5": [ 43.55495695989922, -1.66504304, "div" ], "OpenHands/MiniMax-M2.7": [ 41.69310127494206, -1.686898725, "div" ], "OpenHands/MiniMax-M3": [ 57.28849103213723, 0.088491032, 2 ], "OpenHands/Nemotron-3-Nano": [ 14.027254901960784, -1.452745098, "div" ], "OpenHands/Nemotron-3-Super": [ 35.42411357152561, -0.735886428, "div" ], "OpenHands/Qwen3-Coder-480B": [ 30.074398986319608, -0.865601014, "div" ], "OpenHands/Qwen3-Coder-Next": [ 41.31438893760317, -2.505611062, "div" ], "OpenHands/Qwen3.5-Flash": [ 37.817929464458466, -0.262070536, 2 ], "OpenHands/Qwen3.6-Plus": [ 52.63470839354007, -0.225291606, 2 ], "OpenHands/Trinity-Large-Thinking": [ 32.010535376863324, -0.049464623, 1 ], "OpenHands/claude-fable-5": [ 81.00950373178874, 0.009503732, 1 ], "OpenHands/claude-opus-4-5": [ 57.88712426438949, -2.692875736, "div" ], "OpenHands/claude-opus-4-6": [ 64.04685187700946, -2.673148123, "div" ], "OpenHands/claude-opus-4-7": [ 69.68458835056171, 0.024588351, 1 ], "OpenHands/claude-opus-4-8": [ 71.87669815942334, -0.003301841, 0 ], "OpenHands/claude-sonnet-4-5": [ 50.843958877419894, -2.156041123, "div" ], "OpenHands/claude-sonnet-4-6": [ 42.66870624462256, -1.851293755, "div" ] }, "t0": 1, "t1": 6, "t2": 4, "undef": 0 }, "AS_macro_pub1dp": { "div": 23, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 44.0, -1.68, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 39.239999999999995, -1.42, "div" ], "OpenHands/GLM-4.7": [ 41.040000000000006, -1.22, "div" ], "OpenHands/GLM-5": [ 50.5, 1.06, "div" ], "OpenHands/GLM-5.1": [ 58.14, -0.1, 2 ], "OpenHands/GPT-5.2": [ 56.98, -1.86, "div" ], "OpenHands/GPT-5.2-Codex": [ 55.6, -2.68, "div" ], "OpenHands/GPT-5.4": [ 64.28, 0.0, 0 ], "OpenHands/GPT-5.5": [ 65.94, 0.0, 0 ], "OpenHands/Gemini-3-Flash": [ 47.58, -1.42, "div" ], "OpenHands/Gemini-3-Pro": [ 46.67999999999999, -2.36, "div" ], "OpenHands/Gemini-3.1-Pro": [ 57.8, -2.76, "div" ], "OpenHands/Gemini-3.5-Flash": [ 63.519999999999996, 0.88, "div" ], "OpenHands/Kimi-K2-Thinking": [ 38.84, -2.16, "div" ], "OpenHands/Kimi-K2.5": [ 46.92, -2.26, "div" ], "OpenHands/Kimi-K2.6": [ 57.14000000000001, 0.0, 0 ], "OpenHands/MiniMax-M2.1": [ 40.08, -1.08, "div" ], "OpenHands/MiniMax-M2.5": [ 43.559999999999995, -1.66, "div" ], "OpenHands/MiniMax-M2.7": [ 41.71999999999999, -1.66, "div" ], "OpenHands/MiniMax-M3": [ 57.3, 0.1, 2 ], "OpenHands/Nemotron-3-Nano": [ 14.020000000000001, -1.46, "div" ], "OpenHands/Nemotron-3-Super": [ 35.42, -0.74, "div" ], "OpenHands/Qwen3-Coder-480B": [ 30.060000000000002, -0.88, "div" ], "OpenHands/Qwen3-Coder-Next": [ 41.32, -2.5, "div" ], "OpenHands/Qwen3.5-Flash": [ 37.82000000000001, -0.26, 2 ], "OpenHands/Qwen3.6-Plus": [ 52.64, -0.22, 2 ], "OpenHands/Trinity-Large-Thinking": [ 32.0, -0.06, 2 ], "OpenHands/claude-fable-5": [ 80.99999999999999, -0.0, 0 ], "OpenHands/claude-opus-4-5": [ 57.879999999999995, -2.7, "div" ], "OpenHands/claude-opus-4-6": [ 64.03999999999999, -2.68, "div" ], "OpenHands/claude-opus-4-7": [ 69.66, 0.0, 0 ], "OpenHands/claude-opus-4-8": [ 71.88000000000001, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 50.839999999999996, -2.16, "div" ], "OpenHands/claude-sonnet-4-6": [ 42.660000000000004, -1.86, "div" ] }, "t0": 6, "t1": 0, "t2": 5, "undef": 0 }, "AS_macro_raw_Dall": { "div": 23, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 43.998115036027215, -1.681884964, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 39.232391102987485, -1.427608897, "div" ], "OpenHands/GLM-4.7": [ 41.05123535325375, -1.208764647, "div" ], "OpenHands/GLM-5": [ 50.49956674885523, 1.059566749, "div" ], "OpenHands/GLM-5.1": [ 58.13903125810475, -0.100968742, 2 ], "OpenHands/GPT-5.2": [ 56.97633132537665, -1.863668675, "div" ], "OpenHands/GPT-5.2-Codex": [ 55.596541271978026, -2.683458728, "div" ], "OpenHands/GPT-5.4": [ 64.29554947656158, 0.015549477, 1 ], "OpenHands/GPT-5.5": [ 65.92354493172454, -0.016455068, 1 ], "OpenHands/Gemini-3-Flash": [ 47.56828819104102, -1.431711809, "div" ], "OpenHands/Gemini-3-Pro": [ 46.6689798979898, -2.371020102, "div" ], "OpenHands/Gemini-3.1-Pro": [ 57.807425703852815, -2.752574296, "div" ], "OpenHands/Gemini-3.5-Flash": [ 63.51950458677079, 0.879504587, "div" ], "OpenHands/Kimi-K2-Thinking": [ 38.8498203883695, -2.150179612, "div" ], "OpenHands/Kimi-K2.5": [ 46.916524794874306, -2.263475205, "div" ], "OpenHands/Kimi-K2.6": [ 57.152144841980466, 0.012144842, 1 ], "OpenHands/MiniMax-M2.1": [ 40.07444900025936, -1.085551, "div" ], "OpenHands/MiniMax-M2.5": [ 43.55495695989922, -1.66504304, "div" ], "OpenHands/MiniMax-M2.7": [ 41.69310127494206, -1.686898725, "div" ], "OpenHands/MiniMax-M3": [ 57.28849103213723, 0.088491032, 2 ], "OpenHands/Nemotron-3-Nano": [ 14.027254901960784, -1.452745098, "div" ], "OpenHands/Nemotron-3-Super": [ 35.42411357152561, -0.735886428, "div" ], "OpenHands/Qwen3-Coder-480B": [ 30.074398986319608, -0.865601014, "div" ], "OpenHands/Qwen3-Coder-Next": [ 41.31438893760317, -2.505611062, "div" ], "OpenHands/Qwen3.5-Flash": [ 37.817929464458466, -0.262070536, 2 ], "OpenHands/Qwen3.6-Plus": [ 52.63470839354007, -0.225291606, 2 ], "OpenHands/Trinity-Large-Thinking": [ 32.010535376863324, -0.049464623, 1 ], "OpenHands/claude-fable-5": [ 81.00950373178874, 0.009503732, 1 ], "OpenHands/claude-opus-4-5": [ 57.88712426438949, -2.692875736, "div" ], "OpenHands/claude-opus-4-6": [ 64.04685187700946, -2.673148123, "div" ], "OpenHands/claude-opus-4-7": [ 69.68458835056171, 0.024588351, 1 ], "OpenHands/claude-opus-4-8": [ 71.87669815942334, -0.003301841, 0 ], "OpenHands/claude-sonnet-4-5": [ 50.843958877419894, -2.156041123, "div" ], "OpenHands/claude-sonnet-4-6": [ 42.66870624462256, -1.851293755, "div" ] }, "t0": 1, "t1": 6, "t2": 4, "undef": 0 }, "AS_macro_raw_Dobs": { "div": 21, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 44.08455366781796, -1.595446332, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 39.232391102987485, -1.427608897, "div" ], "OpenHands/GLM-4.7": [ 41.05123535325375, -1.208764647, "div" ], "OpenHands/GLM-5": [ 50.55198375526176, 1.111983755, "div" ], "OpenHands/GLM-5.1": [ 58.13903125810475, -0.100968742, 2 ], "OpenHands/GPT-5.2": [ 56.97633132537665, -1.863668675, "div" ], "OpenHands/GPT-5.2-Codex": [ 55.641192795953955, -2.638807204, "div" ], "OpenHands/GPT-5.4": [ 64.29554947656158, 0.015549477, 1 ], "OpenHands/GPT-5.5": [ 65.92354493172454, -0.016455068, 1 ], "OpenHands/Gemini-3-Flash": [ 48.523215616556676, -0.476784383, 2 ], "OpenHands/Gemini-3-Pro": [ 46.71848484848485, -2.321515152, "div" ], "OpenHands/Gemini-3.1-Pro": [ 57.807425703852815, -2.752574296, "div" ], "OpenHands/Gemini-3.5-Flash": [ 63.51950458677079, 0.879504587, "div" ], "OpenHands/Kimi-K2-Thinking": [ 38.90540271768677, -2.094597282, "div" ], "OpenHands/Kimi-K2.5": [ 47.02869092854509, -2.151309071, "div" ], "OpenHands/Kimi-K2.6": [ 57.152144841980466, 0.012144842, 1 ], "OpenHands/MiniMax-M2.1": [ 40.7023963817436, -0.457603618, 2 ], "OpenHands/MiniMax-M2.5": [ 43.58796026022925, -1.63203974, "div" ], "OpenHands/MiniMax-M2.7": [ 41.69310127494206, -1.686898725, "div" ], "OpenHands/MiniMax-M3": [ 57.28849103213723, 0.088491032, 2 ], "OpenHands/Nemotron-3-Nano": [ 14.099604558041317, -1.380395442, "div" ], "OpenHands/Nemotron-3-Super": [ 35.485570990314024, -0.67442901, "div" ], "OpenHands/Qwen3-Coder-480B": [ 30.074398986319608, -0.865601014, "div" ], "OpenHands/Qwen3-Coder-Next": [ 41.31438893760317, -2.505611062, "div" ], "OpenHands/Qwen3.5-Flash": [ 37.817929464458466, -0.262070536, 2 ], "OpenHands/Qwen3.6-Plus": [ 52.63470839354007, -0.225291606, 2 ], "OpenHands/Trinity-Large-Thinking": [ 32.08516224253496, 0.025162243, 1 ], "OpenHands/claude-fable-5": [ 81.00950373178874, 0.009503732, 1 ], "OpenHands/claude-opus-4-5": [ 57.94256980894395, -2.637430191, "div" ], "OpenHands/claude-opus-4-6": [ 64.04685187700946, -2.673148123, "div" ], "OpenHands/claude-opus-4-7": [ 69.68458835056171, 0.024588351, 1 ], "OpenHands/claude-opus-4-8": [ 71.87669815942334, -0.003301841, 0 ], "OpenHands/claude-sonnet-4-5": [ 50.94591966173361, -2.054080338, "div" ], "OpenHands/claude-sonnet-4-6": [ 42.66870624462256, -1.851293755, "div" ] }, "t0": 1, "t1": 6, "t2": 6, "undef": 0 }, "AS_micro_Dall": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 57.5682382133995, 11.888238213, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 57.36040609137056, 16.700406091, "div" ], "OpenHands/GLM-4.7": [ 57.01244813278009, 14.752448133, "div" ], "OpenHands/GLM-5": [ 62.936436884512084, 13.496436885, "div" ], "OpenHands/GLM-5.1": [ 69.54314720812182, 11.303147208, "div" ], "OpenHands/GPT-5.2": [ 68.34710743801652, 9.507107438, "div" ], "OpenHands/GPT-5.2-Codex": [ 66.28289473684211, 8.002894737, "div" ], "OpenHands/GPT-5.4": [ 72.165820642978, 7.885820643, "div" ], "OpenHands/GPT-5.5": [ 78.42639593908629, 12.486395939, "div" ], "OpenHands/Gemini-3-Flash": [ 65.40362438220758, 16.403624382, "div" ], "OpenHands/Gemini-3-Pro": [ 61.943986820428336, 12.90398682, "div" ], "OpenHands/Gemini-3.1-Pro": [ 72.28618421052632, 11.726184211, "div" ], "OpenHands/Gemini-3.5-Flash": [ 75.87382779198636, 13.233827792, "div" ], "OpenHands/Kimi-K2-Thinking": [ 53.20723684210526, 12.207236842, "div" ], "OpenHands/Kimi-K2.5": [ 61.01973684210526, 11.839736842, "div" ], "OpenHands/Kimi-K2.6": [ 70.47377326565145, 13.333773266, "div" ], "OpenHands/MiniMax-M2.1": [ 56.82565789473684, 15.665657895, "div" ], "OpenHands/MiniMax-M2.5": [ 62.171052631578945, 16.951052632, "div" ], "OpenHands/MiniMax-M2.7": [ 61.01973684210526, 17.639736842, "div" ], "OpenHands/MiniMax-M3": [ 73.83573243014395, 16.63573243, "div" ], "OpenHands/Nemotron-3-Nano": [ 17.565217391304348, 2.085217391, "div" ], "OpenHands/Nemotron-3-Super": [ 48.56175972927242, 12.401759729, "div" ], "OpenHands/Qwen3-Coder-480B": [ 44.82470784641068, 13.884707846, "div" ], "OpenHands/Qwen3-Coder-Next": [ 51.5625, 7.7425, "div" ], "OpenHands/Qwen3.5-Flash": [ 48.47715736040609, 10.39715736, "div" ], "OpenHands/Qwen3.6-Plus": [ 65.9052453468697, 13.045245347, "div" ], "OpenHands/Trinity-Large-Thinking": [ 42.30118443316413, 10.241184433, "div" ], "OpenHands/claude-fable-5": [ 90.86294416243655, 9.862944162, "div" ], "OpenHands/claude-opus-4-5": [ 71.6872427983539, 11.107242798, "div" ], "OpenHands/claude-opus-4-6": [ 73.60197368421052, 6.881973684, "div" ], "OpenHands/claude-opus-4-7": [ 79.01861252115059, 9.358612521, "div" ], "OpenHands/claude-opus-4-8": [ 81.04906937394247, 9.169069374, "div" ], "OpenHands/claude-sonnet-4-5": [ 67.35366859027205, 14.35366859, "div" ], "OpenHands/claude-sonnet-4-6": [ 54.11184210526316, 9.591842105, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "AS_micro_Dobs": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 57.711442786069654, 12.031442786, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 57.36040609137056, 16.700406091, "div" ], "OpenHands/GLM-4.7": [ 57.01244813278009, 14.752448133, "div" ], "OpenHands/GLM-5": [ 62.992831541218635, 13.552831541, "div" ], "OpenHands/GLM-5.1": [ 69.54314720812182, 11.303147208, "div" ], "OpenHands/GPT-5.2": [ 68.34710743801652, 9.507107438, "div" ], "OpenHands/GPT-5.2-Codex": [ 66.33744855967078, 8.05744856, "div" ], "OpenHands/GPT-5.4": [ 72.165820642978, 7.885820643, "div" ], "OpenHands/GPT-5.5": [ 78.42639593908629, 12.486395939, "div" ], "OpenHands/Gemini-3-Flash": [ 66.77880571909168, 17.778805719, "div" ], "OpenHands/Gemini-3-Pro": [ 61.99505358615004, 12.955053586, "div" ], "OpenHands/Gemini-3.1-Pro": [ 72.28618421052632, 11.726184211, "div" ], "OpenHands/Gemini-3.5-Flash": [ 75.87382779198636, 13.233827792, "div" ], "OpenHands/Kimi-K2-Thinking": [ 53.29489291598023, 12.294892916, "div" ], "OpenHands/Kimi-K2.5": [ 61.17065127782358, 11.990651278, "div" ], "OpenHands/Kimi-K2.6": [ 70.47377326565145, 13.333773266, "div" ], "OpenHands/MiniMax-M2.1": [ 57.921207041072925, 16.761207041, "div" ], "OpenHands/MiniMax-M2.5": [ 62.22222222222222, 17.002222222, "div" ], "OpenHands/MiniMax-M2.7": [ 61.01973684210526, 17.639736842, "div" ], "OpenHands/MiniMax-M3": [ 73.83573243014395, 16.63573243, "div" ], "OpenHands/Nemotron-3-Nano": [ 27.2972972972973, 11.817297297, "div" ], "OpenHands/Nemotron-3-Super": [ 48.602878916172735, 12.442878916, "div" ], "OpenHands/Qwen3-Coder-480B": [ 44.82470784641068, 13.884707846, "div" ], "OpenHands/Qwen3-Coder-Next": [ 51.5625, 7.7425, "div" ], "OpenHands/Qwen3.5-Flash": [ 48.47715736040609, 10.39715736, "div" ], "OpenHands/Qwen3.6-Plus": [ 65.9052453468697, 13.045245347, "div" ], "OpenHands/Trinity-Large-Thinking": [ 42.337002540220155, 10.27700254, "div" ], "OpenHands/claude-fable-5": [ 90.86294416243655, 9.862944162, "div" ], "OpenHands/claude-opus-4-5": [ 71.74629324546952, 11.166293245, "div" ], "OpenHands/claude-opus-4-6": [ 73.60197368421052, 6.881973684, "div" ], "OpenHands/claude-opus-4-7": [ 79.01861252115059, 9.358612521, "div" ], "OpenHands/claude-opus-4-8": [ 81.04906937394247, 9.169069374, "div" ], "OpenHands/claude-sonnet-4-5": [ 67.46490503715937, 14.464905037, "div" ], "OpenHands/claude-sonnet-4-6": [ 54.11184210526316, 9.591842105, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "AS_macro_raw_Dall", "published_range": [ 15.48, 81.0 ], "zero_variance": false }, "categories_completed": { "dp": null, "family": "count", "max_abs_delta": 0, "max_abs_rows": [ "OpenHands/DeepSeek-V3.2-Reasoner", "OpenHands/DeepSeek-V4-Pro", "OpenHands/GLM-4.7", "OpenHands/GLM-5", "OpenHands/GLM-5.1", "OpenHands/GPT-5.2", "OpenHands/GPT-5.2-Codex", "OpenHands/GPT-5.4", "OpenHands/GPT-5.5", "OpenHands/Gemini-3-Flash", "OpenHands/Gemini-3-Pro", "OpenHands/Gemini-3.1-Pro", "OpenHands/Gemini-3.5-Flash", "OpenHands/Kimi-K2-Thinking", "OpenHands/Kimi-K2.5", "OpenHands/Kimi-K2.6", "OpenHands/MiniMax-M2.1", "OpenHands/MiniMax-M2.5", "OpenHands/MiniMax-M2.7", "OpenHands/MiniMax-M3", "OpenHands/Nemotron-3-Nano", "OpenHands/Nemotron-3-Super", "OpenHands/Qwen3-Coder-480B", "OpenHands/Qwen3-Coder-Next", "OpenHands/Qwen3.5-Flash", "OpenHands/Qwen3.6-Plus", "OpenHands/Trinity-Large-Thinking", "OpenHands/claude-fable-5", "OpenHands/claude-opus-4-5", "OpenHands/claude-opus-4-6", "OpenHands/claude-opus-4-7", "OpenHands/claude-opus-4-8", "OpenHands/claude-sonnet-4-5", "OpenHands/claude-sonnet-4-6" ], "policies": { "CC1_any_row": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 5, 0, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 5, 0, 0 ], "OpenHands/GLM-4.7": [ 5, 0, 0 ], "OpenHands/GLM-5": [ 5, 0, 0 ], "OpenHands/GLM-5.1": [ 5, 0, 0 ], "OpenHands/GPT-5.2": [ 5, 0, 0 ], "OpenHands/GPT-5.2-Codex": [ 5, 0, 0 ], "OpenHands/GPT-5.4": [ 5, 0, 0 ], "OpenHands/GPT-5.5": [ 5, 0, 0 ], "OpenHands/Gemini-3-Flash": [ 5, 0, 0 ], "OpenHands/Gemini-3-Pro": [ 5, 0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 5, 0, 0 ], "OpenHands/Gemini-3.5-Flash": [ 5, 0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 5, 0, 0 ], "OpenHands/Kimi-K2.5": [ 5, 0, 0 ], "OpenHands/Kimi-K2.6": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.1": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.5": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.7": [ 5, 0, 0 ], "OpenHands/MiniMax-M3": [ 5, 0, 0 ], "OpenHands/Nemotron-3-Nano": [ 5, 0, 0 ], "OpenHands/Nemotron-3-Super": [ 5, 0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 5, 0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 5, 0, 0 ], "OpenHands/Qwen3.5-Flash": [ 5, 0, 0 ], "OpenHands/Qwen3.6-Plus": [ 5, 0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 5, 0, 0 ], "OpenHands/claude-fable-5": [ 5, 0, 0 ], "OpenHands/claude-opus-4-5": [ 5, 0, 0 ], "OpenHands/claude-opus-4-6": [ 5, 0, 0 ], "OpenHands/claude-opus-4-7": [ 5, 0, 0 ], "OpenHands/claude-opus-4-8": [ 5, 0, 0 ], "OpenHands/claude-sonnet-4-5": [ 5, 0, 0 ], "OpenHands/claude-sonnet-4-6": [ 5, 0, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 }, "CC2_any_obs": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 5, 0, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 5, 0, 0 ], "OpenHands/GLM-4.7": [ 5, 0, 0 ], "OpenHands/GLM-5": [ 5, 0, 0 ], "OpenHands/GLM-5.1": [ 5, 0, 0 ], "OpenHands/GPT-5.2": [ 5, 0, 0 ], "OpenHands/GPT-5.2-Codex": [ 5, 0, 0 ], "OpenHands/GPT-5.4": [ 5, 0, 0 ], "OpenHands/GPT-5.5": [ 5, 0, 0 ], "OpenHands/Gemini-3-Flash": [ 5, 0, 0 ], "OpenHands/Gemini-3-Pro": [ 5, 0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 5, 0, 0 ], "OpenHands/Gemini-3.5-Flash": [ 5, 0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 5, 0, 0 ], "OpenHands/Kimi-K2.5": [ 5, 0, 0 ], "OpenHands/Kimi-K2.6": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.1": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.5": [ 5, 0, 0 ], "OpenHands/MiniMax-M2.7": [ 5, 0, 0 ], "OpenHands/MiniMax-M3": [ 5, 0, 0 ], "OpenHands/Nemotron-3-Nano": [ 5, 0, 0 ], "OpenHands/Nemotron-3-Super": [ 5, 0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 5, 0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 5, 0, 0 ], "OpenHands/Qwen3.5-Flash": [ 5, 0, 0 ], "OpenHands/Qwen3.6-Plus": [ 5, 0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 5, 0, 0 ], "OpenHands/claude-fable-5": [ 5, 0, 0 ], "OpenHands/claude-opus-4-5": [ 5, 0, 0 ], "OpenHands/claude-opus-4-6": [ 5, 0, 0 ], "OpenHands/claude-opus-4-7": [ 5, 0, 0 ], "OpenHands/claude-opus-4-8": [ 5, 0, 0 ], "OpenHands/claude-sonnet-4-5": [ 5, 0, 0 ], "OpenHands/claude-sonnet-4-6": [ 5, 0, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "CC1_any_row", "published_range": [ 5.0, 5.0 ], "zero_variance": true }, "frontend_cost": { "dp": 4, "family": "cost", "max_abs_delta": 3.308446717, "max_abs_rows": [ "OpenHands/GLM-5.1" ], "policies": { "C_mean_excl": { "div": 22, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.1820111563529411, -0.007988844, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.3746000000000001, 0.0, 0 ], "OpenHands/GLM-4.7": [ 0.6555361336530613, -0.004463866, 1 ], "OpenHands/GLM-5": [ 0.7938024260810811, 0.213802426, "div" ], "OpenHands/GLM-5.1": [ 3.611553282896166, -3.308446717, "div" ], "OpenHands/GPT-5.2": [ 2.512101350980393, -0.257898649, "div" ], "OpenHands/GPT-5.2-Codex": [ 2.5635493504901965, -0.40645065, "div" ], "OpenHands/GPT-5.4": [ 1.9413332058823527, 0.491333206, "div" ], "OpenHands/GPT-5.5": [ 3.9525263582089547, 1.142526358, "div" ], "OpenHands/Gemini-3-Flash": [ 0.5621339653846156, -0.237866035, "div" ], "OpenHands/Gemini-3-Pro": [ 1.4992268371134014, 0.039226837, 2 ], "OpenHands/Gemini-3.1-Pro": [ 1.3278846752688174, 0.087884675, "div" ], "OpenHands/Gemini-3.5-Flash": [ 3.02804757761194, 1.038047578, "div" ], "OpenHands/Kimi-K2-Thinking": [ 2.124638034313725, -0.185361966, "div" ], "OpenHands/Kimi-K2.5": [ 1.6192795042198747, -0.002020496, 1 ], "OpenHands/Kimi-K2.6": [ 0.9629579069117646, 0.322957907, "div" ], "OpenHands/MiniMax-M2.1": [ 0.8213209529411764, -0.582979047, "div" ], "OpenHands/MiniMax-M2.5": [ 0.5948944569334779, 0.000794457, 1 ], "OpenHands/MiniMax-M2.7": [ 0.388, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.5563755138461535, 0.206375514, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.38356774352941186, -0.016432256, 2 ], "OpenHands/Nemotron-3-Super": [ 0.7705007194650516, 0.002600719, 1 ], "OpenHands/Qwen3-Coder-480B": [ 1.9157792294117648, -0.174220771, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.6038073296511632, 0.08380733, "div" ], "OpenHands/Qwen3.5-Flash": [ 2.6217999999999995, -0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.3157999999999994, -0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 1.085671747878788, 0.345671748, "div" ], "OpenHands/claude-fable-5": [ 6.5844214044117635, 2.194421404, "div" ], "OpenHands/claude-opus-4-5": [ 2.3702183788659785, -0.169781621, "div" ], "OpenHands/claude-opus-4-6": [ 2.4143249175000006, 0.044324918, 2 ], "OpenHands/claude-opus-4-7": [ 4.6340143515625, 1.804014352, "div" ], "OpenHands/claude-opus-4-8": [ 2.7112812389705887, 0.901281239, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.8283343415000002, -0.061665658, "div" ], "OpenHands/claude-sonnet-4-6": [ 2.4000689905263153, 0.160068991, "div" ] }, "t0": 4, "t1": 4, "t2": 4, "undef": 0 }, "C_mean_zero": { "div": 20, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.1820111563529411, -0.007988844, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.3746000000000001, 0.0, 0 ], "OpenHands/GLM-4.7": [ 0.6298288342941177, -0.030171166, 2 ], "OpenHands/GLM-5": [ 0.5758958777450981, -0.004104122, 1 ], "OpenHands/GLM-5.1": [ 3.558442205206516, -3.361557795, "div" ], "OpenHands/GPT-5.2": [ 2.512101350980393, -0.257898649, "div" ], "OpenHands/GPT-5.2-Codex": [ 2.5635493504901965, -0.40645065, "div" ], "OpenHands/GPT-5.4": [ 1.9413332058823527, 0.491333206, "div" ], "OpenHands/GPT-5.5": [ 3.8944009705882348, 1.084400971, "div" ], "OpenHands/Gemini-3-Flash": [ 0.5015116750000002, -0.298488325, "div" ], "OpenHands/Gemini-3-Pro": [ 1.4398515168316826, -0.020148483, 2 ], "OpenHands/Gemini-3.1-Pro": [ 1.2107183803921568, -0.02928162, 2 ], "OpenHands/Gemini-3.5-Flash": [ 3.02804757761194, 1.038047578, "div" ], "OpenHands/Kimi-K2-Thinking": [ 2.124638034313725, -0.185361966, "div" ], "OpenHands/Kimi-K2.5": [ 1.5878371837495857, -0.033462816, 2 ], "OpenHands/Kimi-K2.6": [ 0.9629579069117646, 0.322957907, "div" ], "OpenHands/MiniMax-M2.1": [ 0.8213209529411764, -0.582979047, "div" ], "OpenHands/MiniMax-M2.5": [ 0.5948944569334779, 0.000794457, 1 ], "OpenHands/MiniMax-M2.7": [ 0.388, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.5397672895522385, 0.18976729, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.38356774352941186, -0.016432256, 2 ], "OpenHands/Nemotron-3-Super": [ 0.7705007194650516, 0.002600719, 1 ], "OpenHands/Qwen3-Coder-480B": [ 1.9157792294117648, -0.174220771, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.3522297093137259, -0.167770291, "div" ], "OpenHands/Qwen3.5-Flash": [ 2.544688235294117, -0.077111765, "div" ], "OpenHands/Qwen3.6-Plus": [ 2.3157999999999994, -0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 1.0537402258823532, 0.313740226, "div" ], "OpenHands/claude-fable-5": [ 6.5844214044117635, 2.194421404, "div" ], "OpenHands/claude-opus-4-5": [ 2.2763483440594054, -0.263651656, "div" ], "OpenHands/claude-opus-4-6": [ 2.3669852132352944, -0.003014787, 1 ], "OpenHands/claude-opus-4-7": [ 4.361425272058823, 1.531425272, "div" ], "OpenHands/claude-opus-4-8": [ 2.7112812389705887, 0.901281239, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.792484648529412, -0.097515351, "div" ], "OpenHands/claude-sonnet-4-6": [ 2.2353583735294116, -0.004641626, 1 ] }, "t0": 3, "t1": 5, "t2": 6, "undef": 0 }, "C_median_excl": { "div": 31, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.165514398, -0.024485602, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.3214733896845353, -0.05312661, "div" ], "OpenHands/GLM-4.7": [ 0.6416780025, -0.018321998, 2 ], "OpenHands/GLM-5": [ 0.8238584449999997, 0.243858445, "div" ], "OpenHands/GLM-5.1": [ 2.9187819200040326, -4.00121808, "div" ], "OpenHands/GPT-5.2": [ 2.1343819, -0.6356181, "div" ], "OpenHands/GPT-5.2-Codex": [ 2.109178225, -0.860821775, "div" ], "OpenHands/GPT-5.4": [ 1.70458525, 0.25458525, "div" ], "OpenHands/GPT-5.5": [ 3.2421130000000002, 0.432113, "div" ], "OpenHands/Gemini-3-Flash": [ 0.49406380000000005, -0.3059362, "div" ], "OpenHands/Gemini-3-Pro": [ 1.2934849999999996, -0.166515, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.9932899999999999, -0.24671, "div" ], "OpenHands/Gemini-3.5-Flash": [ 2.0613399000000006, 0.0713399, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.8683008499999998, -0.44169915, "div" ], "OpenHands/Kimi-K2.5": [ 1.3829250122796413, -0.238374988, "div" ], "OpenHands/Kimi-K2.6": [ 0.80022153, 0.16022153, "div" ], "OpenHands/MiniMax-M2.1": [ 0.8168950499999998, -0.58740495, "div" ], "OpenHands/MiniMax-M2.5": [ 0.44694443480751567, -0.147155565, "div" ], "OpenHands/MiniMax-M2.7": [ 0.26639318656570676, -0.121606813, "div" ], "OpenHands/MiniMax-M3": [ 0.45747959999999976, 0.1074796, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.33858642, -0.06141358, "div" ], "OpenHands/Nemotron-3-Super": [ 0.6697484938072398, -0.098151506, "div" ], "OpenHands/Qwen3-Coder-480B": [ 1.6655948999999994, -0.4244051, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.5508106750000001, 0.030810675, 2 ], "OpenHands/Qwen3.5-Flash": [ 2.297444639498628, -0.324355361, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.8306772876620494, -0.485122712, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.8865803749999998, 0.146580375, "div" ], "OpenHands/claude-fable-5": [ 4.778762, 0.388762, "div" ], "OpenHands/claude-opus-4-5": [ 1.9940194999999998, -0.5459805, "div" ], "OpenHands/claude-opus-4-6": [ 1.5307908749999997, -0.839209125, "div" ], "OpenHands/claude-opus-4-7": [ 2.30334125, -0.52665875, "div" ], "OpenHands/claude-opus-4-8": [ 1.7295115000000005, -0.0804885, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.7412478500000006, -0.14875215, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.9322875499999996, -0.30771245, "div" ] }, "t0": 0, "t1": 0, "t2": 3, "undef": 0 }, "C_median_zero": { "div": 31, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.165514398, -0.024485602, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.3214733896845353, -0.05312661, "div" ], "OpenHands/GLM-4.7": [ 0.6233573024999999, -0.036642698, 2 ], "OpenHands/GLM-5": [ 0.5617730200000002, -0.01822698, 2 ], "OpenHands/GLM-5.1": [ 2.8954865300020165, -4.02451347, "div" ], "OpenHands/GPT-5.2": [ 2.1343819, -0.6356181, "div" ], "OpenHands/GPT-5.2-Codex": [ 2.109178225, -0.860821775, "div" ], "OpenHands/GPT-5.4": [ 1.70458525, 0.25458525, "div" ], "OpenHands/GPT-5.5": [ 3.2325645000000005, 0.4225645, "div" ], "OpenHands/Gemini-3-Flash": [ 0.4691379, -0.3308621, "div" ], "OpenHands/Gemini-3-Pro": [ 1.2556868, -0.2043132, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.7221481999999999, -0.5178518, "div" ], "OpenHands/Gemini-3.5-Flash": [ 2.0613399000000006, 0.0713399, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.8683008499999998, -0.44169915, "div" ], "OpenHands/Kimi-K2.5": [ 1.362959429044405, -0.258340571, "div" ], "OpenHands/Kimi-K2.6": [ 0.80022153, 0.16022153, "div" ], "OpenHands/MiniMax-M2.1": [ 0.8168950499999998, -0.58740495, "div" ], "OpenHands/MiniMax-M2.5": [ 0.44694443480751567, -0.147155565, "div" ], "OpenHands/MiniMax-M2.7": [ 0.26639318656570676, -0.121606813, "div" ], "OpenHands/MiniMax-M3": [ 0.4381403999999999, 0.0881404, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.33858642, -0.06141358, "div" ], "OpenHands/Nemotron-3-Super": [ 0.6697484938072398, -0.098151506, "div" ], "OpenHands/Qwen3-Coder-480B": [ 1.6655948999999994, -0.4244051, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.35560135, -0.16439865, "div" ], "OpenHands/Qwen3.5-Flash": [ 2.2595173688239503, -0.362282631, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.8306772876620494, -0.485122712, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.8671109699999997, 0.12711097, "div" ], "OpenHands/claude-fable-5": [ 4.778762, 0.388762, "div" ], "OpenHands/claude-opus-4-5": [ 1.850031, -0.689969, "div" ], "OpenHands/claude-opus-4-6": [ 1.4637753749999995, -0.906224625, "div" ], "OpenHands/claude-opus-4-7": [ 2.1213756249999998, -0.708624375, "div" ], "OpenHands/claude-opus-4-8": [ 1.7295115000000005, -0.0804885, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.72148415, -0.16851585, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.7408153250000002, -0.499184675, "div" ] }, "t0": 0, "t1": 0, "t2": 3, "undef": 0 }, "C_sum_over_resolved": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.977112523578947, 0.787112524, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 1.0189120000000003, 0.644312, "div" ], "OpenHands/GLM-4.7": [ 4.2828360732, 3.622836073, "div" ], "OpenHands/GLM-5": [ 2.1756066492592594, 1.595606649, "div" ], "OpenHands/GLM-5.1": [ 8.641931069787253, 1.72193107, "div" ], "OpenHands/GPT-5.2": [ 12.201635133333339, 9.431635133, "div" ], "OpenHands/GPT-5.2-Codex": [ 11.368784076086957, 8.398784076, "div" ], "OpenHands/GPT-5.4": [ 5.280426319999999, 3.83042632, "div" ], "OpenHands/GPT-5.5": [ 10.185356384615384, 7.375356385, "div" ], "OpenHands/Gemini-3-Flash": [ 3.4102793900000012, 2.61027939, "div" ], "OpenHands/Gemini-3-Pro": [ 5.817000127999997, 4.357000128, "div" ], "OpenHands/Gemini-3.1-Pro": [ 3.983654025806452, 2.743654026, "div" ], "OpenHands/Gemini-3.5-Flash": [ 7.514043988888888, 5.524043989, "div" ], "OpenHands/Kimi-K2-Thinking": [ 9.85059452272727, 7.540594523, "div" ], "OpenHands/Kimi-K2.5": [ 7.433964996645788, 5.812664997, "div" ], "OpenHands/Kimi-K2.6": [ 2.338612059642857, 1.69861206, "div" ], "OpenHands/MiniMax-M2.1": [ 7.615885199999999, 6.2115852, "div" ], "OpenHands/MiniMax-M2.5": [ 3.569366741600867, 2.975266742, "div" ], "OpenHands/MiniMax-M2.7": [ 1.9788000000000001, 1.5908, "div" ], "OpenHands/MiniMax-M3": [ 1.446576335999999, 1.096576336, "div" ], "OpenHands/Nemotron-3-Nano": [ 2.4452443650000006, 2.045244365, "div" ], "OpenHands/Nemotron-3-Super": [ 3.742432065973108, 2.974532066, "div" ], "OpenHands/Qwen3-Coder-480B": [ 10.8560823, 8.7660823, "div" ], "OpenHands/Qwen3-Coder-Next": [ 6.269428652272729, 4.749428652, "div" ], "OpenHands/Qwen3.5-Flash": [ 9.107305263157892, 6.485505263, "div" ], "OpenHands/Qwen3.6-Plus": [ 7.49878095238095, 5.182980952, "div" ], "OpenHands/Trinity-Large-Thinking": [ 4.214960903529413, 3.474960904, "div" ], "OpenHands/claude-fable-5": [ 9.327930322916666, 4.937930323, "div" ], "OpenHands/claude-opus-4-5": [ 8.211113669642854, 5.67111367, "div" ], "OpenHands/claude-opus-4-6": [ 8.325258336206899, 5.955258336, "div" ], "OpenHands/claude-opus-4-7": [ 8.987179348484847, 6.157179348, "div" ], "OpenHands/claude-opus-4-8": [ 5.422562477941177, 3.612562478, "div" ], "OpenHands/claude-sonnet-4-5": [ 7.0320551596153855, 5.14205516, "div" ], "OpenHands/claude-sonnet-4-6": [ 10.363934277272726, 8.123934277, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "C_mean_excl", "published_range": [ 0.19, 6.92 ], "zero_variance": false }, "frontend_score": { "dp": 1, "family": "score", "max_abs_delta": 13.707843137, "max_abs_rows": [ "OpenHands/Gemini-3.1-Pro" ], "policies": { "S1_frac_Dall": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.18627450980392157, -27.71372549, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.36764705882352944, -36.432352941, "div" ], "OpenHands/GLM-4.7": [ 0.14705882352941177, -21.952941176, "div" ], "OpenHands/GLM-5": [ 0.2647058823529412, -35.035294118, "div" ], "OpenHands/GLM-5.1": [ 0.4117647058823529, -40.788235294, "div" ], "OpenHands/GPT-5.2": [ 0.20588235294117646, -30.694117647, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.22549019607843138, -35.674509804, "div" ], "OpenHands/GPT-5.4": [ 0.36764705882352944, -36.432352941, "div" ], "OpenHands/GPT-5.5": [ 0.38235294117647056, -37.817647059, "div" ], "OpenHands/Gemini-3-Flash": [ 0.14705882352941177, -21.952941176, "div" ], "OpenHands/Gemini-3-Pro": [ 0.24752475247524752, -36.552475248, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.30392156862745096, -43.796078431, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.40298507462686567, -39.297014925, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.21568627450980393, -32.184313725, "div" ], "OpenHands/Kimi-K2.5": [ 0.21359223300970873, -32.586407767, "div" ], "OpenHands/Kimi-K2.6": [ 0.4117647058823529, -40.788235294, "div" ], "OpenHands/MiniMax-M2.1": [ 0.10784313725490197, -16.092156863, "div" ], "OpenHands/MiniMax-M2.5": [ 0.16666666666666666, -24.833333333, "div" ], "OpenHands/MiniMax-M2.7": [ 0.19607843137254902, -27.703921569, "div" ], "OpenHands/MiniMax-M3": [ 0.373134328358209, -36.426865672, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.1568627450980392, -20.743137255, "div" ], "OpenHands/Nemotron-3-Super": [ 0.20588235294117646, -20.394117647, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.17647058823529413, -23.323529412, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.21568627450980393, -30.684313725, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.27941176470588236, -27.620588235, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.3088235294117647, -30.591176471, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.25, -24.75, "div" ], "OpenHands/claude-fable-5": [ 0.7058823529411765, -69.894117647, "div" ], "OpenHands/claude-opus-4-5": [ 0.27722772277227725, -40.922772277, "div" ], "OpenHands/claude-opus-4-6": [ 0.28431372549019607, -41.515686275, "div" ], "OpenHands/claude-opus-4-7": [ 0.4852941176470588, -48.014705882, "div" ], "OpenHands/claude-opus-4-8": [ 0.5, -49.5, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.2549019607843137, -36.545098039, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.21568627450980393, -30.684313725, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S1_pct_Dall": { "div": 22, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 18.627450980392158, -9.27254902, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 36.76470588235294, -0.035294118, 0 ], "OpenHands/GLM-4.7": [ 14.705882352941176, -7.394117647, "div" ], "OpenHands/GLM-5": [ 26.470588235294116, -8.829411765, "div" ], "OpenHands/GLM-5.1": [ 41.1764705882353, -0.023529412, 0 ], "OpenHands/GPT-5.2": [ 20.58823529411765, -10.311764706, "div" ], "OpenHands/GPT-5.2-Codex": [ 22.54901960784314, -13.350980392, "div" ], "OpenHands/GPT-5.4": [ 36.76470588235294, -0.035294118, 0 ], "OpenHands/GPT-5.5": [ 38.23529411764706, 0.035294118, 0 ], "OpenHands/Gemini-3-Flash": [ 14.705882352941176, -7.394117647, "div" ], "OpenHands/Gemini-3-Pro": [ 24.752475247524753, -12.047524752, "div" ], "OpenHands/Gemini-3.1-Pro": [ 30.392156862745097, -13.707843137, "div" ], "OpenHands/Gemini-3.5-Flash": [ 40.298507462686565, 0.598507463, "div" ], "OpenHands/Kimi-K2-Thinking": [ 21.568627450980394, -10.831372549, "div" ], "OpenHands/Kimi-K2.5": [ 21.359223300970875, -11.440776699, "div" ], "OpenHands/Kimi-K2.6": [ 41.1764705882353, -0.023529412, 0 ], "OpenHands/MiniMax-M2.1": [ 10.784313725490197, -5.415686275, "div" ], "OpenHands/MiniMax-M2.5": [ 16.666666666666668, -8.333333333, "div" ], "OpenHands/MiniMax-M2.7": [ 19.607843137254903, -8.292156863, "div" ], "OpenHands/MiniMax-M3": [ 37.3134328358209, 0.513432836, "div" ], "OpenHands/Nemotron-3-Nano": [ 15.686274509803921, -5.21372549, "div" ], "OpenHands/Nemotron-3-Super": [ 20.58823529411765, -0.011764706, 0 ], "OpenHands/Qwen3-Coder-480B": [ 17.647058823529413, -5.852941176, "div" ], "OpenHands/Qwen3-Coder-Next": [ 21.568627450980394, -9.331372549, "div" ], "OpenHands/Qwen3.5-Flash": [ 27.941176470588236, 0.041176471, 0 ], "OpenHands/Qwen3.6-Plus": [ 30.88235294117647, -0.017647059, 0 ], "OpenHands/Trinity-Large-Thinking": [ 25.0, 0.0, 0 ], "OpenHands/claude-fable-5": [ 70.58823529411765, -0.011764706, 0 ], "OpenHands/claude-opus-4-5": [ 27.722772277227723, -13.477227723, "div" ], "OpenHands/claude-opus-4-6": [ 28.431372549019606, -13.368627451, "div" ], "OpenHands/claude-opus-4-7": [ 48.529411764705884, 0.029411765, 0 ], "OpenHands/claude-opus-4-8": [ 50.0, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 25.49019607843137, -11.309803922, "div" ], "OpenHands/claude-sonnet-4-6": [ 21.568627450980394, -9.331372549, "div" ] }, "t0": 12, "t1": 0, "t2": 0, "undef": 0 }, "S2_frac_Dobs": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.18627450980392157, -27.71372549, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.36764705882352944, -36.432352941, "div" ], "OpenHands/GLM-4.7": [ 0.14705882352941177, -21.952941176, "div" ], "OpenHands/GLM-5": [ 0.26732673267326734, -35.032673267, "div" ], "OpenHands/GLM-5.1": [ 0.4117647058823529, -40.788235294, "div" ], "OpenHands/GPT-5.2": [ 0.20588235294117646, -30.694117647, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.22772277227722773, -35.672277228, "div" ], "OpenHands/GPT-5.4": [ 0.36764705882352944, -36.432352941, "div" ], "OpenHands/GPT-5.5": [ 0.38235294117647056, -37.817647059, "div" ], "OpenHands/Gemini-3-Flash": [ 0.19480519480519481, -21.905194805, "div" ], "OpenHands/Gemini-3-Pro": [ 0.25, -36.55, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.30392156862745096, -43.796078431, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.40298507462686567, -39.297014925, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.21568627450980393, -32.184313725, "div" ], "OpenHands/Kimi-K2.5": [ 0.21782178217821782, -32.582178218, "div" ], "OpenHands/Kimi-K2.6": [ 0.4117647058823529, -40.788235294, "div" ], "OpenHands/MiniMax-M2.1": [ 0.13924050632911392, -16.060759494, "div" ], "OpenHands/MiniMax-M2.5": [ 0.16831683168316833, -24.831683168, "div" ], "OpenHands/MiniMax-M2.7": [ 0.19607843137254902, -27.703921569, "div" ], "OpenHands/MiniMax-M3": [ 0.373134328358209, -36.426865672, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.15841584158415842, -20.741584158, "div" ], "OpenHands/Nemotron-3-Super": [ 0.208955223880597, -20.391044776, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.17647058823529413, -23.323529412, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.21568627450980393, -30.684313725, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.27941176470588236, -27.620588235, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.3088235294117647, -30.591176471, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.2537313432835821, -24.746268657, "div" ], "OpenHands/claude-fable-5": [ 0.7058823529411765, -69.894117647, "div" ], "OpenHands/claude-opus-4-5": [ 0.28, -40.92, "div" ], "OpenHands/claude-opus-4-6": [ 0.28431372549019607, -41.515686275, "div" ], "OpenHands/claude-opus-4-7": [ 0.4852941176470588, -48.014705882, "div" ], "OpenHands/claude-opus-4-8": [ 0.5, -49.5, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.26, -36.54, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.21568627450980393, -30.684313725, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S2_pct_Dobs": { "div": 22, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 18.627450980392158, -9.27254902, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 36.76470588235294, -0.035294118, 0 ], "OpenHands/GLM-4.7": [ 14.705882352941176, -7.394117647, "div" ], "OpenHands/GLM-5": [ 26.73267326732673, -8.567326733, "div" ], "OpenHands/GLM-5.1": [ 41.1764705882353, -0.023529412, 0 ], "OpenHands/GPT-5.2": [ 20.58823529411765, -10.311764706, "div" ], "OpenHands/GPT-5.2-Codex": [ 22.77227722772277, -13.127722772, "div" ], "OpenHands/GPT-5.4": [ 36.76470588235294, -0.035294118, 0 ], "OpenHands/GPT-5.5": [ 38.23529411764706, 0.035294118, 0 ], "OpenHands/Gemini-3-Flash": [ 19.48051948051948, -2.619480519, "div" ], "OpenHands/Gemini-3-Pro": [ 25.0, -11.8, "div" ], "OpenHands/Gemini-3.1-Pro": [ 30.392156862745097, -13.707843137, "div" ], "OpenHands/Gemini-3.5-Flash": [ 40.298507462686565, 0.598507463, "div" ], "OpenHands/Kimi-K2-Thinking": [ 21.568627450980394, -10.831372549, "div" ], "OpenHands/Kimi-K2.5": [ 21.782178217821784, -11.017821782, "div" ], "OpenHands/Kimi-K2.6": [ 41.1764705882353, -0.023529412, 0 ], "OpenHands/MiniMax-M2.1": [ 13.924050632911392, -2.275949367, "div" ], "OpenHands/MiniMax-M2.5": [ 16.831683168316832, -8.168316832, "div" ], "OpenHands/MiniMax-M2.7": [ 19.607843137254903, -8.292156863, "div" ], "OpenHands/MiniMax-M3": [ 37.3134328358209, 0.513432836, "div" ], "OpenHands/Nemotron-3-Nano": [ 15.841584158415841, -5.058415842, "div" ], "OpenHands/Nemotron-3-Super": [ 20.895522388059703, 0.295522388, 2 ], "OpenHands/Qwen3-Coder-480B": [ 17.647058823529413, -5.852941176, "div" ], "OpenHands/Qwen3-Coder-Next": [ 21.568627450980394, -9.331372549, "div" ], "OpenHands/Qwen3.5-Flash": [ 27.941176470588236, 0.041176471, 0 ], "OpenHands/Qwen3.6-Plus": [ 30.88235294117647, -0.017647059, 0 ], "OpenHands/Trinity-Large-Thinking": [ 25.37313432835821, 0.373134328, 2 ], "OpenHands/claude-fable-5": [ 70.58823529411765, -0.011764706, 0 ], "OpenHands/claude-opus-4-5": [ 28.0, -13.2, "div" ], "OpenHands/claude-opus-4-6": [ 28.431372549019606, -13.368627451, "div" ], "OpenHands/claude-opus-4-7": [ 48.529411764705884, 0.029411765, 0 ], "OpenHands/claude-opus-4-8": [ 50.0, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 26.0, -10.8, "div" ], "OpenHands/claude-sonnet-4-6": [ 21.568627450980394, -9.331372549, "div" ] }, "t0": 10, "t1": 0, "t2": 2, "undef": 0 } }, "primary": "S1_pct_Dall", "published_range": [ 16.2, 70.6 ], "zero_variance": false }, "greenfield_cost": { "dp": 4, "family": "cost", "max_abs_delta": 2.106382731, "max_abs_rows": [ "OpenHands/claude-opus-4-5" ], "policies": { "C_mean_excl": { "div": 19, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.6520039579999999, 0.082003958, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.1577, 0.0, 0 ], "OpenHands/GLM-4.7": [ 1.8795007597142859, 0.23950076, "div" ], "OpenHands/GLM-5": [ 3.2476659372727266, 1.017665937, "div" ], "OpenHands/GLM-5.1": [ 6.531293725411704, 1.221293725, "div" ], "OpenHands/GPT-5.2": [ 5.00858008, 0.30858008, "div" ], "OpenHands/GPT-5.2-Codex": [ 6.28502805, 0.78502805, "div" ], "OpenHands/GPT-5.4": [ 4.03890865625, -0.001091344, 1 ], "OpenHands/GPT-5.5": [ 5.562512249999999, 0.00251225, 1 ], "OpenHands/Gemini-3-Flash": [ 2.9897381035714288, 0.369738104, "div" ], "OpenHands/Gemini-3-Pro": [ 3.1842415499999994, 0.00424155, 1 ], "OpenHands/Gemini-3.1-Pro": [ 3.1461060999999995, 1.9661061, "div" ], "OpenHands/Gemini-3.5-Flash": [ 9.73802318076923, 1.828023181, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.9553134111111111, 0.415313411, "div" ], "OpenHands/Kimi-K2.5": [ 1.3486991733333331, 0.088699173, "div" ], "OpenHands/Kimi-K2.6": [ 1.51895738625, -0.001042614, 1 ], "OpenHands/MiniMax-M2.1": [ 0.3791234849999998, -0.000876515, 1 ], "OpenHands/MiniMax-M2.5": [ 0.523828224, 0.033828224, 2 ], "OpenHands/MiniMax-M2.7": [ 0.8926, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.8978406545454546, 0.277840655, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.07408501714285716, 0.044085017, 2 ], "OpenHands/Nemotron-3-Super": [ 1.2153000000000003, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.01, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 1.0373696968750001, -0.002630303, 1 ], "OpenHands/Qwen3.5-Flash": [ 4.330900000000001, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 7.493600000000002, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 1.5951761258333335, 0.395176126, "div" ], "OpenHands/claude-fable-5": [ 12.491801687500002, 0.001801688, 1 ], "OpenHands/claude-opus-4-5": [ 11.21638273076923, 2.106382731, "div" ], "OpenHands/claude-opus-4-6": [ 8.791387910714286, 1.101387911, "div" ], "OpenHands/claude-opus-4-7": [ 6.502920696428572, 0.812920696, "div" ], "OpenHands/claude-opus-4-8": [ 8.35103668333333, 0.521036683, "div" ], "OpenHands/claude-sonnet-4-5": [ 3.4503085000000002, 0.2203085, "div" ], "OpenHands/claude-sonnet-4-6": [ 7.977369207692307, 1.497369208, "div" ] }, "t0": 6, "t1": 7, "t2": 2, "undef": 0 }, "C_mean_zero": { "div": 3, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.5705034632499999, 0.000503463, 1 ], "OpenHands/DeepSeek-V4-Pro": [ 0.1577, 0.0, 0 ], "OpenHands/GLM-4.7": [ 1.64456316475, 0.004563165, 1 ], "OpenHands/GLM-5": [ 2.2327703318749994, 0.002770332, 1 ], "OpenHands/GLM-5.1": [ 5.306676151897009, -0.003323848, 1 ], "OpenHands/GPT-5.2": [ 4.695543825, -0.004456175, 1 ], "OpenHands/GPT-5.2-Codex": [ 5.49939954375, -0.000600456, 1 ], "OpenHands/GPT-5.4": [ 4.03890865625, -0.001091344, 1 ], "OpenHands/GPT-5.5": [ 5.562512249999999, 0.00251225, 1 ], "OpenHands/Gemini-3-Flash": [ 2.616020840625, -0.003979159, 1 ], "OpenHands/Gemini-3-Pro": [ 3.1842415499999994, 0.00424155, 1 ], "OpenHands/Gemini-3.1-Pro": [ 1.1797897874999999, -0.000210213, 1 ], "OpenHands/Gemini-3.5-Flash": [ 7.912143834375, 0.002143834, 1 ], "OpenHands/Kimi-K2-Thinking": [ 0.53736379375, -0.002636206, 1 ], "OpenHands/Kimi-K2.5": [ 1.2644054749999998, 0.004405475, 1 ], "OpenHands/Kimi-K2.6": [ 1.51895738625, -0.001042614, 1 ], "OpenHands/MiniMax-M2.1": [ 0.3791234849999998, -0.000876515, 1 ], "OpenHands/MiniMax-M2.5": [ 0.4910889599999999, 0.00108896, 1 ], "OpenHands/MiniMax-M2.7": [ 0.66945, -0.22315, "div" ], "OpenHands/MiniMax-M3": [ 0.61726545, -0.00273455, 1 ], "OpenHands/Nemotron-3-Nano": [ 0.032412195000000005, 0.002412195, 1 ], "OpenHands/Nemotron-3-Super": [ 1.0633875000000002, -0.1519125, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.01, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 1.0373696968750001, -0.002630303, 1 ], "OpenHands/Qwen3.5-Flash": [ 4.330900000000001, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 6.0885500000000015, -1.40505, "div" ], "OpenHands/Trinity-Large-Thinking": [ 1.196382094375, -0.003617906, 1 ], "OpenHands/claude-fable-5": [ 12.491801687500002, 0.001801688, 1 ], "OpenHands/claude-opus-4-5": [ 9.11331096875, 0.003310969, 1 ], "OpenHands/claude-opus-4-6": [ 7.692464421875001, 0.002464422, 1 ], "OpenHands/claude-opus-4-7": [ 5.690055609375, 5.5609e-05, 1 ], "OpenHands/claude-opus-4-8": [ 7.829096890624998, -0.000903109, 1 ], "OpenHands/claude-sonnet-4-5": [ 3.2346642187500003, 0.004664219, 1 ], "OpenHands/claude-sonnet-4-6": [ 6.48161248125, 0.001612481, 1 ] }, "t0": 3, "t1": 28, "t2": 0, "undef": 0 }, "C_median_excl": { "div": 30, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.691089784, 0.121089784, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.08841824108404986, -0.069281759, "div" ], "OpenHands/GLM-4.7": [ 1.7564115455, 0.116411546, "div" ], "OpenHands/GLM-5": [ 3.453807429999998, 1.22380743, "div" ], "OpenHands/GLM-5.1": [ 5.636628740392158, 0.32662874, "div" ], "OpenHands/GPT-5.2": [ 3.7721316499999995, -0.92786835, "div" ], "OpenHands/GPT-5.2-Codex": [ 5.855042025000002, 0.355042025, "div" ], "OpenHands/GPT-5.4": [ 3.72321975, -0.31678025, "div" ], "OpenHands/GPT-5.5": [ 4.513803000000001, -1.046197, "div" ], "OpenHands/Gemini-3-Flash": [ 2.6196749749999992, -0.000325025, 1 ], "OpenHands/Gemini-3-Pro": [ 3.328677099999999, 0.1486771, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.5844050999999997, 1.4044051, "div" ], "OpenHands/Gemini-3.5-Flash": [ 7.514200799999995, -0.3957992, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.8943704000000001, 0.3543704, "div" ], "OpenHands/Kimi-K2.5": [ 1.0266720000000003, -0.233328, "div" ], "OpenHands/Kimi-K2.6": [ 1.5478230450000003, 0.027823045, 2 ], "OpenHands/MiniMax-M2.1": [ 0.31107717, -0.06892283, "div" ], "OpenHands/MiniMax-M2.5": [ 0.2915589599999999, -0.19844104, "div" ], "OpenHands/MiniMax-M2.7": [ 0.7662642046554636, -0.126335795, "div" ], "OpenHands/MiniMax-M3": [ 0.7627668, 0.1427668, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.0008373599999999999, -0.02916264, 2 ], "OpenHands/Nemotron-3-Super": [ 1.0505311919034166, -0.164768808, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.01, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 0.79959335, -0.24040665, "div" ], "OpenHands/Qwen3.5-Flash": [ 3.969369126790191, -0.361530873, "div" ], "OpenHands/Qwen3.6-Plus": [ 8.431154349336516, 0.937554349, "div" ], "OpenHands/Trinity-Large-Thinking": [ 1.4588865349999998, 0.258886535, "div" ], "OpenHands/claude-fable-5": [ 7.191163, -5.298837, "div" ], "OpenHands/claude-opus-4-5": [ 9.730220250000002, 0.62022025, "div" ], "OpenHands/claude-opus-4-6": [ 7.412590375000001, -0.277409625, "div" ], "OpenHands/claude-opus-4-7": [ 5.306776875000001, -0.383223125, "div" ], "OpenHands/claude-opus-4-8": [ 5.07949625, -2.75050375, "div" ], "OpenHands/claude-sonnet-4-5": [ 3.29052795, 0.06052795, "div" ], "OpenHands/claude-sonnet-4-6": [ 5.2762765499999995, -1.20372345, "div" ] }, "t0": 1, "t1": 1, "t2": 2, "undef": 0 }, "C_median_zero": { "div": 27, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.6299686540000001, 0.059968654, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.08841824108404986, -0.069281759, "div" ], "OpenHands/GLM-4.7": [ 1.4090156399999996, -0.23098436, "div" ], "OpenHands/GLM-5": [ 0.962089455, -1.267910545, "div" ], "OpenHands/GLM-5.1": [ 4.261745790000001, -1.04825421, "div" ], "OpenHands/GPT-5.2": [ 3.3793063500000002, -1.32069365, "div" ], "OpenHands/GPT-5.2-Codex": [ 5.4550865250000005, -0.044913475, 2 ], "OpenHands/GPT-5.4": [ 3.72321975, -0.31678025, "div" ], "OpenHands/GPT-5.5": [ 4.513803000000001, -1.046197, "div" ], "OpenHands/Gemini-3-Flash": [ 2.5539510499999993, -0.06604895, "div" ], "OpenHands/Gemini-3-Pro": [ 3.328677099999999, 0.1486771, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.0, -1.18, "div" ], "OpenHands/Gemini-3.5-Flash": [ 6.983484075, -0.926515925, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.5555886499999998, 0.01558865, 2 ], "OpenHands/Kimi-K2.5": [ 1.0253487000000003, -0.2346513, "div" ], "OpenHands/Kimi-K2.6": [ 1.5478230450000003, 0.027823045, 2 ], "OpenHands/MiniMax-M2.1": [ 0.31107717, -0.06892283, "div" ], "OpenHands/MiniMax-M2.5": [ 0.29146937999999994, -0.19853062, "div" ], "OpenHands/MiniMax-M2.7": [ 0.6081978960950565, -0.284402104, "div" ], "OpenHands/MiniMax-M3": [ 0.6575517000000002, 0.0375517, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.0, -0.03, 2 ], "OpenHands/Nemotron-3-Super": [ 0.9546393517210041, -0.260660648, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.01, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 0.79959335, -0.24040665, "div" ], "OpenHands/Qwen3.5-Flash": [ 3.969369126790191, -0.361530873, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.8001977400867761, -5.69340226, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.8059589949999998, -0.394041005, "div" ], "OpenHands/claude-fable-5": [ 7.191163, -5.298837, "div" ], "OpenHands/claude-opus-4-5": [ 7.970758, -1.139242, "div" ], "OpenHands/claude-opus-4-6": [ 5.242057624999999, -2.447942375, "div" ], "OpenHands/claude-opus-4-7": [ 3.9501605, -1.7398395, "div" ], "OpenHands/claude-opus-4-8": [ 5.050555375, -2.779444625, "div" ], "OpenHands/claude-sonnet-4-5": [ 3.2190654, -0.0109346, 2 ], "OpenHands/claude-sonnet-4-6": [ 4.611834, -1.868166, "div" ] }, "t0": 1, "t1": 0, "t2": 6, "undef": 0 }, "C_sum_over_resolved": { "div": 33, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 2.2820138529999996, 1.712013853, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 1.2616, 1.1039, "div" ], "OpenHands/GLM-4.7": [ 13.156505318, 11.516505318, "div" ], "OpenHands/GLM-5": [ 7.144865061999998, 4.914865062, "div" ], "OpenHands/GLM-5.1": [ 14.151136405058692, 8.841136405, "div" ], "OpenHands/GPT-5.2": [ 9.39108765, 4.69108765, "div" ], "OpenHands/GPT-5.2-Codex": [ 12.5700561, 7.0700561, "div" ], "OpenHands/GPT-5.4": [ 7.180282055555556, 3.140282056, "div" ], "OpenHands/GPT-5.5": [ 12.714313714285712, 7.154313714, "div" ], "OpenHands/Gemini-3-Flash": [ 13.95211115, 11.33211115, "div" ], "OpenHands/Gemini-3-Pro": [ 12.736966199999998, 9.5569662, "div" ], "OpenHands/Gemini-3.1-Pro": [ 4.719159149999999, 3.53915915, "div" ], "OpenHands/Gemini-3.5-Flash": [ 21.099050225, 13.189050225, "div" ], "OpenHands/Kimi-K2-Thinking": [ 4.29891035, 3.75891035, "div" ], "OpenHands/Kimi-K2.5": [ 6.743495866666666, 5.483495867, "div" ], "OpenHands/Kimi-K2.6": [ 6.075829545, 4.555829545, "div" ], "OpenHands/MiniMax-M2.1": [ 2.021991919999999, 1.64199192, "div" ], "OpenHands/MiniMax-M2.5": [ 3.9287116799999993, 3.43871168, "div" ], "OpenHands/MiniMax-M2.7": [ 3.5704, 2.6778, "div" ], "OpenHands/MiniMax-M3": [ 2.4690618, 1.8490618, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.5185951200000001, 0.48859512, "div" ], "OpenHands/Nemotron-3-Super": [ 8.507100000000001, 7.2918, "div" ], "OpenHands/Qwen3-Coder-480B": [ null, null, null ], "OpenHands/Qwen3-Coder-Next": [ 4.1494787875000005, 3.109478788, "div" ], "OpenHands/Qwen3.5-Flash": [ 34.647200000000005, 30.3163, "div" ], "OpenHands/Qwen3.6-Plus": [ 24.354200000000006, 16.8606, "div" ], "OpenHands/Trinity-Large-Thinking": [ 9.571056755, 8.371056755, "div" ], "OpenHands/claude-fable-5": [ 19.986882700000002, 7.4968827, "div" ], "OpenHands/claude-opus-4-5": [ 24.30216258333333, 15.192162583, "div" ], "OpenHands/claude-opus-4-6": [ 13.675492305555558, 5.985492306, "div" ], "OpenHands/claude-opus-4-7": [ 10.115654416666667, 4.425654417, "div" ], "OpenHands/claude-opus-4-8": [ 12.526555024999997, 4.696555025, "div" ], "OpenHands/claude-sonnet-4-5": [ 25.877313750000003, 22.64731375, "div" ], "OpenHands/claude-sonnet-4-6": [ 12.9632249625, 6.483224962, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 1 } }, "primary": "C_mean_excl", "published_range": [ 0.01, 12.49 ], "zero_variance": false }, "greenfield_score": { "dp": 1, "family": "score", "max_abs_delta": 0.05, "max_abs_rows": [ "OpenHands/GLM-5", "OpenHands/GPT-5.2-Codex", "OpenHands/GPT-5.4", "OpenHands/GPT-5.5", "OpenHands/Gemini-3-Flash", "OpenHands/Kimi-K2.5", "OpenHands/MiniMax-M2.1", "OpenHands/MiniMax-M2.7", "OpenHands/Nemotron-3-Nano", "OpenHands/claude-opus-4-6", "OpenHands/claude-opus-4-7" ], "policies": { "S1_frac_Dall": { "div": 33, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.25, -24.75, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.125, -12.375, "div" ], "OpenHands/GLM-4.7": [ 0.125, -12.375, "div" ], "OpenHands/GLM-5": [ 0.3125, -30.8875, "div" ], "OpenHands/GLM-5.1": [ 0.375, -37.125, "div" ], "OpenHands/GPT-5.2": [ 0.5, -49.5, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.4375, -43.3625, "div" ], "OpenHands/GPT-5.4": [ 0.5625, -55.6375, "div" ], "OpenHands/GPT-5.5": [ 0.4375, -43.3625, "div" ], "OpenHands/Gemini-3-Flash": [ 0.1875, -18.6125, "div" ], "OpenHands/Gemini-3-Pro": [ 0.25, -24.75, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.25, -24.75, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.375, -37.125, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.125, -12.375, "div" ], "OpenHands/Kimi-K2.5": [ 0.1875, -18.6125, "div" ], "OpenHands/Kimi-K2.6": [ 0.25, -24.75, "div" ], "OpenHands/MiniMax-M2.1": [ 0.1875, -18.6125, "div" ], "OpenHands/MiniMax-M2.5": [ 0.125, -12.375, "div" ], "OpenHands/MiniMax-M2.7": [ 0.1875, -18.6125, "div" ], "OpenHands/MiniMax-M3": [ 0.25, -24.75, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.0625, -6.1375, "div" ], "OpenHands/Nemotron-3-Super": [ 0.125, -12.375, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 0.25, -24.75, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.125, -12.375, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.25, -24.75, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.125, -12.375, "div" ], "OpenHands/claude-fable-5": [ 0.625, -61.875, "div" ], "OpenHands/claude-opus-4-5": [ 0.375, -37.125, "div" ], "OpenHands/claude-opus-4-6": [ 0.5625, -55.6375, "div" ], "OpenHands/claude-opus-4-7": [ 0.5625, -55.6375, "div" ], "OpenHands/claude-opus-4-8": [ 0.625, -61.875, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.125, -12.375, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.5, -49.5, "div" ] }, "t0": 1, "t1": 0, "t2": 0, "undef": 0 }, "S1_pct_Dall": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 25.0, 0.0, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 12.5, 0.0, 0 ], "OpenHands/GLM-4.7": [ 12.5, 0.0, 0 ], "OpenHands/GLM-5": [ 31.25, 0.05, 0 ], "OpenHands/GLM-5.1": [ 37.5, 0.0, 0 ], "OpenHands/GPT-5.2": [ 50.0, 0.0, 0 ], "OpenHands/GPT-5.2-Codex": [ 43.75, -0.05, 0 ], "OpenHands/GPT-5.4": [ 56.25, 0.05, 0 ], "OpenHands/GPT-5.5": [ 43.75, -0.05, 0 ], "OpenHands/Gemini-3-Flash": [ 18.75, -0.05, 0 ], "OpenHands/Gemini-3-Pro": [ 25.0, 0.0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 25.0, 0.0, 0 ], "OpenHands/Gemini-3.5-Flash": [ 37.5, 0.0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 12.5, 0.0, 0 ], "OpenHands/Kimi-K2.5": [ 18.75, -0.05, 0 ], "OpenHands/Kimi-K2.6": [ 25.0, 0.0, 0 ], "OpenHands/MiniMax-M2.1": [ 18.75, -0.05, 0 ], "OpenHands/MiniMax-M2.5": [ 12.5, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 18.75, -0.05, 0 ], "OpenHands/MiniMax-M3": [ 25.0, 0.0, 0 ], "OpenHands/Nemotron-3-Nano": [ 6.25, 0.05, 0 ], "OpenHands/Nemotron-3-Super": [ 12.5, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 25.0, 0.0, 0 ], "OpenHands/Qwen3.5-Flash": [ 12.5, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 25.0, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 12.5, 0.0, 0 ], "OpenHands/claude-fable-5": [ 62.5, 0.0, 0 ], "OpenHands/claude-opus-4-5": [ 37.5, 0.0, 0 ], "OpenHands/claude-opus-4-6": [ 56.25, 0.05, 0 ], "OpenHands/claude-opus-4-7": [ 56.25, 0.05, 0 ], "OpenHands/claude-opus-4-8": [ 62.5, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 12.5, 0.0, 0 ], "OpenHands/claude-sonnet-4-6": [ 50.0, 0.0, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 }, "S2_frac_Dobs": { "div": 33, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.25, -24.75, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.125, -12.375, "div" ], "OpenHands/GLM-4.7": [ 0.125, -12.375, "div" ], "OpenHands/GLM-5": [ 0.3125, -30.8875, "div" ], "OpenHands/GLM-5.1": [ 0.375, -37.125, "div" ], "OpenHands/GPT-5.2": [ 0.5, -49.5, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.4375, -43.3625, "div" ], "OpenHands/GPT-5.4": [ 0.5625, -55.6375, "div" ], "OpenHands/GPT-5.5": [ 0.4375, -43.3625, "div" ], "OpenHands/Gemini-3-Flash": [ 0.1875, -18.6125, "div" ], "OpenHands/Gemini-3-Pro": [ 0.25, -24.75, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.25, -24.75, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.375, -37.125, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.125, -12.375, "div" ], "OpenHands/Kimi-K2.5": [ 0.1875, -18.6125, "div" ], "OpenHands/Kimi-K2.6": [ 0.25, -24.75, "div" ], "OpenHands/MiniMax-M2.1": [ 0.1875, -18.6125, "div" ], "OpenHands/MiniMax-M2.5": [ 0.125, -12.375, "div" ], "OpenHands/MiniMax-M2.7": [ 0.1875, -18.6125, "div" ], "OpenHands/MiniMax-M3": [ 0.25, -24.75, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.0625, -6.1375, "div" ], "OpenHands/Nemotron-3-Super": [ 0.125, -12.375, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 0.25, -24.75, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.125, -12.375, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.25, -24.75, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.125, -12.375, "div" ], "OpenHands/claude-fable-5": [ 0.625, -61.875, "div" ], "OpenHands/claude-opus-4-5": [ 0.375, -37.125, "div" ], "OpenHands/claude-opus-4-6": [ 0.5625, -55.6375, "div" ], "OpenHands/claude-opus-4-7": [ 0.5625, -55.6375, "div" ], "OpenHands/claude-opus-4-8": [ 0.625, -61.875, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.125, -12.375, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.5, -49.5, "div" ] }, "t0": 1, "t1": 0, "t2": 0, "undef": 0 }, "S2_pct_Dobs": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 25.0, 0.0, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 12.5, 0.0, 0 ], "OpenHands/GLM-4.7": [ 12.5, 0.0, 0 ], "OpenHands/GLM-5": [ 31.25, 0.05, 0 ], "OpenHands/GLM-5.1": [ 37.5, 0.0, 0 ], "OpenHands/GPT-5.2": [ 50.0, 0.0, 0 ], "OpenHands/GPT-5.2-Codex": [ 43.75, -0.05, 0 ], "OpenHands/GPT-5.4": [ 56.25, 0.05, 0 ], "OpenHands/GPT-5.5": [ 43.75, -0.05, 0 ], "OpenHands/Gemini-3-Flash": [ 18.75, -0.05, 0 ], "OpenHands/Gemini-3-Pro": [ 25.0, 0.0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 25.0, 0.0, 0 ], "OpenHands/Gemini-3.5-Flash": [ 37.5, 0.0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 12.5, 0.0, 0 ], "OpenHands/Kimi-K2.5": [ 18.75, -0.05, 0 ], "OpenHands/Kimi-K2.6": [ 25.0, 0.0, 0 ], "OpenHands/MiniMax-M2.1": [ 18.75, -0.05, 0 ], "OpenHands/MiniMax-M2.5": [ 12.5, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 18.75, -0.05, 0 ], "OpenHands/MiniMax-M3": [ 25.0, 0.0, 0 ], "OpenHands/Nemotron-3-Nano": [ 6.25, 0.05, 0 ], "OpenHands/Nemotron-3-Super": [ 12.5, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 25.0, 0.0, 0 ], "OpenHands/Qwen3.5-Flash": [ 12.5, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 25.0, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 12.5, 0.0, 0 ], "OpenHands/claude-fable-5": [ 62.5, 0.0, 0 ], "OpenHands/claude-opus-4-5": [ 37.5, 0.0, 0 ], "OpenHands/claude-opus-4-6": [ 56.25, 0.05, 0 ], "OpenHands/claude-opus-4-7": [ 56.25, 0.05, 0 ], "OpenHands/claude-opus-4-8": [ 62.5, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 12.5, 0.0, 0 ], "OpenHands/claude-sonnet-4-6": [ 50.0, 0.0, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "S1_pct_Dall", "published_range": [ 0.0, 62.5 ], "zero_variance": false }, "id": { "dp": null, "family": "join_key", "max_abs_delta": null, "max_abs_rows": [], "policies": {}, "primary": null, "published_range": null, "zero_variance": false }, "information_gathering_cost": { "dp": 4, "family": "cost", "max_abs_delta": 0.280643806, "max_abs_rows": [ "OpenHands/Gemini-3.1-Pro" ], "policies": { "C_mean_excl": { "div": 9, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.06528059332450332, 0.005280593, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.15939999999999996, -0.0, 0 ], "OpenHands/GLM-4.7": [ 0.17975775089808912, 0.029757751, 2 ], "OpenHands/GLM-5": [ 0.3714264495597483, 0.01142645, 2 ], "OpenHands/GLM-5.1": [ 0.4576684416330742, -0.012331558, 2 ], "OpenHands/GPT-5.2": [ 0.48936452024539867, 0.00936452, 2 ], "OpenHands/GPT-5.2-Codex": [ 0.6306687190184049, 0.080668719, "div" ], "OpenHands/GPT-5.4": [ 0.5534169754601227, -0.056583025, "div" ], "OpenHands/GPT-5.5": [ 0.6811753481012657, -0.058824652, "div" ], "OpenHands/Gemini-3-Flash": [ 0.4046947438311689, 0.024694744, 2 ], "OpenHands/Gemini-3-Pro": [ 0.60988629689441, 0.109886297, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.080643806060608, 0.280643806, "div" ], "OpenHands/Gemini-3.5-Flash": [ 2.7064906628289465, 0.076490663, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.68039276137931, 0.030392761, 2 ], "OpenHands/Kimi-K2.5": [ 0.37810000000000005, 0.0, 0 ], "OpenHands/Kimi-K2.6": [ 0.410249277160494, -0.009750723, 2 ], "OpenHands/MiniMax-M2.1": [ 0.277392605487805, 0.030292605, 2 ], "OpenHands/MiniMax-M2.5": [ 0.058699999999999995, -0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.21630000000000005, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.31720274810126575, -0.032797252, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.04440001653061222, 0.004400017, 1 ], "OpenHands/Nemotron-3-Super": [ 0.12060000000000005, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.30721981603053433, 0.027219816, 2 ], "OpenHands/Qwen3-Coder-Next": [ 0.2113223853146853, 0.081322385, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.3422, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 0.33769999999999994, -0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.2848449884459461, -0.065155012, "div" ], "OpenHands/claude-fable-5": [ 7.9061990181818205, -0.003800982, 1 ], "OpenHands/claude-opus-4-5": [ 0.548708294303797, -0.001291706, 1 ], "OpenHands/claude-opus-4-6": [ 0.45949041149068326, 0.019490411, 2 ], "OpenHands/claude-opus-4-7": [ 0.789040108490566, -0.100959892, "div" ], "OpenHands/claude-opus-4-8": [ 1.1605617393292686, -0.009438261, 2 ], "OpenHands/claude-sonnet-4-5": [ 0.8742473861963187, 0.004247386, 1 ], "OpenHands/claude-sonnet-4-6": [ 0.40678044078947373, -0.003219559, 1 ] }, "t0": 7, "t1": 5, "t2": 13, "undef": 0 }, "C_mean_zero": { "div": 10, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.05974163389090911, -0.000258366, 1 ], "OpenHands/DeepSeek-V4-Pro": [ 0.15939999999999996, -0.0, 0 ], "OpenHands/GLM-4.7": [ 0.17104222358181814, 0.021042224, 2 ], "OpenHands/GLM-5": [ 0.35792003321212107, -0.002079967, 1 ], "OpenHands/GLM-5.1": [ 0.4160622196664311, -0.05393778, "div" ], "OpenHands/GPT-5.2": [ 0.483432829090909, 0.003432829, 1 ], "OpenHands/GPT-5.2-Codex": [ 0.6230242496969698, 0.07302425, "div" ], "OpenHands/GPT-5.4": [ 0.5467088909090909, -0.063291109, "div" ], "OpenHands/GPT-5.5": [ 0.6522769999999999, -0.087723, "div" ], "OpenHands/Gemini-3-Flash": [ 0.3777150942424243, -0.002284906, 1 ], "OpenHands/Gemini-3-Pro": [ 0.5951011745454546, 0.095101175, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.080643806060608, 0.280643806, "div" ], "OpenHands/Gemini-3.5-Flash": [ 2.6037125363924045, -0.026287464, 2 ], "OpenHands/Kimi-K2-Thinking": [ 0.5979209115151513, -0.052079088, "div" ], "OpenHands/Kimi-K2.5": [ 0.37810000000000005, 0.0, 0 ], "OpenHands/Kimi-K2.6": [ 0.40279019939393956, -0.017209801, 2 ], "OpenHands/MiniMax-M2.1": [ 0.2757114381818183, 0.028611438, 2 ], "OpenHands/MiniMax-M2.5": [ 0.058699999999999995, -0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.21630000000000005, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.30374566181818174, -0.046254338, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.043512016199999975, 0.003512016, 1 ], "OpenHands/Nemotron-3-Super": [ 0.11986909090909097, -0.000730909, 1 ], "OpenHands/Qwen3-Coder-480B": [ 0.24391391454545452, -0.036086085, 2 ], "OpenHands/Qwen3-Coder-Next": [ 0.18314606727272728, 0.053146067, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.3401260606060606, -0.002073939, 1 ], "OpenHands/Qwen3.6-Plus": [ 0.33565333333333325, -0.002046667, 1 ], "OpenHands/Trinity-Large-Thinking": [ 0.25549732296969707, -0.094502677, "div" ], "OpenHands/claude-fable-5": [ 7.9061990181818205, -0.003800982, 1 ], "OpenHands/claude-opus-4-5": [ 0.5254297606060602, -0.024570239, 2 ], "OpenHands/claude-opus-4-6": [ 0.44835125000000003, 0.00835125, 2 ], "OpenHands/claude-opus-4-7": [ 0.7603477409090909, -0.129652259, "div" ], "OpenHands/claude-opus-4-8": [ 1.1535280318181822, -0.016471968, 2 ], "OpenHands/claude-sonnet-4-5": [ 0.8636504481818179, -0.006349552, 2 ], "OpenHands/claude-sonnet-4-6": [ 0.3747310727272728, -0.035268927, 2 ] }, "t0": 4, "t1": 9, "t2": 11, "undef": 0 }, "C_median_excl": { "div": 30, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.05022304, -0.00977696, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.08930843640368438, -0.070091564, "div" ], "OpenHands/GLM-4.7": [ 0.066313125, -0.083686875, "div" ], "OpenHands/GLM-5": [ 0.0881928, -0.2718072, "div" ], "OpenHands/GLM-5.1": [ 0.17050630500000002, -0.299493695, "div" ], "OpenHands/GPT-5.2": [ 0.18107705000000002, -0.29892295, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.20562885, -0.34437115, "div" ], "OpenHands/GPT-5.4": [ 0.28382199999999996, -0.326178, "div" ], "OpenHands/GPT-5.5": [ 0.4245055, -0.3154945, "div" ], "OpenHands/Gemini-3-Flash": [ 0.312177675, -0.067822325, "div" ], "OpenHands/Gemini-3-Pro": [ 0.367967, -0.132033, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.43093200000000004, -1.369068, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.59361365, -1.03638635, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.27893199999999996, -0.371068, "div" ], "OpenHands/Kimi-K2.5": [ 0.18760170611924032, -0.190498294, "div" ], "OpenHands/Kimi-K2.6": [ 0.20976899, -0.21023101, "div" ], "OpenHands/MiniMax-M2.1": [ 0.09799440000000001, -0.1491056, "div" ], "OpenHands/MiniMax-M2.5": [ 0.028846871163512422, -0.029853129, 2 ], "OpenHands/MiniMax-M2.7": [ 0.07672765166930572, -0.139572348, "div" ], "OpenHands/MiniMax-M3": [ 0.06569910000000001, -0.2843009, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.009388979999999998, -0.03061102, 2 ], "OpenHands/Nemotron-3-Super": [ 0.03201666760098899, -0.088583332, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.11246984999999998, -0.16753015, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.10975399999999999, -0.020246, 2 ], "OpenHands/Qwen3.5-Flash": [ 0.23287494564312872, -0.109325054, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.10692821199667121, -0.230771788, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.11773562, -0.23226438, "div" ], "OpenHands/claude-fable-5": [ 0.7165235, -7.1934765, "div" ], "OpenHands/claude-opus-4-5": [ 0.305778625, -0.244221375, "div" ], "OpenHands/claude-opus-4-6": [ 0.15049275, -0.28950725, "div" ], "OpenHands/claude-opus-4-7": [ 0.32159925000000006, -0.56840075, "div" ], "OpenHands/claude-opus-4-8": [ 0.387172125, -0.782827875, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.6312273, -0.2387727, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.15689437500000003, -0.253105625, "div" ] }, "t0": 0, "t1": 0, "t2": 4, "undef": 0 }, "C_median_zero": { "div": 30, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.042831768, -0.017168232, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.08930843640368438, -0.070091564, "div" ], "OpenHands/GLM-4.7": [ 0.06217917300000001, -0.087820827, "div" ], "OpenHands/GLM-5": [ 0.07900104999999999, -0.28099895, "div" ], "OpenHands/GLM-5.1": [ 0.13467588, -0.33532412, "div" ], "OpenHands/GPT-5.2": [ 0.17996545, -0.30003455, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.20450219999999997, -0.3454978, "div" ], "OpenHands/GPT-5.4": [ 0.27178, -0.33822, "div" ], "OpenHands/GPT-5.5": [ 0.40503, -0.33497, "div" ], "OpenHands/Gemini-3-Flash": [ 0.29892845, -0.08107155, "div" ], "OpenHands/Gemini-3-Pro": [ 0.3611848, -0.1388152, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.43093200000000004, -1.369068, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.5277238250000003, -1.102276175, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.1949251, -0.4550749, "div" ], "OpenHands/Kimi-K2.5": [ 0.18760170611924032, -0.190498294, "div" ], "OpenHands/Kimi-K2.6": [ 0.20524272000000005, -0.21475728, "div" ], "OpenHands/MiniMax-M2.1": [ 0.09738959999999999, -0.1497104, "div" ], "OpenHands/MiniMax-M2.5": [ 0.028846871163512422, -0.029853129, 2 ], "OpenHands/MiniMax-M2.7": [ 0.07672765166930572, -0.139572348, "div" ], "OpenHands/MiniMax-M3": [ 0.0519438, -0.2980562, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.009279749999999996, -0.03072025, 2 ], "OpenHands/Nemotron-3-Super": [ 0.031725836911757725, -0.088874163, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.07856685, -0.20143315, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.08378539999999998, -0.0462146, 2 ], "OpenHands/Qwen3.5-Flash": [ 0.23015216434002775, -0.112047836, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.10603267498941618, -0.231667325, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.08927792, -0.26072208, "div" ], "OpenHands/claude-fable-5": [ 0.7165235, -7.1934765, "div" ], "OpenHands/claude-opus-4-5": [ 0.27900600000000003, -0.270994, "div" ], "OpenHands/claude-opus-4-6": [ 0.14412925000000001, -0.29587075, "div" ], "OpenHands/claude-opus-4-7": [ 0.30366575, -0.58633425, "div" ], "OpenHands/claude-opus-4-8": [ 0.38697099999999995, -0.783029, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.6295092, -0.2404908, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.12842699999999999, -0.281573, "div" ] }, "t0": 0, "t1": 0, "t2": 4, "undef": 0 }, "C_sum_over_resolved": { "div": 32, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.118763489060241, 0.058763489, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 1.252428571428571, 1.093028571, "div" ], "OpenHands/GLM-4.7": [ 0.3171007515842696, 0.167100752, "div" ], "OpenHands/GLM-5": [ 0.5965333886868684, 0.236533389, "div" ], "OpenHands/GLM-5.1": [ 0.6184708670717219, 0.148470867, "div" ], "OpenHands/GPT-5.2": [ 0.7385779333333332, 0.258577933, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.878623941880342, 0.328623942, "div" ], "OpenHands/GPT-5.4": [ 0.6632865220588235, 0.053286522, "div" ], "OpenHands/GPT-5.5": [ 0.7579274999999999, 0.0179275, 2 ], "OpenHands/Gemini-3-Flash": [ 0.6425050572164949, 0.262505057, "div" ], "OpenHands/Gemini-3-Pro": [ 1.345091695890411, 0.845091696, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.5430090962962986, 0.743009096, "div" ], "OpenHands/Gemini-3.5-Flash": [ 3.070049110074626, 0.44004911, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.3702354222222217, 0.720235422, "div" ], "OpenHands/Kimi-K2.5": [ 0.5941571428571429, 0.216057143, "div" ], "OpenHands/Kimi-K2.6": [ 0.5403283162601629, 0.120328316, "div" ], "OpenHands/MiniMax-M2.1": [ 0.6789908552238809, 0.431890855, "div" ], "OpenHands/MiniMax-M2.5": [ 0.1226012658227848, 0.063901266, "div" ], "OpenHands/MiniMax-M2.7": [ 0.8497500000000002, 0.63345, "div" ], "OpenHands/MiniMax-M3": [ 0.4556184927272726, 0.105618493, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.31080011571428556, 0.270800116, "div" ], "OpenHands/Nemotron-3-Super": [ 0.2996727272727274, 0.179072727, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.7186749267857142, 0.438674927, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.3597512035714286, 0.229751204, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.6844, 0.3422, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.46540168067226884, 0.127701681, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.78068626462963, 0.430686265, "div" ], "OpenHands/claude-fable-5": [ 9.385056388489211, 1.475056388, "div" ], "OpenHands/claude-opus-4-5": [ 0.7604904429824555, 0.210490443, "div" ], "OpenHands/claude-opus-4-6": [ 0.5604390625000001, 0.120439063, "div" ], "OpenHands/claude-opus-4-7": [ 0.9362490839552239, 0.046249084, 2 ], "OpenHands/claude-opus-4-8": [ 1.4640932711538466, 0.294093271, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.1875193662499997, 0.317519366, "div" ], "OpenHands/claude-sonnet-4-6": [ 2.810483045454546, 2.400483045, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 0 } }, "primary": "C_mean_excl", "published_range": [ 0.04, 7.91 ], "zero_variance": false }, "information_gathering_score": { "dp": 1, "family": "score", "max_abs_delta": 5.5, "max_abs_rows": [ "OpenHands/Nemotron-3-Nano" ], "policies": { "S1_frac_Dall": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.503030303030303, -49.796969697, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.12727272727272726, -12.572727273, "div" ], "OpenHands/GLM-4.7": [ 0.5393939393939394, -53.360606061, "div" ], "OpenHands/GLM-5": [ 0.6, -59.4, "div" ], "OpenHands/GLM-5.1": [ 0.6727272727272727, -66.627272727, "div" ], "OpenHands/GPT-5.2": [ 0.6545454545454545, -64.845454545, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.7090909090909091, -70.190909091, "div" ], "OpenHands/GPT-5.4": [ 0.8242424242424242, -81.575757576, "div" ], "OpenHands/GPT-5.5": [ 0.8606060606060606, -85.239393939, "div" ], "OpenHands/Gemini-3-Flash": [ 0.5878787878787879, -58.212121212, "div" ], "OpenHands/Gemini-3-Pro": [ 0.44242424242424244, -43.757575758, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.8181818181818182, -80.981818182, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.8481012658227848, -80.351898734, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.43636363636363634, -43.163636364, "div" ], "OpenHands/Kimi-K2.5": [ 0.6363636363636364, -62.963636364, "div" ], "OpenHands/Kimi-K2.6": [ 0.7454545454545455, -73.754545455, "div" ], "OpenHands/MiniMax-M2.1": [ 0.40606060606060607, -40.193939394, "div" ], "OpenHands/MiniMax-M2.5": [ 0.47878787878787876, -47.421212121, "div" ], "OpenHands/MiniMax-M2.7": [ 0.2545454545454545, -25.245454545, "div" ], "OpenHands/MiniMax-M3": [ 0.6666666666666666, -66.033333333, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.14, -8.36, "div" ], "OpenHands/Nemotron-3-Super": [ 0.4, -39.6, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.3393939393939394, -33.560606061, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.509090909090909, -50.390909091, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.49696969696969695, -49.203030303, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.7212121212121212, -71.378787879, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.32727272727272727, -32.372727273, "div" ], "OpenHands/claude-fable-5": [ 0.8424242424242424, -83.357575758, "div" ], "OpenHands/claude-opus-4-5": [ 0.6909090909090909, -68.409090909, "div" ], "OpenHands/claude-opus-4-6": [ 0.8, -79.2, "div" ], "OpenHands/claude-opus-4-7": [ 0.8121212121212121, -80.387878788, "div" ], "OpenHands/claude-opus-4-8": [ 0.7878787878787878, -78.012121212, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.7272727272727273, -71.972727273, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.13333333333333333, -13.166666667, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S1_pct_Dall": { "div": 2, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 50.303030303030305, 0.003030303, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 12.727272727272727, 0.027272727, 0 ], "OpenHands/GLM-4.7": [ 53.93939393939394, 0.039393939, 0 ], "OpenHands/GLM-5": [ 60.0, 0.0, 0 ], "OpenHands/GLM-5.1": [ 67.27272727272727, -0.027272727, 0 ], "OpenHands/GPT-5.2": [ 65.45454545454545, -0.045454545, 0 ], "OpenHands/GPT-5.2-Codex": [ 70.9090909090909, 0.009090909, 0 ], "OpenHands/GPT-5.4": [ 82.42424242424242, 0.024242424, 0 ], "OpenHands/GPT-5.5": [ 86.06060606060606, -0.039393939, 0 ], "OpenHands/Gemini-3-Flash": [ 58.78787878787879, -0.012121212, 0 ], "OpenHands/Gemini-3-Pro": [ 44.24242424242424, 0.042424242, 0 ], "OpenHands/Gemini-3.1-Pro": [ 81.81818181818181, 0.018181818, 0 ], "OpenHands/Gemini-3.5-Flash": [ 84.81012658227849, 3.610126582, "div" ], "OpenHands/Kimi-K2-Thinking": [ 43.63636363636363, 0.036363636, 0 ], "OpenHands/Kimi-K2.5": [ 63.63636363636363, 0.036363636, 0 ], "OpenHands/Kimi-K2.6": [ 74.54545454545455, 0.045454545, 0 ], "OpenHands/MiniMax-M2.1": [ 40.60606060606061, 0.006060606, 0 ], "OpenHands/MiniMax-M2.5": [ 47.878787878787875, -0.021212121, 0 ], "OpenHands/MiniMax-M2.7": [ 25.454545454545453, -0.045454545, 0 ], "OpenHands/MiniMax-M3": [ 66.66666666666667, -0.033333333, 0 ], "OpenHands/Nemotron-3-Nano": [ 14.0, 5.5, "div" ], "OpenHands/Nemotron-3-Super": [ 40.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 33.93939393939394, 0.039393939, 0 ], "OpenHands/Qwen3-Coder-Next": [ 50.90909090909091, 0.009090909, 0 ], "OpenHands/Qwen3.5-Flash": [ 49.696969696969695, -0.003030303, 0 ], "OpenHands/Qwen3.6-Plus": [ 72.12121212121212, 0.021212121, 0 ], "OpenHands/Trinity-Large-Thinking": [ 32.72727272727273, 0.027272727, 0 ], "OpenHands/claude-fable-5": [ 84.24242424242425, 0.042424242, 0 ], "OpenHands/claude-opus-4-5": [ 69.0909090909091, -0.009090909, 0 ], "OpenHands/claude-opus-4-6": [ 80.0, 0.0, 0 ], "OpenHands/claude-opus-4-7": [ 81.21212121212122, 0.012121212, 0 ], "OpenHands/claude-opus-4-8": [ 78.78787878787878, -0.012121212, 0 ], "OpenHands/claude-sonnet-4-5": [ 72.72727272727273, 0.027272727, 0 ], "OpenHands/claude-sonnet-4-6": [ 13.333333333333334, 0.033333333, 0 ] }, "t0": 32, "t1": 0, "t2": 0, "undef": 0 }, "S2_frac_Dobs": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.503030303030303, -49.796969697, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.12727272727272726, -12.572727273, "div" ], "OpenHands/GLM-4.7": [ 0.5393939393939394, -53.360606061, "div" ], "OpenHands/GLM-5": [ 0.6, -59.4, "div" ], "OpenHands/GLM-5.1": [ 0.6727272727272727, -66.627272727, "div" ], "OpenHands/GPT-5.2": [ 0.6545454545454545, -64.845454545, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.7090909090909091, -70.190909091, "div" ], "OpenHands/GPT-5.4": [ 0.8242424242424242, -81.575757576, "div" ], "OpenHands/GPT-5.5": [ 0.8606060606060606, -85.239393939, "div" ], "OpenHands/Gemini-3-Flash": [ 0.5878787878787879, -58.212121212, "div" ], "OpenHands/Gemini-3-Pro": [ 0.44242424242424244, -43.757575758, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.8181818181818182, -80.981818182, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.8481012658227848, -80.351898734, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.43636363636363634, -43.163636364, "div" ], "OpenHands/Kimi-K2.5": [ 0.6363636363636364, -62.963636364, "div" ], "OpenHands/Kimi-K2.6": [ 0.7454545454545455, -73.754545455, "div" ], "OpenHands/MiniMax-M2.1": [ 0.40606060606060607, -40.193939394, "div" ], "OpenHands/MiniMax-M2.5": [ 0.47878787878787876, -47.421212121, "div" ], "OpenHands/MiniMax-M2.7": [ 0.2545454545454545, -25.245454545, "div" ], "OpenHands/MiniMax-M3": [ 0.6666666666666666, -66.033333333, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.14, -8.36, "div" ], "OpenHands/Nemotron-3-Super": [ 0.4, -39.6, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.3393939393939394, -33.560606061, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.509090909090909, -50.390909091, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.49696969696969695, -49.203030303, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.7212121212121212, -71.378787879, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.32727272727272727, -32.372727273, "div" ], "OpenHands/claude-fable-5": [ 0.8424242424242424, -83.357575758, "div" ], "OpenHands/claude-opus-4-5": [ 0.6909090909090909, -68.409090909, "div" ], "OpenHands/claude-opus-4-6": [ 0.8, -79.2, "div" ], "OpenHands/claude-opus-4-7": [ 0.8121212121212121, -80.387878788, "div" ], "OpenHands/claude-opus-4-8": [ 0.7878787878787878, -78.012121212, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.7272727272727273, -71.972727273, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.13333333333333333, -13.166666667, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S2_pct_Dobs": { "div": 2, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 50.303030303030305, 0.003030303, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 12.727272727272727, 0.027272727, 0 ], "OpenHands/GLM-4.7": [ 53.93939393939394, 0.039393939, 0 ], "OpenHands/GLM-5": [ 60.0, 0.0, 0 ], "OpenHands/GLM-5.1": [ 67.27272727272727, -0.027272727, 0 ], "OpenHands/GPT-5.2": [ 65.45454545454545, -0.045454545, 0 ], "OpenHands/GPT-5.2-Codex": [ 70.9090909090909, 0.009090909, 0 ], "OpenHands/GPT-5.4": [ 82.42424242424242, 0.024242424, 0 ], "OpenHands/GPT-5.5": [ 86.06060606060606, -0.039393939, 0 ], "OpenHands/Gemini-3-Flash": [ 58.78787878787879, -0.012121212, 0 ], "OpenHands/Gemini-3-Pro": [ 44.24242424242424, 0.042424242, 0 ], "OpenHands/Gemini-3.1-Pro": [ 81.81818181818181, 0.018181818, 0 ], "OpenHands/Gemini-3.5-Flash": [ 84.81012658227849, 3.610126582, "div" ], "OpenHands/Kimi-K2-Thinking": [ 43.63636363636363, 0.036363636, 0 ], "OpenHands/Kimi-K2.5": [ 63.63636363636363, 0.036363636, 0 ], "OpenHands/Kimi-K2.6": [ 74.54545454545455, 0.045454545, 0 ], "OpenHands/MiniMax-M2.1": [ 40.60606060606061, 0.006060606, 0 ], "OpenHands/MiniMax-M2.5": [ 47.878787878787875, -0.021212121, 0 ], "OpenHands/MiniMax-M2.7": [ 25.454545454545453, -0.045454545, 0 ], "OpenHands/MiniMax-M3": [ 66.66666666666667, -0.033333333, 0 ], "OpenHands/Nemotron-3-Nano": [ 14.0, 5.5, "div" ], "OpenHands/Nemotron-3-Super": [ 40.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 33.93939393939394, 0.039393939, 0 ], "OpenHands/Qwen3-Coder-Next": [ 50.90909090909091, 0.009090909, 0 ], "OpenHands/Qwen3.5-Flash": [ 49.696969696969695, -0.003030303, 0 ], "OpenHands/Qwen3.6-Plus": [ 72.12121212121212, 0.021212121, 0 ], "OpenHands/Trinity-Large-Thinking": [ 32.72727272727273, 0.027272727, 0 ], "OpenHands/claude-fable-5": [ 84.24242424242425, 0.042424242, 0 ], "OpenHands/claude-opus-4-5": [ 69.0909090909091, -0.009090909, 0 ], "OpenHands/claude-opus-4-6": [ 80.0, 0.0, 0 ], "OpenHands/claude-opus-4-7": [ 81.21212121212122, 0.012121212, 0 ], "OpenHands/claude-opus-4-8": [ 78.78787878787878, -0.012121212, 0 ], "OpenHands/claude-sonnet-4-5": [ 72.72727272727273, 0.027272727, 0 ], "OpenHands/claude-sonnet-4-6": [ 13.333333333333334, 0.033333333, 0 ] }, "t0": 32, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "S1_pct_Dall", "published_range": [ 8.5, 86.1 ], "zero_variance": false }, "issue_resolution_cost": { "dp": 4, "family": "cost", "max_abs_delta": 0.134640434, "max_abs_rows": [ "OpenHands/Kimi-K2-Thinking" ], "policies": { "C_mean_excl": { "div": 2, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.15462661065600003, -0.005373389, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03949999999999999, -0.0, 0 ], "OpenHands/GLM-4.7": [ 0.5129603136445784, -0.046639686, 2 ], "OpenHands/GLM-5": [ 1.0423479928800001, -0.017652007, 2 ], "OpenHands/GLM-5.1": [ 1.538904090771475, -0.001095909, 1 ], "OpenHands/GPT-5.2": [ 0.8333830224999987, -0.026616978, 2 ], "OpenHands/GPT-5.2-Codex": [ 0.9286473301000008, -0.01135267, 2 ], "OpenHands/GPT-5.4": [ 0.6316178729999997, 0.001617873, 1 ], "OpenHands/GPT-5.5": [ 1.5082915039999998, -0.011708496, 2 ], "OpenHands/Gemini-3-Flash": [ 0.41371285399999963, -0.006287146, 2 ], "OpenHands/Gemini-3-Pro": [ 0.9551172570281126, 0.005117257, 2 ], "OpenHands/Gemini-3.1-Pro": [ 1.8995741519038059, 0.029574152, 2 ], "OpenHands/Gemini-3.5-Flash": [ 1.5651510373493975, 0.035151037, 2 ], "OpenHands/Kimi-K2-Thinking": [ 1.8653595662650575, -0.134640434, "div" ], "OpenHands/Kimi-K2.5": [ 0.40595709846030376, -0.000342902, 1 ], "OpenHands/Kimi-K2.6": [ 0.6624678518999992, -0.007532148, 2 ], "OpenHands/MiniMax-M2.1": [ 0.6623975316000001, -0.049402468, 2 ], "OpenHands/MiniMax-M2.5": [ 0.3066000000000002, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.1794999999999999, -0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.15350868719999988, -0.016491313, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.23180510328000012, -0.008194897, 2 ], "OpenHands/Nemotron-3-Super": [ 0.46630000000000016, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 1.2243786578999993, -0.035621342, 2 ], "OpenHands/Qwen3-Coder-Next": [ 1.3589741296334013, -0.00102587, 1 ], "OpenHands/Qwen3.5-Flash": [ 2.1639000000000004, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 1.5379000000000003, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.6926327924144868, 0.002632792, 1 ], "OpenHands/claude-fable-5": [ 1.4318962289999988, 0.001896229, 1 ], "OpenHands/claude-opus-4-5": [ 1.7953082925000017, -0.024691707, 2 ], "OpenHands/claude-opus-4-6": [ 0.7719499304999994, 0.00194993, 1 ], "OpenHands/claude-opus-4-7": [ 1.2743059469999996, -0.055694053, "div" ], "OpenHands/claude-opus-4-8": [ 0.7417815640000004, -0.008218436, 2 ], "OpenHands/claude-sonnet-4-5": [ 1.1505599043, -0.039440096, 2 ], "OpenHands/claude-sonnet-4-6": [ 1.0274985228456905, -0.002501477, 1 ] }, "t0": 6, "t1": 8, "t2": 18, "undef": 0 }, "C_mean_zero": { "div": 2, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.15462661065600003, -0.005373389, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03949999999999999, -0.0, 0 ], "OpenHands/GLM-4.7": [ 0.5109084723900001, -0.048691528, 2 ], "OpenHands/GLM-5": [ 1.0423479928800001, -0.017652007, 2 ], "OpenHands/GLM-5.1": [ 1.5327484744083892, -0.007251526, 2 ], "OpenHands/GPT-5.2": [ 0.8333830224999987, -0.026616978, 2 ], "OpenHands/GPT-5.2-Codex": [ 0.9286473301000008, -0.01135267, 2 ], "OpenHands/GPT-5.4": [ 0.6316178729999997, 0.001617873, 1 ], "OpenHands/GPT-5.5": [ 1.5082915039999998, -0.011708496, 2 ], "OpenHands/Gemini-3-Flash": [ 0.41371285399999963, -0.006287146, 2 ], "OpenHands/Gemini-3-Pro": [ 0.9512967880000001, 0.001296788, 1 ], "OpenHands/Gemini-3.1-Pro": [ 1.895775003599998, 0.025775004, 2 ], "OpenHands/Gemini-3.5-Flash": [ 1.5588904332, 0.028890433, 2 ], "OpenHands/Kimi-K2-Thinking": [ 1.8578981279999973, -0.142101872, "div" ], "OpenHands/Kimi-K2.5": [ 0.40595709846030376, -0.000342902, 1 ], "OpenHands/Kimi-K2.6": [ 0.6624678518999992, -0.007532148, 2 ], "OpenHands/MiniMax-M2.1": [ 0.6623975316000001, -0.049402468, 2 ], "OpenHands/MiniMax-M2.5": [ 0.3066000000000002, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.1794999999999999, -0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.15350868719999988, -0.016491313, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.23180510328000012, -0.008194897, 2 ], "OpenHands/Nemotron-3-Super": [ 0.46630000000000016, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 1.2243786578999993, -0.035621342, 2 ], "OpenHands/Qwen3-Coder-Next": [ 1.3345125952999999, -0.025487405, 2 ], "OpenHands/Qwen3.5-Flash": [ 2.1595722000000004, -0.0043278, 1 ], "OpenHands/Qwen3.6-Plus": [ 1.5379000000000003, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.68847699566, -0.001523004, 1 ], "OpenHands/claude-fable-5": [ 1.4318962289999988, 0.001896229, 1 ], "OpenHands/claude-opus-4-5": [ 1.7953082925000017, -0.024691707, 2 ], "OpenHands/claude-opus-4-6": [ 0.7719499304999994, 0.00194993, 1 ], "OpenHands/claude-opus-4-7": [ 1.2743059469999996, -0.055694053, "div" ], "OpenHands/claude-opus-4-8": [ 0.7417815640000004, -0.008218436, 2 ], "OpenHands/claude-sonnet-4-5": [ 1.1505599043, -0.039440096, 2 ], "OpenHands/claude-sonnet-4-6": [ 1.0254435257999992, -0.004556474, 1 ] }, "t0": 5, "t1": 8, "t2": 19, "undef": 0 }, "C_median_excl": { "div": 32, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.11980838799999999, -0.040191612, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.031967913241326244, -0.007532087, 2 ], "OpenHands/GLM-4.7": [ 0.4669269495000001, -0.09267305, "div" ], "OpenHands/GLM-5": [ 1.0050420249999998, -0.054957975, "div" ], "OpenHands/GLM-5.1": [ 1.0652025800000002, -0.47479742, "div" ], "OpenHands/GPT-5.2": [ 0.68183115, -0.17816885, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.7194934250000001, -0.220506575, "div" ], "OpenHands/GPT-5.4": [ 0.5731489999999999, -0.056851, "div" ], "OpenHands/GPT-5.5": [ 1.3596955, -0.1603045, "div" ], "OpenHands/Gemini-3-Flash": [ 0.3047496500000001, -0.11525035, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6771354, -0.2728646, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.5835877999999997, -0.2864122, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.1405875500000002, -0.38941245, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.6481845499999999, -0.35181545, "div" ], "OpenHands/Kimi-K2.5": [ 0.32123088233332775, -0.085069118, "div" ], "OpenHands/Kimi-K2.6": [ 0.47504731, -0.19495269, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5481379499999999, -0.16366205, "div" ], "OpenHands/MiniMax-M2.5": [ 0.21766092493416794, -0.088939075, "div" ], "OpenHands/MiniMax-M2.7": [ 0.12236214184360827, -0.057137858, "div" ], "OpenHands/MiniMax-M3": [ 0.11246969999999999, -0.0575303, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.18920687999999997, -0.05079312, "div" ], "OpenHands/Nemotron-3-Super": [ 0.35390485941162797, -0.112395141, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.9790391249999999, -0.280960875, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.2521055499999996, -0.10789445, "div" ], "OpenHands/Qwen3.5-Flash": [ 1.9793823690334558, -0.184517631, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.2778424606238672, -0.260057539, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.47197278999999986, -0.21802721, "div" ], "OpenHands/claude-fable-5": [ 1.1368085, -0.2931915, "div" ], "OpenHands/claude-opus-4-5": [ 1.4294380000000002, -0.390562, "div" ], "OpenHands/claude-opus-4-6": [ 0.38488050000000007, -0.3851195, "div" ], "OpenHands/claude-opus-4-7": [ 0.698624625, -0.631375375, "div" ], "OpenHands/claude-opus-4-8": [ 0.387703, -0.362297, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.0257755999999998, -0.1642244, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.7069893, -0.3230107, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 0 }, "C_median_zero": { "div": 32, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.11980838799999999, -0.040191612, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.031967913241326244, -0.007532087, 2 ], "OpenHands/GLM-4.7": [ 0.46532744150000005, -0.094272558, "div" ], "OpenHands/GLM-5": [ 1.0050420249999998, -0.054957975, "div" ], "OpenHands/GLM-5.1": [ 1.05478432, -0.48521568, "div" ], "OpenHands/GPT-5.2": [ 0.68183115, -0.17816885, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.7194934250000001, -0.220506575, "div" ], "OpenHands/GPT-5.4": [ 0.5731489999999999, -0.056851, "div" ], "OpenHands/GPT-5.5": [ 1.3596955, -0.1603045, "div" ], "OpenHands/Gemini-3-Flash": [ 0.3047496500000001, -0.11525035, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6755781, -0.2744219, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.582515, -0.287485, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.13619285, -0.39380715, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.6474434499999997, -0.35255655, "div" ], "OpenHands/Kimi-K2.5": [ 0.32123088233332775, -0.085069118, "div" ], "OpenHands/Kimi-K2.6": [ 0.47504731, -0.19495269, "div" ], "OpenHands/MiniMax-M2.1": [ 0.5481379499999999, -0.16366205, "div" ], "OpenHands/MiniMax-M2.5": [ 0.21766092493416794, -0.088939075, "div" ], "OpenHands/MiniMax-M2.7": [ 0.12236214184360827, -0.057137858, "div" ], "OpenHands/MiniMax-M3": [ 0.11246969999999999, -0.0575303, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.18920687999999997, -0.05079312, "div" ], "OpenHands/Nemotron-3-Super": [ 0.35390485941162797, -0.112395141, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.9790391249999999, -0.280960875, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.235629775, -0.124370225, "div" ], "OpenHands/Qwen3.5-Flash": [ 1.9792643533575913, -0.184635647, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.2778424606238672, -0.260057539, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.46411347500000005, -0.225886525, "div" ], "OpenHands/claude-fable-5": [ 1.1368085, -0.2931915, "div" ], "OpenHands/claude-opus-4-5": [ 1.4294380000000002, -0.390562, "div" ], "OpenHands/claude-opus-4-6": [ 0.38488050000000007, -0.3851195, "div" ], "OpenHands/claude-opus-4-7": [ 0.698624625, -0.631375375, "div" ], "OpenHands/claude-opus-4-8": [ 0.387703, -0.362297, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.0257755999999998, -0.1642244, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.706664475, -0.323335525, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 0 }, "C_sum_over_resolved": { "div": 32, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.21595895343016763, 0.055958953, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.05396174863387977, 0.014461749, 2 ], "OpenHands/GLM-4.7": [ 0.6960605890871936, 0.136460589, "div" ], "OpenHands/GLM-5": [ 1.420092633351499, 0.360092633, "div" ], "OpenHands/GLM-5.1": [ 2.043664632544519, 0.503664633, "div" ], "OpenHands/GPT-5.2": [ 1.1171354189008027, 0.257135419, "div" ], "OpenHands/GPT-5.2-Codex": [ 1.258329715582657, 0.318329716, "div" ], "OpenHands/GPT-5.4": [ 0.8354733769841266, 0.205473377, "div" ], "OpenHands/GPT-5.5": [ 1.928761514066496, 0.408761514, "div" ], "OpenHands/Gemini-3-Flash": [ 0.5545748713136724, 0.134574871, "div" ], "OpenHands/Gemini-3-Pro": [ 1.3474458753541079, 0.397445875, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.4620454592207768, 0.592045459, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.983321161832061, 0.453321162, "div" ], "OpenHands/Kimi-K2-Thinking": [ 2.68482388439306, 0.684823884, "div" ], "OpenHands/Kimi-K2.5": [ 0.5900539221806741, 0.183753922, "div" ], "OpenHands/Kimi-K2.6": [ 0.8880266111260043, 0.218026611, "div" ], "OpenHands/MiniMax-M2.1": [ 0.9627871098837211, 0.25098711, "div" ], "OpenHands/MiniMax-M2.5": [ 0.42231404958677715, 0.11571405, "div" ], "OpenHands/MiniMax-M2.7": [ 0.23743386243386233, 0.057933862, "div" ], "OpenHands/MiniMax-M3": [ 0.20092760104712026, 0.030927601, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.6777926996491231, 0.4377927, "div" ], "OpenHands/Nemotron-3-Super": [ 0.7520967741935487, 0.285796774, "div" ], "OpenHands/Qwen3-Coder-480B": [ 1.9621452850961527, 0.702145285, "div" ], "OpenHands/Qwen3-Coder-Next": [ 2.0037726656156156, 0.643772666, "div" ], "OpenHands/Qwen3.5-Flash": [ 3.4831809677419363, 1.319280968, "div" ], "OpenHands/Qwen3.6-Plus": [ 2.0726415094339625, 0.534741509, "div" ], "OpenHands/Trinity-Large-Thinking": [ 1.212107386725352, 0.522107387, "div" ], "OpenHands/claude-fable-5": [ 1.494672472860124, 0.064672473, "div" ], "OpenHands/claude-opus-4-5": [ 2.343744507180159, 0.523744507, "div" ], "OpenHands/claude-opus-4-6": [ 1.005143138671874, 0.235143139, "div" ], "OpenHands/claude-opus-4-7": [ 1.5616494448529408, 0.231649445, "div" ], "OpenHands/claude-opus-4-8": [ 0.8851808639618143, 0.135180864, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.5506198171159031, 0.360619817, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.3782843088709666, 0.348284309, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 0 } }, "primary": "C_mean_excl", "published_range": [ 0.0395, 2.1639 ], "zero_variance": false }, "issue_resolution_score": { "dp": 1, "family": "score", "max_abs_delta": 0.2, "max_abs_rows": [ "OpenHands/Gemini-3.1-Pro" ], "policies": { "S1_frac_Dall": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.716, -70.884, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.732, -72.468, "div" ], "OpenHands/GLM-4.7": [ 0.734, -72.666, "div" ], "OpenHands/GLM-5": [ 0.734, -72.666, "div" ], "OpenHands/GLM-5.1": [ 0.75, -74.25, "div" ], "OpenHands/GPT-5.2": [ 0.746, -73.854, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.738, -73.062, "div" ], "OpenHands/GPT-5.4": [ 0.756, -74.844, "div" ], "OpenHands/GPT-5.5": [ 0.782, -77.418, "div" ], "OpenHands/Gemini-3-Flash": [ 0.746, -73.854, "div" ], "OpenHands/Gemini-3-Pro": [ 0.706, -69.894, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.77, -76.03, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.786, -77.814, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.692, -68.508, "div" ], "OpenHands/Kimi-K2.5": [ 0.688, -68.112, "div" ], "OpenHands/Kimi-K2.6": [ 0.746, -73.854, "div" ], "OpenHands/MiniMax-M2.1": [ 0.688, -68.112, "div" ], "OpenHands/MiniMax-M2.5": [ 0.726, -71.874, "div" ], "OpenHands/MiniMax-M2.7": [ 0.756, -74.844, "div" ], "OpenHands/MiniMax-M3": [ 0.764, -75.636, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.342, -33.858, "div" ], "OpenHands/Nemotron-3-Super": [ 0.62, -61.38, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.624, -61.776, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.666, -65.934, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.62, -61.38, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.742, -73.458, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.568, -56.232, "div" ], "OpenHands/claude-fable-5": [ 0.958, -94.842, "div" ], "OpenHands/claude-opus-4-5": [ 0.766, -75.834, "div" ], "OpenHands/claude-opus-4-6": [ 0.768, -76.032, "div" ], "OpenHands/claude-opus-4-7": [ 0.816, -80.784, "div" ], "OpenHands/claude-opus-4-8": [ 0.838, -82.962, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.742, -73.458, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.744, -73.656, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S1_pct_Dall": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 71.6, 0.0, 0 ], "OpenHands/DeepSeek-V4-Pro": [ 73.2, 0.0, 0 ], "OpenHands/GLM-4.7": [ 73.4, 0.0, 0 ], "OpenHands/GLM-5": [ 73.4, 0.0, 0 ], "OpenHands/GLM-5.1": [ 75.0, 0.0, 0 ], "OpenHands/GPT-5.2": [ 74.6, 0.0, 0 ], "OpenHands/GPT-5.2-Codex": [ 73.8, 0.0, 0 ], "OpenHands/GPT-5.4": [ 75.6, 0.0, 0 ], "OpenHands/GPT-5.5": [ 78.2, 0.0, 0 ], "OpenHands/Gemini-3-Flash": [ 74.6, 0.0, 0 ], "OpenHands/Gemini-3-Pro": [ 70.6, 0.0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 77.0, 0.2, 2 ], "OpenHands/Gemini-3.5-Flash": [ 78.6, 0.0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 69.2, 0.0, 0 ], "OpenHands/Kimi-K2.5": [ 68.8, 0.0, 0 ], "OpenHands/Kimi-K2.6": [ 74.6, 0.0, 0 ], "OpenHands/MiniMax-M2.1": [ 68.8, 0.0, 0 ], "OpenHands/MiniMax-M2.5": [ 72.6, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 75.6, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 76.4, 0.0, 0 ], "OpenHands/Nemotron-3-Nano": [ 34.2, 0.0, 0 ], "OpenHands/Nemotron-3-Super": [ 62.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 62.4, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 66.6, 0.0, 0 ], "OpenHands/Qwen3.5-Flash": [ 62.0, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 74.2, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 56.8, 0.0, 0 ], "OpenHands/claude-fable-5": [ 95.8, 0.0, 0 ], "OpenHands/claude-opus-4-5": [ 76.6, 0.0, 0 ], "OpenHands/claude-opus-4-6": [ 76.8, 0.0, 0 ], "OpenHands/claude-opus-4-7": [ 81.6, 0.0, 0 ], "OpenHands/claude-opus-4-8": [ 83.8, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 74.2, 0.0, 0 ], "OpenHands/claude-sonnet-4-6": [ 74.4, 0.0, 0 ] }, "t0": 33, "t1": 0, "t2": 1, "undef": 0 }, "S2_frac_Dobs": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.7203219315895373, -70.879678068, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.732, -72.468, "div" ], "OpenHands/GLM-4.7": [ 0.734, -72.666, "div" ], "OpenHands/GLM-5": [ 0.734, -72.666, "div" ], "OpenHands/GLM-5.1": [ 0.75, -74.25, "div" ], "OpenHands/GPT-5.2": [ 0.746, -73.854, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.738, -73.062, "div" ], "OpenHands/GPT-5.4": [ 0.756, -74.844, "div" ], "OpenHands/GPT-5.5": [ 0.782, -77.418, "div" ], "OpenHands/Gemini-3-Flash": [ 0.746, -73.854, "div" ], "OpenHands/Gemini-3-Pro": [ 0.706, -69.894, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.77, -76.03, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.786, -77.814, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.6947791164658634, -68.505220884, "div" ], "OpenHands/Kimi-K2.5": [ 0.6893787575150301, -68.110621242, "div" ], "OpenHands/Kimi-K2.6": [ 0.746, -73.854, "div" ], "OpenHands/MiniMax-M2.1": [ 0.688, -68.112, "div" ], "OpenHands/MiniMax-M2.5": [ 0.726, -71.874, "div" ], "OpenHands/MiniMax-M2.7": [ 0.756, -74.844, "div" ], "OpenHands/MiniMax-M3": [ 0.764, -75.636, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.3440643863179074, -33.855935614, "div" ], "OpenHands/Nemotron-3-Super": [ 0.62, -61.38, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.624, -61.776, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.666, -65.934, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.62, -61.38, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.742, -73.458, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.568, -56.232, "div" ], "OpenHands/claude-fable-5": [ 0.958, -94.842, "div" ], "OpenHands/claude-opus-4-5": [ 0.766, -75.834, "div" ], "OpenHands/claude-opus-4-6": [ 0.768, -76.032, "div" ], "OpenHands/claude-opus-4-7": [ 0.816, -80.784, "div" ], "OpenHands/claude-opus-4-8": [ 0.838, -82.962, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.742, -73.458, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.744, -73.656, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S2_pct_Dobs": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 72.03219315895372, 0.432193159, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 73.2, 0.0, 0 ], "OpenHands/GLM-4.7": [ 73.4, 0.0, 0 ], "OpenHands/GLM-5": [ 73.4, 0.0, 0 ], "OpenHands/GLM-5.1": [ 75.0, 0.0, 0 ], "OpenHands/GPT-5.2": [ 74.6, 0.0, 0 ], "OpenHands/GPT-5.2-Codex": [ 73.8, 0.0, 0 ], "OpenHands/GPT-5.4": [ 75.6, 0.0, 0 ], "OpenHands/GPT-5.5": [ 78.2, 0.0, 0 ], "OpenHands/Gemini-3-Flash": [ 74.6, 0.0, 0 ], "OpenHands/Gemini-3-Pro": [ 70.6, 0.0, 0 ], "OpenHands/Gemini-3.1-Pro": [ 77.0, 0.2, 2 ], "OpenHands/Gemini-3.5-Flash": [ 78.6, 0.0, 0 ], "OpenHands/Kimi-K2-Thinking": [ 69.47791164658635, 0.277911647, 2 ], "OpenHands/Kimi-K2.5": [ 68.937875751503, 0.137875752, 2 ], "OpenHands/Kimi-K2.6": [ 74.6, 0.0, 0 ], "OpenHands/MiniMax-M2.1": [ 68.8, 0.0, 0 ], "OpenHands/MiniMax-M2.5": [ 72.6, 0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 75.6, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 76.4, 0.0, 0 ], "OpenHands/Nemotron-3-Nano": [ 34.40643863179074, 0.206438632, 2 ], "OpenHands/Nemotron-3-Super": [ 62.0, 0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 62.4, 0.0, 0 ], "OpenHands/Qwen3-Coder-Next": [ 66.6, 0.0, 0 ], "OpenHands/Qwen3.5-Flash": [ 62.0, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 74.2, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 56.8, 0.0, 0 ], "OpenHands/claude-fable-5": [ 95.8, 0.0, 0 ], "OpenHands/claude-opus-4-5": [ 76.6, 0.0, 0 ], "OpenHands/claude-opus-4-6": [ 76.8, 0.0, 0 ], "OpenHands/claude-opus-4-7": [ 81.6, 0.0, 0 ], "OpenHands/claude-opus-4-8": [ 83.8, 0.0, 0 ], "OpenHands/claude-sonnet-4-5": [ 74.2, 0.0, 0 ], "OpenHands/claude-sonnet-4-6": [ 74.4, 0.0, 0 ] }, "t0": 29, "t1": 0, "t2": 5, "undef": 0 } }, "primary": "S1_pct_Dall", "published_range": [ 34.2, 95.8 ], "zero_variance": false }, "language_model": { "dp": null, "family": "categorical", "max_abs_delta": null, "max_abs_rows": [], "policies": { "U_major": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "DeepSeek-V3.2-Reasoner", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "DeepSeek-V4-Pro", null, 0 ], "OpenHands/GLM-4.7": [ "GLM-4.7", null, 0 ], "OpenHands/GLM-5": [ "GLM-5", null, 0 ], "OpenHands/GLM-5.1": [ "GLM-5.1", null, 0 ], "OpenHands/GPT-5.2": [ "GPT-5.2", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "GPT-5.2-Codex", null, 0 ], "OpenHands/GPT-5.4": [ "GPT-5.4", null, 0 ], "OpenHands/GPT-5.5": [ "GPT-5.5", null, 0 ], "OpenHands/Gemini-3-Flash": [ "Gemini-3-Flash", null, 0 ], "OpenHands/Gemini-3-Pro": [ "Gemini-3-Pro", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "Gemini-3.1-Pro", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "Gemini-3.5-Flash", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "Kimi-K2-Thinking", null, 0 ], "OpenHands/Kimi-K2.5": [ "Kimi-K2.5", null, 0 ], "OpenHands/Kimi-K2.6": [ "Kimi-K2.6", null, 0 ], "OpenHands/MiniMax-M2.1": [ "MiniMax-M2.1", null, 0 ], "OpenHands/MiniMax-M2.5": [ "MiniMax-M2.5", null, 0 ], "OpenHands/MiniMax-M2.7": [ "Minimax-2.7", null, 0 ], "OpenHands/MiniMax-M3": [ "MiniMax-M3", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "Nemotron-3-Nano", null, 0 ], "OpenHands/Nemotron-3-Super": [ "Nemotron-3-Super", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "Qwen3-Coder-480B", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "Qwen3-Coder-Next", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "Qwen3.5-Flash", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "Qwen3.6-Plus", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "Trinity-Large-Thinking", null, 0 ], "OpenHands/claude-fable-5": [ "claude-fable-5", null, 0 ], "OpenHands/claude-opus-4-5": [ "claude-opus-4-5", null, 0 ], "OpenHands/claude-opus-4-6": [ "claude-opus-4-6", null, 0 ], "OpenHands/claude-opus-4-7": [ "claude-opus-4-7", null, 0 ], "OpenHands/claude-opus-4-8": [ "claude-opus-4-8", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "claude-sonnet-4-5", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "claude-sonnet-4-6", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 }, "U_unique": { "div": 0, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ "DeepSeek-V3.2-Reasoner", null, 0 ], "OpenHands/DeepSeek-V4-Pro": [ "DeepSeek-V4-Pro", null, 0 ], "OpenHands/GLM-4.7": [ "GLM-4.7", null, 0 ], "OpenHands/GLM-5": [ "GLM-5", null, 0 ], "OpenHands/GLM-5.1": [ "GLM-5.1", null, 0 ], "OpenHands/GPT-5.2": [ "GPT-5.2", null, 0 ], "OpenHands/GPT-5.2-Codex": [ "GPT-5.2-Codex", null, 0 ], "OpenHands/GPT-5.4": [ "GPT-5.4", null, 0 ], "OpenHands/GPT-5.5": [ "GPT-5.5", null, 0 ], "OpenHands/Gemini-3-Flash": [ "Gemini-3-Flash", null, 0 ], "OpenHands/Gemini-3-Pro": [ "Gemini-3-Pro", null, 0 ], "OpenHands/Gemini-3.1-Pro": [ "Gemini-3.1-Pro", null, 0 ], "OpenHands/Gemini-3.5-Flash": [ "Gemini-3.5-Flash", null, 0 ], "OpenHands/Kimi-K2-Thinking": [ "Kimi-K2-Thinking", null, 0 ], "OpenHands/Kimi-K2.5": [ "Kimi-K2.5", null, 0 ], "OpenHands/Kimi-K2.6": [ "Kimi-K2.6", null, 0 ], "OpenHands/MiniMax-M2.1": [ "MiniMax-M2.1", null, 0 ], "OpenHands/MiniMax-M2.5": [ "MiniMax-M2.5", null, 0 ], "OpenHands/MiniMax-M2.7": [ "Minimax-2.7", null, 0 ], "OpenHands/MiniMax-M3": [ "MiniMax-M3", null, 0 ], "OpenHands/Nemotron-3-Nano": [ "Nemotron-3-Nano", null, 0 ], "OpenHands/Nemotron-3-Super": [ "Nemotron-3-Super", null, 0 ], "OpenHands/Qwen3-Coder-480B": [ "Qwen3-Coder-480B", null, 0 ], "OpenHands/Qwen3-Coder-Next": [ "Qwen3-Coder-Next", null, 0 ], "OpenHands/Qwen3.5-Flash": [ "Qwen3.5-Flash", null, 0 ], "OpenHands/Qwen3.6-Plus": [ "Qwen3.6-Plus", null, 0 ], "OpenHands/Trinity-Large-Thinking": [ "Trinity-Large-Thinking", null, 0 ], "OpenHands/claude-fable-5": [ "claude-fable-5", null, 0 ], "OpenHands/claude-opus-4-5": [ "claude-opus-4-5", null, 0 ], "OpenHands/claude-opus-4-6": [ "claude-opus-4-6", null, 0 ], "OpenHands/claude-opus-4-7": [ "claude-opus-4-7", null, 0 ], "OpenHands/claude-opus-4-8": [ "claude-opus-4-8", null, 0 ], "OpenHands/claude-sonnet-4-5": [ "claude-sonnet-4-5", null, 0 ], "OpenHands/claude-sonnet-4-6": [ "claude-sonnet-4-6", null, 0 ] }, "t0": 34, "t1": 0, "t2": 0, "undef": 0 } }, "primary": "U_unique", "published_range": null, "zero_variance": false }, "testing_cost": { "dp": 4, "family": "cost", "max_abs_delta": 0.148855991, "max_abs_rows": [ "OpenHands/Kimi-K2-Thinking" ], "policies": { "C_mean_excl": { "div": 3, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.10911414470422534, -0.010885855, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03540000000000003, 0.0, 0 ], "OpenHands/GLM-4.7": [ 0.37043347483412287, 0.000433475, 1 ], "OpenHands/GLM-5": [ 0.9085117246706582, -0.001488275, 1 ], "OpenHands/GLM-5.1": [ 0.7573554853472347, -0.002644515, 1 ], "OpenHands/GPT-5.2": [ 0.5417488901639343, -0.01825111, 2 ], "OpenHands/GPT-5.2-Codex": [ 0.6495457556581984, -0.010454244, 2 ], "OpenHands/GPT-5.4": [ 0.4703474145496539, 0.000347415, 1 ], "OpenHands/GPT-5.5": [ 0.9164391085450346, -0.003560891, 1 ], "OpenHands/Gemini-3-Flash": [ 0.29794644477958243, -0.002053555, 1 ], "OpenHands/Gemini-3-Pro": [ 0.9197614574074069, -0.090238543, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.6359732775981524, 0.015973278, 2 ], "OpenHands/Gemini-3.5-Flash": [ 1.025110233916084, -0.004889766, 1 ], "OpenHands/Kimi-K2-Thinking": [ 1.241144009237876, -0.148855991, "div" ], "OpenHands/Kimi-K2.5": [ 0.4205543541666671, -0.004045646, 1 ], "OpenHands/Kimi-K2.6": [ 0.3337739642725172, 0.003773964, 1 ], "OpenHands/MiniMax-M2.1": [ 0.5220043926096996, -0.022095607, 2 ], "OpenHands/MiniMax-M2.5": [ 0.2238999999999999, -0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.12830000000000008, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.10038684387990766, -0.009613156, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.18069706472222222, -0.009302935, 2 ], "OpenHands/Nemotron-3-Super": [ 0.3332999999999997, -0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.9196778385542168, -0.050322161, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.0865237926179254, -0.013476207, 2 ], "OpenHands/Qwen3.5-Flash": [ 1.7107000000000012, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.0850000000000013, 0.0, 0 ], "OpenHands/Trinity-Large-Thinking": [ 0.7766997015704385, -0.023300298, 2 ], "OpenHands/claude-fable-5": [ 1.4665276431870662, -0.003472357, 1 ], "OpenHands/claude-opus-4-5": [ 1.3401414249422638, -0.039858575, 2 ], "OpenHands/claude-opus-4-6": [ 0.42992733602771355, -7.2664e-05, 1 ], "OpenHands/claude-opus-4-7": [ 0.8205347909930718, 0.000534791, 1 ], "OpenHands/claude-opus-4-8": [ 0.7326379907621243, 0.002637991, 1 ], "OpenHands/claude-sonnet-4-5": [ 0.9359994345348833, -0.044000565, 2 ], "OpenHands/claude-sonnet-4-6": [ 0.8534012847575057, -0.016598715, 2 ] }, "t0": 6, "t1": 13, "t2": 12, "undef": 0 }, "C_mean_zero": { "div": 3, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.10911414470422534, -0.010885855, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03540000000000003, 0.0, 0 ], "OpenHands/GLM-4.7": [ 0.37043347483412287, 0.000433475, 1 ], "OpenHands/GLM-5": [ 0.9085117246706582, -0.001488275, 1 ], "OpenHands/GLM-5.1": [ 0.7573554853472347, -0.002644515, 1 ], "OpenHands/GPT-5.2": [ 0.5417488901639343, -0.01825111, 2 ], "OpenHands/GPT-5.2-Codex": [ 0.6495457556581984, -0.010454244, 2 ], "OpenHands/GPT-5.4": [ 0.4703474145496539, 0.000347415, 1 ], "OpenHands/GPT-5.5": [ 0.9164391085450346, -0.003560891, 1 ], "OpenHands/Gemini-3-Flash": [ 0.29794644477958243, -0.002053555, 1 ], "OpenHands/Gemini-3-Pro": [ 0.9197614574074069, -0.090238543, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.6359732775981524, 0.015973278, 2 ], "OpenHands/Gemini-3.5-Flash": [ 1.0179914128472223, -0.012008587, 2 ], "OpenHands/Kimi-K2-Thinking": [ 1.241144009237876, -0.148855991, "div" ], "OpenHands/Kimi-K2.5": [ 0.4205543541666671, -0.004045646, 1 ], "OpenHands/Kimi-K2.6": [ 0.3337739642725172, 0.003773964, 1 ], "OpenHands/MiniMax-M2.1": [ 0.5220043926096996, -0.022095607, 2 ], "OpenHands/MiniMax-M2.5": [ 0.2238999999999999, -0.0, 0 ], "OpenHands/MiniMax-M2.7": [ 0.12830000000000008, 0.0, 0 ], "OpenHands/MiniMax-M3": [ 0.10038684387990766, -0.009613156, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.18069706472222222, -0.009302935, 2 ], "OpenHands/Nemotron-3-Super": [ 0.3332999999999997, -0.0, 0 ], "OpenHands/Qwen3-Coder-480B": [ 0.9196778385542168, -0.050322161, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.0639401572055436, -0.036059843, 2 ], "OpenHands/Qwen3.5-Flash": [ 1.7107000000000012, 0.0, 0 ], "OpenHands/Qwen3.6-Plus": [ 2.0801847575057746, -0.004815242, 1 ], "OpenHands/Trinity-Large-Thinking": [ 0.7766997015704385, -0.023300298, 2 ], "OpenHands/claude-fable-5": [ 1.4665276431870662, -0.003472357, 1 ], "OpenHands/claude-opus-4-5": [ 1.3401414249422638, -0.039858575, 2 ], "OpenHands/claude-opus-4-6": [ 0.42992733602771355, -7.2664e-05, 1 ], "OpenHands/claude-opus-4-7": [ 0.8205347909930718, 0.000534791, 1 ], "OpenHands/claude-opus-4-8": [ 0.7326379907621243, 0.002637991, 1 ], "OpenHands/claude-sonnet-4-5": [ 0.9359994345348833, -0.044000565, 2 ], "OpenHands/claude-sonnet-4-6": [ 0.8534012847575057, -0.016598715, 2 ] }, "t0": 5, "t1": 13, "t2": 13, "undef": 0 }, "C_median_excl": { "div": 25, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.09155969199999997, -0.028440308, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03116543620178306, -0.004234564, 1 ], "OpenHands/GLM-4.7": [ 0.34780073699999997, -0.022199263, 2 ], "OpenHands/GLM-5": [ 0.8954641500000001, -0.01453585, 2 ], "OpenHands/GLM-5.1": [ 0.4942202050000001, -0.265779795, "div" ], "OpenHands/GPT-5.2": [ 0.48120450000000003, -0.0787955, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.5182127999999999, -0.1417872, "div" ], "OpenHands/GPT-5.4": [ 0.4221235, -0.0478765, 2 ], "OpenHands/GPT-5.5": [ 0.828551, -0.091449, "div" ], "OpenHands/Gemini-3-Flash": [ 0.1922447, -0.1077553, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6785021, -0.3314979, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.3387421999999995, -0.2812578, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.7863897000000001, -0.2436103, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.1231365, -0.2668635, "div" ], "OpenHands/Kimi-K2.5": [ 0.3673494, -0.0572506, "div" ], "OpenHands/Kimi-K2.6": [ 0.26144413, -0.06855587, "div" ], "OpenHands/MiniMax-M2.1": [ 0.4545770999999999, -0.0895229, "div" ], "OpenHands/MiniMax-M2.5": [ 0.1790636373878777, -0.044836363, 2 ], "OpenHands/MiniMax-M2.7": [ 0.10584185013997713, -0.02245815, 2 ], "OpenHands/MiniMax-M3": [ 0.07701239999999998, -0.0329876, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.15017547000000003, -0.03982453, 2 ], "OpenHands/Nemotron-3-Super": [ 0.24660954552577688, -0.086690454, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.8102582999999999, -0.1597417, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.0197606399999999, -0.08023936, "div" ], "OpenHands/Qwen3.5-Flash": [ 1.463956401882727, -0.246743598, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.6067004562954859, -0.478299544, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.6762773899999998, -0.12372261, "div" ], "OpenHands/claude-fable-5": [ 1.2748425, -0.1951575, "div" ], "OpenHands/claude-opus-4-5": [ 1.181363, -0.198637, "div" ], "OpenHands/claude-opus-4-6": [ 0.33018899999999995, -0.099811, "div" ], "OpenHands/claude-opus-4-7": [ 0.642374, -0.177626, "div" ], "OpenHands/claude-opus-4-8": [ 0.54986775, -0.18013225, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.9175493999999998, -0.0624506, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.6653367000000001, -0.2046633, "div" ] }, "t0": 0, "t1": 1, "t2": 8, "undef": 0 }, "C_median_zero": { "div": 25, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.09155969199999997, -0.028440308, 2 ], "OpenHands/DeepSeek-V4-Pro": [ 0.03116543620178306, -0.004234564, 1 ], "OpenHands/GLM-4.7": [ 0.34780073699999997, -0.022199263, 2 ], "OpenHands/GLM-5": [ 0.8954641500000001, -0.01453585, 2 ], "OpenHands/GLM-5.1": [ 0.4942202050000001, -0.265779795, "div" ], "OpenHands/GPT-5.2": [ 0.48120450000000003, -0.0787955, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.5182127999999999, -0.1417872, "div" ], "OpenHands/GPT-5.4": [ 0.4221235, -0.0478765, 2 ], "OpenHands/GPT-5.5": [ 0.828551, -0.091449, "div" ], "OpenHands/Gemini-3-Flash": [ 0.1922447, -0.1077553, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6785021, -0.3314979, "div" ], "OpenHands/Gemini-3.1-Pro": [ 1.3387421999999995, -0.2812578, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.7845948, -0.2454052, "div" ], "OpenHands/Kimi-K2-Thinking": [ 1.1231365, -0.2668635, "div" ], "OpenHands/Kimi-K2.5": [ 0.3673494, -0.0572506, "div" ], "OpenHands/Kimi-K2.6": [ 0.26144413, -0.06855587, "div" ], "OpenHands/MiniMax-M2.1": [ 0.4545770999999999, -0.0895229, "div" ], "OpenHands/MiniMax-M2.5": [ 0.1790636373878777, -0.044836363, 2 ], "OpenHands/MiniMax-M2.7": [ 0.10584185013997713, -0.02245815, 2 ], "OpenHands/MiniMax-M3": [ 0.07701239999999998, -0.0329876, 2 ], "OpenHands/Nemotron-3-Nano": [ 0.15017547000000003, -0.03982453, 2 ], "OpenHands/Nemotron-3-Super": [ 0.24660954552577688, -0.086690454, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.8102582999999999, -0.1597417, "div" ], "OpenHands/Qwen3-Coder-Next": [ 1.0126834200000003, -0.08731658, "div" ], "OpenHands/Qwen3.5-Flash": [ 1.463956401882727, -0.246743598, "div" ], "OpenHands/Qwen3.6-Plus": [ 1.5904274678840629, -0.494572532, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.6762773899999998, -0.12372261, "div" ], "OpenHands/claude-fable-5": [ 1.2748425, -0.1951575, "div" ], "OpenHands/claude-opus-4-5": [ 1.181363, -0.198637, "div" ], "OpenHands/claude-opus-4-6": [ 0.33018899999999995, -0.099811, "div" ], "OpenHands/claude-opus-4-7": [ 0.642374, -0.177626, "div" ], "OpenHands/claude-opus-4-8": [ 0.54986775, -0.18013225, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.9175493999999998, -0.0624506, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.6653367000000001, -0.2046633, "div" ] }, "t0": 0, "t1": 1, "t2": 8, "undef": 0 }, "C_sum_over_resolved": { "div": 31, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.20035614501724136, 0.080356145, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.05806136363636368, 0.022661364, 2 ], "OpenHands/GLM-4.7": [ 0.7304809643925226, 0.360480964, "div" ], "OpenHands/GLM-5": [ 1.4802093465365844, 0.570209347, "div" ], "OpenHands/GLM-5.1": [ 1.0858772356137505, 0.325877236, "div" ], "OpenHands/GPT-5.2": [ 0.7297374640378549, 0.169737464, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.9698390075862067, 0.309839008, "div" ], "OpenHands/GPT-5.4": [ 0.6677391163934431, 0.197739116, "div" ], "OpenHands/GPT-5.5": [ 1.0992192077562326, 0.179219208, "div" ], "OpenHands/Gemini-3-Flash": [ 0.41965659379084974, 0.119656594, "div" ], "OpenHands/Gemini-3-Pro": [ 1.3378348471380466, 0.327834847, "div" ], "OpenHands/Gemini-3.1-Pro": [ 2.1863470037037036, 0.566347004, "div" ], "OpenHands/Gemini-3.5-Flash": [ 1.3326433040909091, 0.302643304, "div" ], "OpenHands/Kimi-K2-Thinking": [ 2.621538321951221, 1.231538322, "div" ], "OpenHands/Kimi-K2.5": [ 0.6779085111940305, 0.253308511, "div" ], "OpenHands/Kimi-K2.6": [ 0.47384959518032777, 0.143849595, "div" ], "OpenHands/MiniMax-M2.1": [ 0.8497289548872178, 0.305628955, "div" ], "OpenHands/MiniMax-M2.5": [ 0.328639661016949, 0.104739661, "div" ], "OpenHands/MiniMax-M2.7": [ 0.18579899665551852, 0.057498997, "div" ], "OpenHands/MiniMax-M3": [ 0.12383904102564108, 0.013839041, 2 ], "OpenHands/Nemotron-3-Nano": [ null, null, null ], "OpenHands/Nemotron-3-Super": [ 0.7929609890109883, 0.459660989, "div" ], "OpenHands/Qwen3-Coder-480B": [ 2.527591410596026, 1.557591411, "div" ], "OpenHands/Qwen3-Coder-Next": [ 2.503728739510872, 1.40372874, "div" ], "OpenHands/Qwen3.5-Flash": [ 4.629581875000003, 2.918881875, "div" ], "OpenHands/Qwen3.6-Plus": [ 3.411818181818184, 1.326818182, "div" ], "OpenHands/Trinity-Large-Thinking": [ 2.35182497048951, 1.55182497, "div" ], "OpenHands/claude-fable-5": [ 1.595493641959798, 0.125493642, "div" ], "OpenHands/claude-opus-4-5": [ 1.706709520588236, 0.326709521, "div" ], "OpenHands/claude-opus-4-6": [ 0.5459194618768327, 0.115919462, "div" ], "OpenHands/claude-opus-4-7": [ 1.015118755714286, 0.195118756, "div" ], "OpenHands/claude-opus-4-8": [ 0.8691294520547941, 0.139129452, "div" ], "OpenHands/claude-sonnet-4-5": [ 1.3506032109060397, 0.370603211, "div" ], "OpenHands/claude-sonnet-4-6": [ 1.579157078205128, 0.709157078, "div" ] }, "t0": 0, "t1": 0, "t2": 2, "undef": 1 } }, "primary": "C_mean_excl", "published_range": [ 0.0354, 2.085 ], "zero_variance": false }, "testing_score": { "dp": 1, "family": "score", "max_abs_delta": 14.077245509, "max_abs_rows": [ "OpenHands/GLM-5" ], "policies": { "S1_frac_Dall": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.5446009389671361, -53.055399061, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.6096997690531177, -67.490300231, "div" ], "OpenHands/GLM-4.7": [ 0.5071090047393365, -48.892890995, "div" ], "OpenHands/GLM-5": [ 0.6137724550898204, -46.686227545, "div" ], "OpenHands/GLM-5.1": [ 0.6974595842956121, -69.502540416, "div" ], "OpenHands/GPT-5.2": [ 0.7423887587822015, -72.457611241, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.6697459584295612, -66.330254042, "div" ], "OpenHands/GPT-5.4": [ 0.7043879907621247, -69.695612009, "div" ], "OpenHands/GPT-5.5": [ 0.8337182448036952, -82.566281755, "div" ], "OpenHands/Gemini-3-Flash": [ 0.7099767981438515, -69.990023202, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6875, -67.9125, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.7482678983833718, -74.351732102, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.7638888888888888, -75.436111111, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.47344110854503463, -46.826558891, "div" ], "OpenHands/Kimi-K2.5": [ 0.6203703703703703, -61.27962963, "div" ], "OpenHands/Kimi-K2.6": [ 0.7043879907621247, -69.695612009, "div" ], "OpenHands/MiniMax-M2.1": [ 0.6143187066974596, -60.785681293, "div" ], "OpenHands/MiniMax-M2.5": [ 0.6812933025404158, -67.418706697, "div" ], "OpenHands/MiniMax-M2.7": [ 0.6905311778290993, -68.409468822, "div" ], "OpenHands/MiniMax-M3": [ 0.8106235565819861, -80.289376443, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.0, -7.6, "div" ], "OpenHands/Nemotron-3-Super": [ 0.42032332563510394, -45.279676674, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.363855421686747, -34.536144578, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.42494226327944573, -45.275057737, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.3695150115473441, -37.930484988, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.6096997690531177, -61.490300231, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.3302540415704388, -32.969745958, "div" ], "OpenHands/claude-fable-5": [ 0.9191685912240185, -90.980831409, "div" ], "OpenHands/claude-opus-4-5": [ 0.7852193995381063, -77.7147806, "div" ], "OpenHands/claude-opus-4-6": [ 0.7875288683602771, -78.012471132, "div" ], "OpenHands/claude-opus-4-7": [ 0.8083140877598153, -79.991685912, "div" ], "OpenHands/claude-opus-4-8": [ 0.8429561200923787, -83.45704388, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.6930232558139535, -68.106976744, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.5404157043879908, -53.459584296, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S1_pct_Dall": { "div": 12, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 54.460093896713616, 0.860093897, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 60.969976905311775, -7.130023095, "div" ], "OpenHands/GLM-4.7": [ 50.71090047393365, 1.310900474, "div" ], "OpenHands/GLM-5": [ 61.377245508982035, 14.077245509, "div" ], "OpenHands/GLM-5.1": [ 69.7459584295612, -0.45404157, 2 ], "OpenHands/GPT-5.2": [ 74.23887587822014, 1.038875878, "div" ], "OpenHands/GPT-5.2-Codex": [ 66.97459584295612, -0.025404157, 0 ], "OpenHands/GPT-5.4": [ 70.43879907621248, 0.038799076, 0 ], "OpenHands/GPT-5.5": [ 83.37182448036951, -0.02817552, 0 ], "OpenHands/Gemini-3-Flash": [ 70.99767981438515, 0.297679814, 2 ], "OpenHands/Gemini-3-Pro": [ 68.75, 0.15, 2 ], "OpenHands/Gemini-3.1-Pro": [ 74.82678983833718, -0.273210162, 2 ], "OpenHands/Gemini-3.5-Flash": [ 76.38888888888889, 0.188888889, 2 ], "OpenHands/Kimi-K2-Thinking": [ 47.34411085450346, 0.044110855, 0 ], "OpenHands/Kimi-K2.5": [ 62.03703703703704, 0.137037037, 2 ], "OpenHands/Kimi-K2.6": [ 70.43879907621248, 0.038799076, 0 ], "OpenHands/MiniMax-M2.1": [ 61.43187066974596, 0.03187067, 0 ], "OpenHands/MiniMax-M2.5": [ 68.12933025404158, 0.029330254, 0 ], "OpenHands/MiniMax-M2.7": [ 69.05311778290994, -0.046882217, 0 ], "OpenHands/MiniMax-M3": [ 81.06235565819861, -0.037644342, 0 ], "OpenHands/Nemotron-3-Nano": [ 0.0, -7.6, "div" ], "OpenHands/Nemotron-3-Super": [ 42.032332563510394, -3.667667436, "div" ], "OpenHands/Qwen3-Coder-480B": [ 36.3855421686747, 1.485542169, "div" ], "OpenHands/Qwen3-Coder-Next": [ 42.494226327944574, -3.205773672, "div" ], "OpenHands/Qwen3.5-Flash": [ 36.95150115473441, -1.348498845, "div" ], "OpenHands/Qwen3.6-Plus": [ 60.969976905311775, -1.130023095, "div" ], "OpenHands/Trinity-Large-Thinking": [ 33.02540415704388, -0.274595843, 2 ], "OpenHands/claude-fable-5": [ 91.91685912240185, 0.016859122, 0 ], "OpenHands/claude-opus-4-5": [ 78.52193995381063, 0.021939954, 0 ], "OpenHands/claude-opus-4-6": [ 78.75288683602771, -0.047113164, 0 ], "OpenHands/claude-opus-4-7": [ 80.83140877598153, 0.031408776, 0 ], "OpenHands/claude-opus-4-8": [ 84.29561200923787, -0.004387991, 0 ], "OpenHands/claude-sonnet-4-5": [ 69.30232558139535, 0.502325581, "div" ], "OpenHands/claude-sonnet-4-6": [ 54.04157043879908, 0.041570439, 0 ] }, "t0": 15, "t1": 0, "t2": 7, "undef": 0 }, "S2_frac_Dobs": { "div": 34, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 0.5446009389671361, -53.055399061, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 0.6096997690531177, -67.490300231, "div" ], "OpenHands/GLM-4.7": [ 0.5071090047393365, -48.892890995, "div" ], "OpenHands/GLM-5": [ 0.6137724550898204, -46.686227545, "div" ], "OpenHands/GLM-5.1": [ 0.6974595842956121, -69.502540416, "div" ], "OpenHands/GPT-5.2": [ 0.7423887587822015, -72.457611241, "div" ], "OpenHands/GPT-5.2-Codex": [ 0.6697459584295612, -66.330254042, "div" ], "OpenHands/GPT-5.4": [ 0.7043879907621247, -69.695612009, "div" ], "OpenHands/GPT-5.5": [ 0.8337182448036952, -82.566281755, "div" ], "OpenHands/Gemini-3-Flash": [ 0.7099767981438515, -69.990023202, "div" ], "OpenHands/Gemini-3-Pro": [ 0.6875, -67.9125, "div" ], "OpenHands/Gemini-3.1-Pro": [ 0.7482678983833718, -74.351732102, "div" ], "OpenHands/Gemini-3.5-Flash": [ 0.7638888888888888, -75.436111111, "div" ], "OpenHands/Kimi-K2-Thinking": [ 0.47344110854503463, -46.826558891, "div" ], "OpenHands/Kimi-K2.5": [ 0.6203703703703703, -61.27962963, "div" ], "OpenHands/Kimi-K2.6": [ 0.7043879907621247, -69.695612009, "div" ], "OpenHands/MiniMax-M2.1": [ 0.6143187066974596, -60.785681293, "div" ], "OpenHands/MiniMax-M2.5": [ 0.6812933025404158, -67.418706697, "div" ], "OpenHands/MiniMax-M2.7": [ 0.6905311778290993, -68.409468822, "div" ], "OpenHands/MiniMax-M3": [ 0.8106235565819861, -80.289376443, "div" ], "OpenHands/Nemotron-3-Nano": [ 0.0, -7.6, "div" ], "OpenHands/Nemotron-3-Super": [ 0.42032332563510394, -45.279676674, "div" ], "OpenHands/Qwen3-Coder-480B": [ 0.363855421686747, -34.536144578, "div" ], "OpenHands/Qwen3-Coder-Next": [ 0.42494226327944573, -45.275057737, "div" ], "OpenHands/Qwen3.5-Flash": [ 0.3695150115473441, -37.930484988, "div" ], "OpenHands/Qwen3.6-Plus": [ 0.6096997690531177, -61.490300231, "div" ], "OpenHands/Trinity-Large-Thinking": [ 0.3302540415704388, -32.969745958, "div" ], "OpenHands/claude-fable-5": [ 0.9191685912240185, -90.980831409, "div" ], "OpenHands/claude-opus-4-5": [ 0.7852193995381063, -77.7147806, "div" ], "OpenHands/claude-opus-4-6": [ 0.7875288683602771, -78.012471132, "div" ], "OpenHands/claude-opus-4-7": [ 0.8083140877598153, -79.991685912, "div" ], "OpenHands/claude-opus-4-8": [ 0.8429561200923787, -83.45704388, "div" ], "OpenHands/claude-sonnet-4-5": [ 0.6930232558139535, -68.106976744, "div" ], "OpenHands/claude-sonnet-4-6": [ 0.5404157043879908, -53.459584296, "div" ] }, "t0": 0, "t1": 0, "t2": 0, "undef": 0 }, "S2_pct_Dobs": { "div": 12, "per_row": { "OpenHands/DeepSeek-V3.2-Reasoner": [ 54.460093896713616, 0.860093897, "div" ], "OpenHands/DeepSeek-V4-Pro": [ 60.969976905311775, -7.130023095, "div" ], "OpenHands/GLM-4.7": [ 50.71090047393365, 1.310900474, "div" ], "OpenHands/GLM-5": [ 61.377245508982035, 14.077245509, "div" ], "OpenHands/GLM-5.1": [ 69.7459584295612, -0.45404157, 2 ], "OpenHands/GPT-5.2": [ 74.23887587822014, 1.038875878, "div" ], "OpenHands/GPT-5.2-Codex": [ 66.97459584295612, -0.025404157, 0 ], "OpenHands/GPT-5.4": [ 70.43879907621248, 0.038799076, 0 ], "OpenHands/GPT-5.5": [ 83.37182448036951, -0.02817552, 0 ], "OpenHands/Gemini-3-Flash": [ 70.99767981438515, 0.297679814, 2 ], "OpenHands/Gemini-3-Pro": [ 68.75, 0.15, 2 ], "OpenHands/Gemini-3.1-Pro": [ 74.82678983833718, -0.273210162, 2 ], "OpenHands/Gemini-3.5-Flash": [ 76.38888888888889, 0.188888889, 2 ], "OpenHands/Kimi-K2-Thinking": [ 47.34411085450346, 0.044110855, 0 ], "OpenHands/Kimi-K2.5": [ 62.03703703703704, 0.137037037, 2 ], "OpenHands/Kimi-K2.6": [ 70.43879907621248, 0.038799076, 0 ], "OpenHands/MiniMax-M2.1": [ 61.43187066974596, 0.03187067, 0 ], "OpenHands/MiniMax-M2.5": [ 68.12933025404158, 0.029330254, 0 ], "OpenHands/MiniMax-M2.7": [ 69.05311778290994, -0.046882217, 0 ], "OpenHands/MiniMax-M3": [ 81.06235565819861, -0.037644342, 0 ], "OpenHands/Nemotron-3-Nano": [ 0.0, -7.6, "div" ], "OpenHands/Nemotron-3-Super": [ 42.032332563510394, -3.667667436, "div" ], "OpenHands/Qwen3-Coder-480B": [ 36.3855421686747, 1.485542169, "div" ], "OpenHands/Qwen3-Coder-Next": [ 42.494226327944574, -3.205773672, "div" ], "OpenHands/Qwen3.5-Flash": [ 36.95150115473441, -1.348498845, "div" ], "OpenHands/Qwen3.6-Plus": [ 60.969976905311775, -1.130023095, "div" ], "OpenHands/Trinity-Large-Thinking": [ 33.02540415704388, -0.274595843, 2 ], "OpenHands/claude-fable-5": [ 91.91685912240185, 0.016859122, 0 ], "OpenHands/claude-opus-4-5": [ 78.52193995381063, 0.021939954, 0 ], "OpenHands/claude-opus-4-6": [ 78.75288683602771, -0.047113164, 0 ], "OpenHands/claude-opus-4-7": [ 80.83140877598153, 0.031408776, 0 ], "OpenHands/claude-opus-4-8": [ 84.29561200923787, -0.004387991, 0 ], "OpenHands/claude-sonnet-4-5": [ 69.30232558139535, 0.502325581, "div" ], "OpenHands/claude-sonnet-4-6": [ 54.04157043879908, 0.041570439, 0 ] }, "t0": 15, "t1": 0, "t2": 7, "undef": 0 } }, "primary": "S1_pct_Dall", "published_range": [ 7.6, 91.9 ], "zero_variance": false } }, "hash_check": { "equal": false, "executed": true, "recomputed": "3c89483162c11ca7a8c44085b3e85fb08e310864267c8f4fd77aa2e52874ba00", "shipped_source_hash": "886a3ebabd70879b712deb36fcfc190199e9c91a0ea538a89b156a20663fda29" }, "instances_side": { "cost_nulls": 436, "distinct_benchmarks": [ "commit0", "gaia", "swe-bench", "swe-bench-multimodal", "swt-bench" ], "groups_without_test_row": [], "n_groups": 34, "n_rows": 40643, "resolved_nulls": 475, "test_rows_without_group": [] }, "join": { "candidates": { "K1_id": { "coverage": 1.0, "n_instance_groups": 34, "test_key_unique": true }, "K2_language_model": { "coverage": 1.0, "n_instance_groups": 34, "test_key_unique": true }, "K3_agentname_lm": { "coverage": 1.0, "n_instance_groups": 34, "test_key_unique": true }, "K4_id_lower": { "coverage": 1.0, "n_instance_groups": 34, "test_key_unique": true } }, "runner_up": "K2_language_model", "winner": "K1_id" }, "meta": { "impl": "A", "prereg": "PREREG_A2-id_2026-07-25.md" }, "nulls": { "agent_name": { "note": "null distribution degenerate; floor structural, ratios undefined", "structural_zero_variance": true }, "agent_type": { "note": "null distribution degenerate; floor structural, ratios undefined", "structural_zero_variance": true }, "average_cost": { "mad_median": 1.2744023866994088, "mad_p5": 1.065878555799962, "n_draws": 1000, "observed_mad": 0.14412172246358107, "observed_t0": 4, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "average_score": { "mad_median": 14.87084154072382, "mad_p5": 12.431161272943912, "n_draws": 1000, "observed_mad": 1.23388068302469, "observed_t0": 1, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 0.0 }, "categories_completed": { "note": "null distribution degenerate; floor structural, ratios undefined", "structural_zero_variance": true }, "frontend_cost": { "mad_median": 1.4079336703719385, "mad_p5": 1.1789419649502273, "n_draws": 1000, "observed_mad": 0.4263151833319061, "observed_t0": 4, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "frontend_score": { "mad_median": 13.302934514794456, "mad_p5": 11.36521759638249, "n_draws": 1000, "observed_mad": 5.761275053697801, "observed_t0": 12, "seed": 20260725, "t0_median": 1.0, "t0_p5": 0.0, "t0_p95": 3.0 }, "greenfield_cost": { "mad_median": 3.6332918523573388, "mad_p5": 3.0273756710987274, "n_draws": 1000, "observed_mad": 0.4513671909637705, "observed_t0": 6, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "greenfield_score": { "mad_median": 19.125, "mad_p5": 15.807352941176468, "n_draws": 1000, "observed_mad": 0.016176470588234976, "observed_t0": 34, "seed": 20260725, "t0_median": 5.0, "t0_p5": 2.0, "t0_p95": 8.0 }, "information_gathering_cost": { "mad_median": 0.8783791735777886, "mad_p5": 0.6930726342166862, "n_draws": 1000, "observed_mad": 0.03469798662041565, "observed_t0": 7, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "information_gathering_score": { "mad_median": 24.224094632099096, "mad_p5": 19.62963830411336, "n_draws": 1000, "observed_mad": 0.28995915972833286, "observed_t0": 32, "seed": 20260725, "t0_median": 1.0, "t0_p5": 0.0, "t0_p95": 3.0 }, "issue_resolution_cost": { "mad_median": 0.6530343725176413, "mad_p5": 0.5379216007036475, "n_draws": 1000, "observed_mad": 0.017307726345861647, "observed_t0": 6, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "issue_resolution_score": { "mad_median": 8.94705882352941, "mad_p5": 7.381764705882351, "n_draws": 1000, "observed_mad": 0.005882352941176554, "observed_t0": 33, "seed": 20260725, "t0_median": 1.0, "t0_p5": 0.0, "t0_p95": 4.0 }, "language_model": { "mad_median": null, "mad_p5": null, "n_draws": 1000, "observed_mad": null, "observed_t0": 34, "seed": 20260725, "t0_median": 1.0, "t0_p5": 0.0, "t0_p95": 3.0 }, "testing_cost": { "mad_median": 0.5581489595324347, "mad_p5": 0.4608154310215198, "n_draws": 1000, "observed_mad": 0.01627007491202649, "observed_t0": 6, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 1.0 }, "testing_score": { "mad_median": 19.239281272155146, "mad_p5": 15.853197193541973, "n_draws": 1000, "observed_mad": 1.3416681934848422, "observed_t0": 15, "seed": 20260725, "t0_median": 0.0, "t0_p5": 0.0, "t0_p95": 2.0 } }, "self_id": { "A2_match_type": "exact", "A2_rows": { "0": { "file_order_rank_1based": 1, "published_row": { "agent_name": "OpenHands", "agent_type": "OpenHands", "average_cost": 5.538, "average_runtime": 458.6, "average_score": 81.0, "categories_completed": 5, "country": "us", "frontend_cost": 4.39, "frontend_logs_url": "https://results.eval.all-hands.dev/swebenchmultimodal/litellm_proxy-anthropic-claude-fable-5/27366967295/results.tar.gz", "frontend_runtime": 551.0, "frontend_score": 70.6, "frontend_visualization_url": "https://laminar.sh/shared/evals/53fd0c2a-c7bf-4efa-b74d-14258d81ed7a", "greenfield_cost": 12.49, "greenfield_logs_url": "https://results.eval.all-hands.dev/commit0/litellm_proxy-anthropic-claude-fable-5/27377500391/results.tar.gz", "greenfield_runtime": 1094.0, "greenfield_score": 62.5, "greenfield_visualization_url": "https://laminar.sh/shared/evals/95690e32-8d2e-4106-9a37-2f7d424a8494", "id": "OpenHands/claude-fable-5", "information_gathering_cost": 7.91, "information_gathering_logs_url": "https://results.eval.all-hands.dev/gaia/litellm_proxy-anthropic-claude-fable-5/27377526161/results.tar.gz", "information_gathering_runtime": 206.0, "information_gathering_score": 84.2, "information_gathering_visualization_url": "https://laminar.sh/shared/evals/1171bd32-73bf-4795-b4a3-40b42cf90fa3", "issue_resolution_cost": 1.43, "issue_resolution_logs_url": "https://results.eval.all-hands.dev/swebench/litellm_proxy-anthropic-claude-fable-5/27367020821/results.tar.gz", "issue_resolution_runtime": 222.0, "issue_resolution_score": 95.8, "issue_resolution_visualization_url": "https://laminar.sh/shared/evals/09f76346-1f26-4c2b-a2b9-d23108569bab", "language_model": "claude-fable-5", "openness": "closed_api_available", "release_date": "2026-06-09", "sdk_version": "v1.18.1", "supports_vision": true, "testing_cost": 1.47, "testing_logs_url": "https://results.eval.all-hands.dev/swtbench/litellm_proxy-anthropic-claude-fable-5/27383481991/results.tar.gz", "testing_runtime": 220.0, "testing_score": 91.9, "testing_visualization_url": "https://laminar.sh/shared/evals/e927fb1c-feb7-4d96-9e2c-f969cd6feccb" }, "rank_by_average_score": 1 } }, "B2_candidate_rows": { "18": { "file_order_rank_1based": 19, "published_row": { "agent_name": "OpenHands", "agent_type": "OpenHands", "average_cost": 1.42, "average_runtime": 1090.6, "average_score": 49.04, "categories_completed": 5, "country": "us", "frontend_cost": 1.46, "frontend_logs_url": "https://results.eval.all-hands.dev/eval-21320838747-gemini-3-p_litellm_proxy-gemini-gemini-3-pro-preview_26-01-24-22-39.tar.gz", "frontend_runtime": 710.0, "frontend_score": 36.8, "frontend_visualization_url": null, "greenfield_cost": 3.18, "greenfield_logs_url": "https://results.eval.all-hands.dev/commit0/litellm_proxy-gemini-3-pro-preview/22100005487/results.tar.gz", "greenfield_runtime": 2239.0, "greenfield_score": 25.0, "greenfield_visualization_url": "https://laminar.sh/shared/evals/bfba3513-7461-42da-9944-874848240c29", "id": "OpenHands/Gemini-3-Pro", "information_gathering_cost": 0.5, "information_gathering_logs_url": "https://results.eval.all-hands.dev/eval-21272815800-gemini-3-p_litellm_proxy-gemini-gemini-3-pro-preview_26-01-24-00-30.tar.gz", "information_gathering_runtime": 1775.0, "information_gathering_score": 44.2, "information_gathering_visualization_url": null, "issue_resolution_cost": 0.95, "issue_resolution_logs_url": "https://results.eval.all-hands.dev/eval-21011486666-gemini-3-p_litellm_proxy-gemini-gemini-3-pro-preview_26-01-15-07-43.tar.gz", "issue_resolution_runtime": 343.0, "issue_resolution_score": 70.6, "issue_resolution_visualization_url": null, "language_model": "Gemini-3-Pro", "openness": "closed_api_available", "release_date": "2025-11-18", "sdk_version": "v1.8.3", "supports_vision": true, "testing_cost": 1.01, "testing_logs_url": "https://results.eval.all-hands.dev/eval-21186630675-gemini-3-p_litellm_proxy-gemini-gemini-3-pro-preview_26-01-21-07-16.tar.gz", "testing_runtime": 386.0, "testing_score": 68.6, "testing_visualization_url": null }, "rank_by_average_score": 19 }, "19": { "file_order_rank_1based": 20, "published_row": { "agent_name": "OpenHands", "agent_type": "OpenHands", "average_cost": 0.904, "average_runtime": 767.6, "average_score": 49.0, "categories_completed": 5, "country": "us", "frontend_cost": 0.8, "frontend_logs_url": "https://results.eval.all-hands.dev/eval-21324688868-gemini-3-f_litellm_proxy-gemini-gemini-3-flash-preview_26-01-25-09-36.tar.gz", "frontend_runtime": 1152.0, "frontend_score": 22.1, "frontend_visualization_url": null, "greenfield_cost": 2.62, "greenfield_logs_url": "https://results.eval.all-hands.dev/commit0/litellm_proxy-gemini-3-flash-preview/25033158735/results.tar.gz", "greenfield_runtime": 1732.0, "greenfield_score": 18.8, "greenfield_visualization_url": "https://laminar.sh/shared/evals/3501752b-0081-4611-9807-335281599b80", "id": "OpenHands/Gemini-3-Flash", "information_gathering_cost": 0.38, "information_gathering_logs_url": "https://results.eval.all-hands.dev/eval-20821586318-gemini-3-f_litellm_proxy-gemini-gemini-3-flash-preview_26-01-08-20-24.tar.gz", "information_gathering_runtime": 398.0, "information_gathering_score": 58.8, "information_gathering_visualization_url": null, "issue_resolution_cost": 0.42, "issue_resolution_logs_url": "https://results.eval.all-hands.dev/eval-21179606180-gemini-3-f_litellm_proxy-gemini-gemini-3-flash-preview_26-01-21-03-09.tar.gz", "issue_resolution_runtime": 343.0, "issue_resolution_score": 74.6, "issue_resolution_visualization_url": null, "language_model": "Gemini-3-Flash", "openness": "closed_api_available", "release_date": "2025-12-22", "sdk_version": "v1.8.3", "supports_vision": true, "testing_cost": 0.3, "testing_logs_url": "https://results.eval.all-hands.dev/eval-21179607467-gemini-3-f_litellm_proxy-gemini-gemini-3-flash-preview_26-01-21-01-44.tar.gz", "testing_runtime": 213.0, "testing_score": 70.7, "testing_visualization_url": null }, "rank_by_average_score": 20 }, "6": { "file_order_rank_1based": 7, "published_row": { "agent_name": "OpenHands", "agent_type": "OpenHands", "average_cost": 3.018, "average_runtime": 783.6, "average_score": 62.64, "categories_completed": 5, "country": "us", "frontend_cost": 1.99, "frontend_logs_url": "https://results.eval.all-hands.dev/swebenchmultimodal/litellm_proxy-gemini-3-5-flash/27302024201/results.tar.gz", "frontend_runtime": 706.0, "frontend_score": 39.7, "frontend_visualization_url": "https://laminar.sh/shared/evals/20d43172-1fef-4d83-af1b-01784887e7ca", "greenfield_cost": 7.91, "greenfield_logs_url": "https://results.eval.all-hands.dev/commit0/litellm_proxy-gemini-3-5-flash/27302032453/results.tar.gz", "greenfield_runtime": 1760.0, "greenfield_score": 37.5, "greenfield_visualization_url": "https://laminar.sh/shared/evals/f73dcfa9-a6de-4b41-bcf6-abc1d2b0ba58", "id": "OpenHands/Gemini-3.5-Flash", "information_gathering_cost": 2.63, "information_gathering_logs_url": "https://results.eval.all-hands.dev/gaia/litellm_proxy-gemini-3-5-flash/27377762610/results.tar.gz", "information_gathering_runtime": 656.0, "information_gathering_score": 81.2, "information_gathering_visualization_url": "https://laminar.sh/shared/evals/0e76f325-8100-4491-8a2d-37fb1ba84d62", "issue_resolution_cost": 1.53, "issue_resolution_logs_url": "https://results.eval.all-hands.dev/swebench/litellm_proxy-gemini-3-5-flash/27377757089/results.tar.gz", "issue_resolution_runtime": 428.0, "issue_resolution_score": 78.6, "issue_resolution_visualization_url": "https://laminar.sh/shared/evals/60edc213-0c2b-4e3f-9c72-ba4bfce1cb32", "language_model": "Gemini-3.5-Flash", "openness": "closed_api_available", "release_date": "2026-05-19", "sdk_version": "v1.24.0", "supports_vision": true, "testing_cost": 1.03, "testing_logs_url": "https://results.eval.all-hands.dev/swtbench/litellm_proxy-gemini-3-5-flash/27324130544/results.tar.gz", "testing_runtime": 368.0, "testing_score": 76.2, "testing_visualization_url": "https://laminar.sh/shared/evals/88a0e23e-741b-4eba-a6a6-87056fb23905" }, "rank_by_average_score": 7 }, "8": { "file_order_rank_1based": 9, "published_row": { "agent_name": "OpenHands", "agent_type": "OpenHands", "average_cost": 1.542, "average_runtime": 786.0, "average_score": 60.56, "categories_completed": 5, "country": "us", "frontend_cost": 1.24, "frontend_logs_url": "https://results.eval.all-hands.dev/swebenchmultimodal/litellm_proxy-gemini-3-1-pro-preview/22671316816/results.tar.gz", "frontend_runtime": 1868.0, "frontend_score": 44.1, "frontend_visualization_url": "https://laminar.sh/shared/evals/cca1e142-0f1b-45ef-95fc-fde8e4c9d3aa", "greenfield_cost": 1.18, "greenfield_logs_url": "https://results.eval.all-hands.dev/commit0/litellm_proxy-gemini-3-1-pro-preview/25033174380/results.tar.gz", "greenfield_runtime": 446.0, "greenfield_score": 25.0, "greenfield_visualization_url": "https://laminar.sh/shared/evals/3817b8ac-4a44-42f7-b6ce-08f747db0061", "id": "OpenHands/Gemini-3.1-Pro", "information_gathering_cost": 1.8, "information_gathering_logs_url": "https://results.eval.all-hands.dev/gaia/litellm_proxy-gemini-3-1-pro-preview/26882098006/results.tar.gz", "information_gathering_runtime": 434.0, "information_gathering_score": 81.8, "information_gathering_visualization_url": "https://laminar.sh/shared/evals/3a131e0e-fdcd-4898-bfeb-7320496bbafd", "issue_resolution_cost": 1.87, "issue_resolution_logs_url": "https://results.eval.all-hands.dev/swebench/litellm_proxy-gemini-3-1-pro-preview/27326680626/results.tar.gz", "issue_resolution_runtime": 716.0, "issue_resolution_score": 76.8, "issue_resolution_visualization_url": "https://laminar.sh/shared/evals/1e2fadc9-bfce-4a2d-80a2-da24b6e707ad", "language_model": "Gemini-3.1-Pro", "openness": "closed_api_available", "release_date": "2026-01-15", "sdk_version": "v1.11.5", "supports_vision": true, "testing_cost": 1.62, "testing_logs_url": "https://results.eval.all-hands.dev/swtbench/litellm_proxy-gemini-3-1-pro-preview/27236525259/results.tar.gz", "testing_runtime": 466.0, "testing_score": 75.1, "testing_visualization_url": "https://laminar.sh/shared/evals/c44c25a6-214d-4f87-aa3c-fdb8c3531d13" }, "rank_by_average_score": 9 } }, "rule": "exact lower(language_model)=='claude-fable-5', fallback substring 'fable'; B2: startswith 'gemini' (pin unknown to this arm)" }, "tie_census": { "published_average_score": { "adjacent_ties_in_desc_sort": 0, "distinct": 34, "n": 34, "strictly_decidable": true, "tied_pairs": 0 }, "published_sort_descending": true, "recomputed_primary_2dp": { "adjacent_ties_in_desc_sort": 0, "distinct": 34, "n": 34, "strictly_decidable": true, "tied_pairs": 0 }, "recomputed_primary_raw": { "adjacent_ties_in_desc_sort": 0, "distinct": 34, "n": 34, "strictly_decidable": true, "tied_pairs": 0 } } }