[ { "name": "Python", "data": [ { "name": "Verified", "results": [ { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 261, "resolvedEasy": 139, "resolvedMedium": 117, "resolvedHard": 5, "resolvedRate": 0.522, "resolvedEasyRate": 0.7164948453608248, "resolvedMediumRate": 0.4482758620689655, "resolvedHardRate": 0.1111111111111111, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 245, "resolvedEasy": 130, "resolvedMedium": 110, "resolvedHard": 5, "resolvedRate": 0.49, "resolvedEasyRate": 0.6701030927835051, "resolvedMediumRate": 0.421455938697318, "resolvedHardRate": 0.1111111111111111, "date": "2025-04-26", "path": "python/verified/20250426_Agentless_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 241, "resolvedEasy": 132, "resolvedMedium": 105, "resolvedHard": 4, "resolvedRate": 0.482, "resolvedEasyRate": 0.6804123711340206, "resolvedMediumRate": 0.40229885057471265, "resolvedHardRate": 0.08888888888888889, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 232, "resolvedEasy": 130, "resolvedMedium": 100, "resolvedHard": 2, "resolvedRate": 0.464, "resolvedEasyRate": 0.6701030927835051, "resolvedMediumRate": 0.3831417624521073, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 229, "resolvedEasy": 120, "resolvedMedium": 106, "resolvedHard": 3, "resolvedRate": 0.458, "resolvedEasyRate": 0.6185567010309279, "resolvedMediumRate": 0.4061302681992337, "resolvedHardRate": 0.06666666666666667, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 229, "resolvedEasy": 124, "resolvedMedium": 103, "resolvedHard": 2, "resolvedRate": 0.458, "resolvedEasyRate": 0.6391752577319587, "resolvedMediumRate": 0.3946360153256705, "resolvedHardRate": 0.044444444444444446, "date": "2025-04-26", "path": "python/verified/20250426_OpenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 224, "resolvedEasy": 121, "resolvedMedium": 99, "resolvedHard": 4, "resolvedRate": 0.448, "resolvedEasyRate": 0.6237113402061856, "resolvedMediumRate": 0.3793103448275862, "resolvedHardRate": 0.08888888888888889, "date": "2025-04-26", "path": "python/verified/20250426_Agentless_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 223, "resolvedEasy": 125, "resolvedMedium": 93, "resolvedHard": 5, "resolvedRate": 0.446, "resolvedEasyRate": 0.6443298969072165, "resolvedMediumRate": 0.3563218390804598, "resolvedHardRate": 0.1111111111111111, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 212, "resolvedEasy": 120, "resolvedMedium": 89, "resolvedHard": 3, "resolvedRate": 0.424, "resolvedEasyRate": 0.6185567010309279, "resolvedMediumRate": 0.34099616858237547, "resolvedHardRate": 0.06666666666666667, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 211, "resolvedEasy": 114, "resolvedMedium": 94, "resolvedHard": 3, "resolvedRate": 0.422, "resolvedEasyRate": 0.5876288659793815, "resolvedMediumRate": 0.36015325670498083, "resolvedHardRate": 0.06666666666666667, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 205, "resolvedEasy": 112, "resolvedMedium": 91, "resolvedHard": 2, "resolvedRate": 0.41, "resolvedEasyRate": 0.5773195876288659, "resolvedMediumRate": 0.3486590038314176, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 195, "resolvedEasy": 95, "resolvedMedium": 94, "resolvedHard": 6, "resolvedRate": 0.39, "resolvedEasyRate": 0.4896907216494845, "resolvedMediumRate": 0.36015325670498083, "resolvedHardRate": 0.13333333333333333, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 189, "resolvedEasy": 119, "resolvedMedium": 69, "resolvedHard": 1, "resolvedRate": 0.378, "resolvedEasyRate": 0.6134020618556701, "resolvedMediumRate": 0.26436781609195403, "resolvedHardRate": 0.022222222222222223, "date": "2025-04-26", "path": "python/verified/20250426_Agentless_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 181, "resolvedEasy": 107, "resolvedMedium": 73, "resolvedHard": 1, "resolvedRate": 0.362, "resolvedEasyRate": 0.5515463917525774, "resolvedMediumRate": 0.2796934865900383, "resolvedHardRate": 0.022222222222222223, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 153, "resolvedEasy": 80, "resolvedMedium": 70, "resolvedHard": 3, "resolvedRate": 0.306, "resolvedEasyRate": 0.41237113402061853, "resolvedMediumRate": 0.2681992337164751, "resolvedHardRate": 0.06666666666666667, "date": "2025-04-26", "path": "python/verified/20250426_SWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 144, "resolvedEasy": 79, "resolvedMedium": 63, "resolvedHard": 2, "resolvedRate": 0.288, "resolvedEasyRate": 0.4072164948453608, "resolvedMediumRate": 0.2413793103448276, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 143, "resolvedEasy": 83, "resolvedMedium": 56, "resolvedHard": 4, "resolvedRate": 0.286, "resolvedEasyRate": 0.42783505154639173, "resolvedMediumRate": 0.21455938697318008, "resolvedHardRate": 0.08888888888888889, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 139, "resolvedEasy": 80, "resolvedMedium": 56, "resolvedHard": 3, "resolvedRate": 0.278, "resolvedEasyRate": 0.41237113402061853, "resolvedMediumRate": 0.21455938697318008, "resolvedHardRate": 0.06666666666666667, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 139, "resolvedEasy": 83, "resolvedMedium": 55, "resolvedHard": 1, "resolvedRate": 0.278, "resolvedEasyRate": 0.42783505154639173, "resolvedMediumRate": 0.210727969348659, "resolvedHardRate": 0.022222222222222223, "date": "2025-04-26", "path": "python/verified/20250426_OpenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 139, "resolvedEasy": 75, "resolvedMedium": 62, "resolvedHard": 2, "resolvedRate": 0.278, "resolvedEasyRate": 0.3865979381443299, "resolvedMediumRate": 0.23754789272030652, "resolvedHardRate": 0.044444444444444446, "date": "2025-04-26", "path": "python/verified/20250426_SWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 134, "resolvedEasy": 86, "resolvedMedium": 48, "resolvedHard": 0, "resolvedRate": 0.268, "resolvedEasyRate": 0.44329896907216493, "resolvedMediumRate": 0.1839080459770115, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/OpenAutoCoder/Agentless", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 131, "resolvedEasy": 76, "resolvedMedium": 53, "resolvedHard": 2, "resolvedRate": 0.262, "resolvedEasyRate": 0.3917525773195876, "resolvedMediumRate": 0.20306513409961685, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_Agentless_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 130, "resolvedEasy": 80, "resolvedMedium": 50, "resolvedHard": 0, "resolvedRate": 0.26, "resolvedEasyRate": 0.41237113402061853, "resolvedMediumRate": 0.19157088122605365, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 128, "resolvedEasy": 75, "resolvedMedium": 51, "resolvedHard": 2, "resolvedRate": 0.256, "resolvedEasyRate": 0.3865979381443299, "resolvedMediumRate": 0.19540229885057472, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 124, "resolvedEasy": 55, "resolvedMedium": 67, "resolvedHard": 2, "resolvedRate": 0.248, "resolvedEasyRate": 0.28350515463917525, "resolvedMediumRate": 0.2567049808429119, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 102, "resolvedEasy": 61, "resolvedMedium": 38, "resolvedHard": 3, "resolvedRate": 0.204, "resolvedEasyRate": 0.31443298969072164, "resolvedMediumRate": 0.14559386973180077, "resolvedHardRate": 0.06666666666666667, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 94, "resolvedEasy": 50, "resolvedMedium": 42, "resolvedHard": 2, "resolvedRate": 0.188, "resolvedEasyRate": 0.25773195876288657, "resolvedMediumRate": 0.16091954022988506, "resolvedHardRate": 0.044444444444444446, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 80, "resolvedEasy": 36, "resolvedMedium": 43, "resolvedHard": 1, "resolvedRate": 0.16, "resolvedEasyRate": 0.18556701030927836, "resolvedMediumRate": 0.16475095785440613, "resolvedHardRate": 0.022222222222222223, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 72, "resolvedEasy": 46, "resolvedMedium": 24, "resolvedHard": 2, "resolvedRate": 0.144, "resolvedEasyRate": 0.23711340206185566, "resolvedMediumRate": 0.09195402298850575, "resolvedHardRate": 0.044444444444444446, "date": "2025-04-26", "path": "python/verified/20250426_OpenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 62, "resolvedEasy": 34, "resolvedMedium": 28, "resolvedHard": 0, "resolvedRate": 0.124, "resolvedEasyRate": 0.17525773195876287, "resolvedMediumRate": 0.10727969348659004, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 44, "resolvedEasy": 30, "resolvedMedium": 13, "resolvedHard": 1, "resolvedRate": 0.088, "resolvedEasyRate": 0.15463917525773196, "resolvedMediumRate": 0.04980842911877394, "resolvedHardRate": 0.022222222222222223, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 43, "resolvedEasy": 30, "resolvedMedium": 13, "resolvedHard": 0, "resolvedRate": 0.086, "resolvedEasyRate": 0.15463917525773196, "resolvedMediumRate": 0.04980842911877394, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/All-Hands-AI/OpenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 22, "resolvedEasy": 13, "resolvedMedium": 9, "resolvedHard": 0, "resolvedRate": 0.044, "resolvedEasyRate": 0.06701030927835051, "resolvedMediumRate": 0.034482758620689655, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_OpenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 21, "resolvedEasy": 14, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.042, "resolvedEasyRate": 0.07216494845360824, "resolvedMediumRate": 0.02681992337164751, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 5, "resolvedMedium": 5, "resolvedHard": 0, "resolvedRate": 0.02, "resolvedEasyRate": 0.02577319587628866, "resolvedMediumRate": 0.019157088122605363, "resolvedHardRate": 0, "date": "2025-03-29", "path": "python/verified/20250329_SWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/princeton-nlp/SWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 6, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.02, "resolvedEasyRate": 0.030927835051546393, "resolvedMediumRate": 0.01532567049808429, "resolvedHardRate": 0, "date": "2025-04-26", "path": "python/verified/20250426_SWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "astropy__astropy-12907", "astropy__astropy-13033", "astropy__astropy-13236", "astropy__astropy-13398", "astropy__astropy-13453", "astropy__astropy-13579", "astropy__astropy-13977", "astropy__astropy-14096", "astropy__astropy-14182", "astropy__astropy-14309", "astropy__astropy-14365", "astropy__astropy-14369", "astropy__astropy-14508", "astropy__astropy-14539", "astropy__astropy-14598", "astropy__astropy-14995", "astropy__astropy-7166", "astropy__astropy-7336", "astropy__astropy-7606", "astropy__astropy-7671", "astropy__astropy-8707", "astropy__astropy-8872", "django__django-10097", "django__django-10554", "django__django-10880", "django__django-10914", "django__django-10973", "django__django-10999", "django__django-11066", "django__django-11087", "django__django-11095", "django__django-11099", "django__django-11119", "django__django-11133", "django__django-11138", "django__django-11141", "django__django-11149", "django__django-11163", "django__django-11179", "django__django-11206", "django__django-11211", "django__django-11239", "django__django-11265", "django__django-11276", "django__django-11292", "django__django-11299", "django__django-11333", "django__django-11400", "django__django-11433", "django__django-11451", "django__django-11477", "django__django-11490", "django__django-11532", "django__django-11551", "django__django-11555", "django__django-11603", "django__django-11728", "django__django-11734", "django__django-11740", "django__django-11749", "django__django-11790", "django__django-11815", "django__django-11820", "django__django-11848", "django__django-11880", "django__django-11885", "django__django-11951", "django__django-11964", "django__django-11999", "django__django-12039", "django__django-12050", "django__django-12125", "django__django-12143", "django__django-12155", "django__django-12193", "django__django-12209", "django__django-12262", "django__django-12273", "django__django-12276", "django__django-12304", "django__django-12308", "django__django-12325", "django__django-12406", "django__django-12419", "django__django-12663", "django__django-12708", "django__django-12713", "django__django-12741", "django__django-12754", "django__django-12774", "django__django-12858", "django__django-12965", "django__django-13012", "django__django-13023", "django__django-13028", "django__django-13033", "django__django-13089", "django__django-13109", "django__django-13112", "django__django-13121", "django__django-13128", "django__django-13158", "django__django-13195", "django__django-13212", "django__django-13279", "django__django-13297", "django__django-13315", "django__django-13343", "django__django-13344", "django__django-13346", "django__django-13363", "django__django-13401", "django__django-13406", "django__django-13410", "django__django-13417", "django__django-13449", "django__django-13512", "django__django-13513", "django__django-13516", "django__django-13551", "django__django-13568", "django__django-13569", "django__django-13590", "django__django-13658", "django__django-13670", "django__django-13741", "django__django-13786", "django__django-13794", "django__django-13807", "django__django-13809", "django__django-13810", "django__django-13820", "django__django-13821", "django__django-13837", "django__django-13925", "django__django-13933", "django__django-13964", "django__django-14007", "django__django-14011", "django__django-14017", "django__django-14034", "django__django-14053", "django__django-14089", "django__django-14122", "django__django-14140", "django__django-14155", "django__django-14170", "django__django-14238", "django__django-14311", "django__django-14315", "django__django-14349", "django__django-14351", "django__django-14373", "django__django-14376", "django__django-14404", "django__django-14434", "django__django-14493", "django__django-14500", "django__django-14534", "django__django-14539", "django__django-14559", "django__django-14580", "django__django-14608", "django__django-14631", "django__django-14672", "django__django-14725", "django__django-14752", "django__django-14765", "django__django-14771", "django__django-14787", "django__django-14792", "django__django-14855", "django__django-14915", "django__django-14999", "django__django-15022", "django__django-15037", "django__django-15098", "django__django-15103", "django__django-15104", "django__django-15127", "django__django-15128", "django__django-15161", "django__django-15252", "django__django-15268", "django__django-15277", "django__django-15278", "django__django-15280", "django__django-15315", "django__django-15368", "django__django-15375", "django__django-15380", "django__django-15382", "django__django-15467", "django__django-15499", "django__django-15503", "django__django-15525", "django__django-15554", "django__django-15561", "django__django-15563", "django__django-15569", "django__django-15572", "django__django-15629", "django__django-15695", "django__django-15731", "django__django-15732", "django__django-15741", "django__django-15814", "django__django-15851", "django__django-15863", "django__django-15916", "django__django-15930", "django__django-15957", "django__django-15973", "django__django-15987", "django__django-16032", "django__django-16082", "django__django-16100", "django__django-16116", "django__django-16136", "django__django-16139", "django__django-16145", "django__django-16255", "django__django-16256", "django__django-16263", "django__django-16315", "django__django-16333", "django__django-16429", "django__django-16454", "django__django-16485", "django__django-16493", "django__django-16502", "django__django-16527", "django__django-16560", "django__django-16569", "django__django-16595", "django__django-16612", "django__django-16631", "django__django-16642", "django__django-16661", "django__django-16662", "django__django-16667", "django__django-16801", "django__django-16819", "django__django-16877", "django__django-16899", "django__django-16901", "django__django-16938", "django__django-16950", "django__django-17029", "django__django-17084", "django__django-17087", "django__django-7530", "django__django-9296", "matplotlib__matplotlib-13989", "matplotlib__matplotlib-14623", "matplotlib__matplotlib-20488", "matplotlib__matplotlib-20676", "matplotlib__matplotlib-20826", "matplotlib__matplotlib-20859", "matplotlib__matplotlib-21568", "matplotlib__matplotlib-22719", "matplotlib__matplotlib-22865", "matplotlib__matplotlib-22871", "matplotlib__matplotlib-23299", "matplotlib__matplotlib-23314", "matplotlib__matplotlib-23412", "matplotlib__matplotlib-23476", "matplotlib__matplotlib-24026", "matplotlib__matplotlib-24149", "matplotlib__matplotlib-24177", "matplotlib__matplotlib-24570", "matplotlib__matplotlib-24627", "matplotlib__matplotlib-24637", "matplotlib__matplotlib-24870", "matplotlib__matplotlib-24970", "matplotlib__matplotlib-25122", "matplotlib__matplotlib-25287", "matplotlib__matplotlib-25311", "matplotlib__matplotlib-25332", "matplotlib__matplotlib-25479", "matplotlib__matplotlib-25775", "matplotlib__matplotlib-25960", "matplotlib__matplotlib-26113", "matplotlib__matplotlib-26208", "matplotlib__matplotlib-26291", "matplotlib__matplotlib-26342", "matplotlib__matplotlib-26466", "mwaskom__seaborn-3069", "mwaskom__seaborn-3187", "pallets__flask-5014", "psf__requests-1142", "psf__requests-1724", "psf__requests-1766", "psf__requests-1921", "psf__requests-2317", "psf__requests-2931", "psf__requests-5414", "psf__requests-6028", "pydata__xarray-2905", "pydata__xarray-3095", "pydata__xarray-3151", "pydata__xarray-3305", "pydata__xarray-3677", "pydata__xarray-3993", "pydata__xarray-4075", "pydata__xarray-4094", "pydata__xarray-4356", "pydata__xarray-4629", "pydata__xarray-4687", "pydata__xarray-4695", "pydata__xarray-4966", "pydata__xarray-6461", "pydata__xarray-6599", "pydata__xarray-6721", "pydata__xarray-6744", "pydata__xarray-6938", "pydata__xarray-6992", "pydata__xarray-7229", "pydata__xarray-7233", "pydata__xarray-7393", "pylint-dev__pylint-4551", "pylint-dev__pylint-4604", "pylint-dev__pylint-4661", "pylint-dev__pylint-4970", "pylint-dev__pylint-6386", "pylint-dev__pylint-6528", "pylint-dev__pylint-6903", "pylint-dev__pylint-7080", "pylint-dev__pylint-7277", "pylint-dev__pylint-8898", "pytest-dev__pytest-10051", "pytest-dev__pytest-10081", "pytest-dev__pytest-10356", "pytest-dev__pytest-5262", "pytest-dev__pytest-5631", "pytest-dev__pytest-5787", "pytest-dev__pytest-5809", "pytest-dev__pytest-5840", "pytest-dev__pytest-6197", "pytest-dev__pytest-6202", "pytest-dev__pytest-7205", "pytest-dev__pytest-7236", "pytest-dev__pytest-7324", "pytest-dev__pytest-7432", "pytest-dev__pytest-7490", "pytest-dev__pytest-7521", "pytest-dev__pytest-7571", "pytest-dev__pytest-7982", "pytest-dev__pytest-8399", "scikit-learn__scikit-learn-10297", "scikit-learn__scikit-learn-10844", "scikit-learn__scikit-learn-10908", "scikit-learn__scikit-learn-11310", "scikit-learn__scikit-learn-11578", "scikit-learn__scikit-learn-12585", "scikit-learn__scikit-learn-12682", "scikit-learn__scikit-learn-12973", "scikit-learn__scikit-learn-13124", "scikit-learn__scikit-learn-13135", "scikit-learn__scikit-learn-13142", "scikit-learn__scikit-learn-13328", "scikit-learn__scikit-learn-13439", "scikit-learn__scikit-learn-13496", "scikit-learn__scikit-learn-13779", "scikit-learn__scikit-learn-14053", "scikit-learn__scikit-learn-14087", "scikit-learn__scikit-learn-14141", "scikit-learn__scikit-learn-14496", "scikit-learn__scikit-learn-14629", "scikit-learn__scikit-learn-14710", "scikit-learn__scikit-learn-14894", "scikit-learn__scikit-learn-14983", "scikit-learn__scikit-learn-15100", "scikit-learn__scikit-learn-25102", "scikit-learn__scikit-learn-25232", "scikit-learn__scikit-learn-25747", "scikit-learn__scikit-learn-25931", "scikit-learn__scikit-learn-25973", "scikit-learn__scikit-learn-26194", "scikit-learn__scikit-learn-26323", "scikit-learn__scikit-learn-9288", "sphinx-doc__sphinx-10323", "sphinx-doc__sphinx-10435", "sphinx-doc__sphinx-10449", "sphinx-doc__sphinx-10466", "sphinx-doc__sphinx-10614", "sphinx-doc__sphinx-10673", "sphinx-doc__sphinx-11445", "sphinx-doc__sphinx-11510", "sphinx-doc__sphinx-7440", "sphinx-doc__sphinx-7454", "sphinx-doc__sphinx-7462", "sphinx-doc__sphinx-7590", "sphinx-doc__sphinx-7748", "sphinx-doc__sphinx-7757", "sphinx-doc__sphinx-7889", "sphinx-doc__sphinx-7910", "sphinx-doc__sphinx-7985", "sphinx-doc__sphinx-8035", "sphinx-doc__sphinx-8056", "sphinx-doc__sphinx-8120", "sphinx-doc__sphinx-8265", "sphinx-doc__sphinx-8269", "sphinx-doc__sphinx-8459", "sphinx-doc__sphinx-8475", "sphinx-doc__sphinx-8548", "sphinx-doc__sphinx-8551", "sphinx-doc__sphinx-8593", "sphinx-doc__sphinx-8595", "sphinx-doc__sphinx-8621", "sphinx-doc__sphinx-8638", "sphinx-doc__sphinx-8721", "sphinx-doc__sphinx-9229", "sphinx-doc__sphinx-9230", "sphinx-doc__sphinx-9258", "sphinx-doc__sphinx-9281", "sphinx-doc__sphinx-9320", "sphinx-doc__sphinx-9367", "sphinx-doc__sphinx-9461", "sphinx-doc__sphinx-9591", "sphinx-doc__sphinx-9602", "sphinx-doc__sphinx-9658", "sphinx-doc__sphinx-9673", "sphinx-doc__sphinx-9698", "sphinx-doc__sphinx-9711", "sympy__sympy-11618", "sympy__sympy-12096", "sympy__sympy-12419", "sympy__sympy-12481", "sympy__sympy-12489", "sympy__sympy-13031", "sympy__sympy-13091", "sympy__sympy-13372", "sympy__sympy-13480", "sympy__sympy-13551", "sympy__sympy-13615", "sympy__sympy-13647", "sympy__sympy-13757", "sympy__sympy-13798", "sympy__sympy-13852", "sympy__sympy-13877", "sympy__sympy-13878", "sympy__sympy-13974", "sympy__sympy-14248", "sympy__sympy-14531", "sympy__sympy-14711", "sympy__sympy-14976", "sympy__sympy-15017", "sympy__sympy-15345", "sympy__sympy-15349", "sympy__sympy-15599", "sympy__sympy-15809", "sympy__sympy-15875", "sympy__sympy-15976", "sympy__sympy-16450", "sympy__sympy-16597", "sympy__sympy-16766", "sympy__sympy-16792", "sympy__sympy-16886", "sympy__sympy-17139", "sympy__sympy-17318", "sympy__sympy-17630", "sympy__sympy-17655", "sympy__sympy-18189", "sympy__sympy-18199", "sympy__sympy-18211", "sympy__sympy-18698", "sympy__sympy-18763", "sympy__sympy-19040", "sympy__sympy-19346", "sympy__sympy-19495", "sympy__sympy-19637", "sympy__sympy-19783", "sympy__sympy-19954", "sympy__sympy-20154", "sympy__sympy-20428", "sympy__sympy-20438", "sympy__sympy-20590", "sympy__sympy-20801", "sympy__sympy-20916", "sympy__sympy-21379", "sympy__sympy-21596", "sympy__sympy-21612", "sympy__sympy-21847", "sympy__sympy-21930", "sympy__sympy-22080", "sympy__sympy-22456", "sympy__sympy-22714", "sympy__sympy-22914", "sympy__sympy-23262", "sympy__sympy-23413", "sympy__sympy-23534", "sympy__sympy-23824", "sympy__sympy-23950", "sympy__sympy-24066", "sympy__sympy-24213", "sympy__sympy-24443", "sympy__sympy-24539", "sympy__sympy-24562", "sympy__sympy-24661" ], "easy_ids": [ "astropy__astropy-14309", "astropy__astropy-14995", "astropy__astropy-7166", "astropy__astropy-7336", "django__django-10097", "django__django-10880", "django__django-10914", "django__django-10999", "django__django-11066", "django__django-11099", "django__django-11119", "django__django-11133", "django__django-11163", "django__django-11179", "django__django-11239", "django__django-11299", "django__django-11433", "django__django-11451", "django__django-11490", "django__django-11555", "django__django-11603", "django__django-11820", "django__django-11880", "django__django-11951", "django__django-11964", "django__django-12125", "django__django-12193", "django__django-12209", "django__django-12276", "django__django-12304", "django__django-12308", "django__django-12419", "django__django-12741", "django__django-13023", "django__django-13089", "django__django-13109", "django__django-13112", "django__django-13297", "django__django-13363", "django__django-13406", "django__django-13410", "django__django-13417", "django__django-13512", "django__django-13551", "django__django-13741", "django__django-13794", "django__django-13807", "django__django-13821", "django__django-13933", "django__django-14089", "django__django-14315", "django__django-14373", "django__django-14376", "django__django-14404", "django__django-14493", "django__django-14534", "django__django-14539", "django__django-14580", "django__django-14608", "django__django-14752", "django__django-14765", "django__django-14787", "django__django-14792", "django__django-14855", "django__django-14915", "django__django-14999", "django__django-15104", "django__django-15127", "django__django-15277", "django__django-15315", "django__django-15368", "django__django-15380", "django__django-15467", "django__django-15499", "django__django-15569", "django__django-15572", "django__django-15741", "django__django-15851", "django__django-15863", "django__django-15930", "django__django-15987", "django__django-16100", "django__django-16116", "django__django-16139", "django__django-16145", "django__django-16255", "django__django-16333", "django__django-16429", "django__django-16569", "django__django-16595", "django__django-16642", "django__django-16801", "django__django-16899", "django__django-17029", "django__django-17087", "django__django-9296", "matplotlib__matplotlib-13989", "matplotlib__matplotlib-20676", "matplotlib__matplotlib-20859", "matplotlib__matplotlib-22719", "matplotlib__matplotlib-23476", "matplotlib__matplotlib-24149", "matplotlib__matplotlib-24177", "matplotlib__matplotlib-24570", "matplotlib__matplotlib-25122", "matplotlib__matplotlib-25287", "matplotlib__matplotlib-25311", "matplotlib__matplotlib-25332", "matplotlib__matplotlib-25479", "matplotlib__matplotlib-26113", "matplotlib__matplotlib-26208", "pallets__flask-5014", "psf__requests-1142", "psf__requests-1724", "psf__requests-1766", "psf__requests-1921", "psf__requests-2317", "psf__requests-5414", "pydata__xarray-4075", "pydata__xarray-4094", "pydata__xarray-4356", "pydata__xarray-4629", "pydata__xarray-6461", "pylint-dev__pylint-4970", "pylint-dev__pylint-6903", "pylint-dev__pylint-7277", "pytest-dev__pytest-10081", "pytest-dev__pytest-5262", "pytest-dev__pytest-5809", "pytest-dev__pytest-6202", "pytest-dev__pytest-7205", "pytest-dev__pytest-7432", "pytest-dev__pytest-7521", "pytest-dev__pytest-7982", "scikit-learn__scikit-learn-11310", "scikit-learn__scikit-learn-12585", "scikit-learn__scikit-learn-13135", "scikit-learn__scikit-learn-13142", "scikit-learn__scikit-learn-13328", "scikit-learn__scikit-learn-13439", "scikit-learn__scikit-learn-13496", "scikit-learn__scikit-learn-13779", "scikit-learn__scikit-learn-14141", "scikit-learn__scikit-learn-14496", "scikit-learn__scikit-learn-14983", "scikit-learn__scikit-learn-25232", "scikit-learn__scikit-learn-25973", "sphinx-doc__sphinx-10323", "sphinx-doc__sphinx-10435", "sphinx-doc__sphinx-10449", "sphinx-doc__sphinx-7440", "sphinx-doc__sphinx-7454", "sphinx-doc__sphinx-7462", "sphinx-doc__sphinx-7889", "sphinx-doc__sphinx-7910", "sphinx-doc__sphinx-8269", "sphinx-doc__sphinx-8459", "sphinx-doc__sphinx-8475", "sphinx-doc__sphinx-8595", "sphinx-doc__sphinx-8621", "sphinx-doc__sphinx-8721", "sphinx-doc__sphinx-9230", "sphinx-doc__sphinx-9258", "sphinx-doc__sphinx-9281", "sphinx-doc__sphinx-9320", "sphinx-doc__sphinx-9367", "sphinx-doc__sphinx-9591", "sphinx-doc__sphinx-9698", "sphinx-doc__sphinx-9711", "sympy__sympy-12096", "sympy__sympy-12481", "sympy__sympy-13372", "sympy__sympy-13480", "sympy__sympy-14711", "sympy__sympy-15017", "sympy__sympy-15345", "sympy__sympy-15809", "sympy__sympy-15875", "sympy__sympy-16450", "sympy__sympy-16766", "sympy__sympy-16886", "sympy__sympy-17139", "sympy__sympy-17655", "sympy__sympy-18189", "sympy__sympy-18763", "sympy__sympy-19495", "sympy__sympy-19637", "sympy__sympy-19954", "sympy__sympy-20916", "sympy__sympy-21847", "sympy__sympy-22714", "sympy__sympy-23534", "sympy__sympy-24539", "sympy__sympy-24562" ], "medium_ids": [ "astropy__astropy-12907", "astropy__astropy-13033", "astropy__astropy-13236", "astropy__astropy-13453", "astropy__astropy-13977", "astropy__astropy-14096", "astropy__astropy-14182", "astropy__astropy-14365", "astropy__astropy-14508", "astropy__astropy-14539", "astropy__astropy-14598", "astropy__astropy-7606", "astropy__astropy-7671", "astropy__astropy-8707", "astropy__astropy-8872", "django__django-10973", "django__django-11087", "django__django-11095", "django__django-11141", "django__django-11149", "django__django-11206", "django__django-11211", "django__django-11265", "django__django-11276", "django__django-11292", "django__django-11333", "django__django-11477", "django__django-11532", "django__django-11551", "django__django-11728", "django__django-11734", "django__django-11740", "django__django-11749", "django__django-11790", "django__django-11815", "django__django-11848", "django__django-11999", "django__django-12039", "django__django-12050", "django__django-12143", "django__django-12155", "django__django-12262", "django__django-12273", "django__django-12406", "django__django-12663", "django__django-12713", "django__django-12754", "django__django-12774", "django__django-12858", "django__django-12965", "django__django-13012", "django__django-13028", "django__django-13033", "django__django-13121", "django__django-13158", "django__django-13195", "django__django-13279", "django__django-13315", "django__django-13343", "django__django-13346", "django__django-13401", "django__django-13513", "django__django-13516", "django__django-13568", "django__django-13569", "django__django-13590", "django__django-13658", "django__django-13670", "django__django-13786", "django__django-13809", "django__django-13810", "django__django-13820", "django__django-13925", "django__django-13964", "django__django-14017", "django__django-14034", "django__django-14053", "django__django-14122", "django__django-14140", "django__django-14155", "django__django-14170", "django__django-14238", "django__django-14311", "django__django-14349", "django__django-14351", "django__django-14434", "django__django-14500", "django__django-14559", "django__django-14672", "django__django-14725", "django__django-14771", "django__django-15022", "django__django-15037", "django__django-15098", "django__django-15103", "django__django-15161", "django__django-15252", "django__django-15278", "django__django-15280", "django__django-15375", "django__django-15382", "django__django-15525", "django__django-15554", "django__django-15561", "django__django-15563", "django__django-15695", "django__django-15731", "django__django-15732", "django__django-15814", "django__django-15916", "django__django-15973", "django__django-16032", "django__django-16082", "django__django-16136", "django__django-16256", "django__django-16315", "django__django-16454", "django__django-16485", "django__django-16493", "django__django-16502", "django__django-16527", "django__django-16612", "django__django-16661", "django__django-16662", "django__django-16667", "django__django-16819", "django__django-16877", "django__django-16901", "django__django-16938", "django__django-16950", "django__django-17084", "django__django-7530", "matplotlib__matplotlib-14623", "matplotlib__matplotlib-20488", "matplotlib__matplotlib-20826", "matplotlib__matplotlib-21568", "matplotlib__matplotlib-22865", "matplotlib__matplotlib-22871", "matplotlib__matplotlib-23299", "matplotlib__matplotlib-23314", "matplotlib__matplotlib-23412", "matplotlib__matplotlib-24026", "matplotlib__matplotlib-24627", "matplotlib__matplotlib-24637", "matplotlib__matplotlib-24870", "matplotlib__matplotlib-24970", "matplotlib__matplotlib-25775", "matplotlib__matplotlib-25960", "matplotlib__matplotlib-26291", "matplotlib__matplotlib-26342", "matplotlib__matplotlib-26466", "mwaskom__seaborn-3069", "mwaskom__seaborn-3187", "psf__requests-2931", "psf__requests-6028", "pydata__xarray-2905", "pydata__xarray-3095", "pydata__xarray-3151", "pydata__xarray-3305", "pydata__xarray-3677", "pydata__xarray-4687", "pydata__xarray-4695", "pydata__xarray-4966", "pydata__xarray-6599", "pydata__xarray-6721", "pydata__xarray-6744", "pydata__xarray-6938", "pydata__xarray-7229", "pydata__xarray-7233", "pydata__xarray-7393", "pylint-dev__pylint-4604", "pylint-dev__pylint-4661", "pylint-dev__pylint-6386", "pylint-dev__pylint-6528", "pylint-dev__pylint-7080", "pytest-dev__pytest-10051", "pytest-dev__pytest-5631", "pytest-dev__pytest-5840", "pytest-dev__pytest-7236", "pytest-dev__pytest-7324", "pytest-dev__pytest-7490", "pytest-dev__pytest-7571", "pytest-dev__pytest-8399", "scikit-learn__scikit-learn-10297", "scikit-learn__scikit-learn-10844", "scikit-learn__scikit-learn-10908", "scikit-learn__scikit-learn-11578", "scikit-learn__scikit-learn-12682", "scikit-learn__scikit-learn-12973", "scikit-learn__scikit-learn-13124", "scikit-learn__scikit-learn-14053", "scikit-learn__scikit-learn-14087", "scikit-learn__scikit-learn-14629", "scikit-learn__scikit-learn-14710", "scikit-learn__scikit-learn-14894", "scikit-learn__scikit-learn-15100", "scikit-learn__scikit-learn-25747", "scikit-learn__scikit-learn-25931", "scikit-learn__scikit-learn-26194", "scikit-learn__scikit-learn-26323", "scikit-learn__scikit-learn-9288", "sphinx-doc__sphinx-10466", "sphinx-doc__sphinx-10614", "sphinx-doc__sphinx-10673", "sphinx-doc__sphinx-11445", "sphinx-doc__sphinx-7748", "sphinx-doc__sphinx-7757", "sphinx-doc__sphinx-7985", "sphinx-doc__sphinx-8035", "sphinx-doc__sphinx-8056", "sphinx-doc__sphinx-8120", "sphinx-doc__sphinx-8265", "sphinx-doc__sphinx-8551", "sphinx-doc__sphinx-8593", "sphinx-doc__sphinx-8638", "sphinx-doc__sphinx-9602", "sphinx-doc__sphinx-9658", "sphinx-doc__sphinx-9673", "sympy__sympy-11618", "sympy__sympy-12419", "sympy__sympy-13031", "sympy__sympy-13091", "sympy__sympy-13551", "sympy__sympy-13615", "sympy__sympy-13647", "sympy__sympy-13757", "sympy__sympy-13798", "sympy__sympy-13877", "sympy__sympy-13974", "sympy__sympy-14531", "sympy__sympy-14976", "sympy__sympy-15349", "sympy__sympy-15599", "sympy__sympy-15976", "sympy__sympy-16792", "sympy__sympy-17318", "sympy__sympy-18211", "sympy__sympy-18698", "sympy__sympy-19040", "sympy__sympy-19346", "sympy__sympy-19783", "sympy__sympy-20154", "sympy__sympy-20428", "sympy__sympy-20438", "sympy__sympy-20590", "sympy__sympy-20801", "sympy__sympy-21379", "sympy__sympy-21596", "sympy__sympy-21612", "sympy__sympy-21930", "sympy__sympy-22080", "sympy__sympy-22456", "sympy__sympy-22914", "sympy__sympy-23262", "sympy__sympy-23413", "sympy__sympy-23824", "sympy__sympy-23950", "sympy__sympy-24066", "sympy__sympy-24213", "sympy__sympy-24443", "sympy__sympy-24661" ], "hard_ids": [ "astropy__astropy-13398", "astropy__astropy-13579", "astropy__astropy-14369", "django__django-10554", "django__django-11138", "django__django-11400", "django__django-11885", "django__django-12325", "django__django-12708", "django__django-13128", "django__django-13212", "django__django-13344", "django__django-13449", "django__django-13837", "django__django-14007", "django__django-14011", "django__django-14631", "django__django-15128", "django__django-15268", "django__django-15503", "django__django-15629", "django__django-15957", "django__django-16263", "django__django-16560", "django__django-16631", "pydata__xarray-3993", "pydata__xarray-6992", "pylint-dev__pylint-4551", "pylint-dev__pylint-8898", "pytest-dev__pytest-10356", "pytest-dev__pytest-5787", "pytest-dev__pytest-6197", "scikit-learn__scikit-learn-25102", "sphinx-doc__sphinx-11510", "sphinx-doc__sphinx-7590", "sphinx-doc__sphinx-8548", "sphinx-doc__sphinx-9229", "sphinx-doc__sphinx-9461", "sympy__sympy-12489", "sympy__sympy-13852", "sympy__sympy-13878", "sympy__sympy-14248", "sympy__sympy-16597", "sympy__sympy-17630", "sympy__sympy-18199" ] } } ] }, { "name": "Java", "data": [ { "name": "Verified", "results": [ { "name": "CodeArts Agent + CodeArts-MiniMax-M2.5", "site": "https://www.huaweicloud.com/intl/zh-cn/product/codearts/ai.html", "oss": false, "verified": false, "orgIcon": "https://res-static.hc-cdn.cn/cloudbu-site/china/zh-cn/pangudoer/logo_1@2x.png", "resolved": 56, "resolvedEasy": 23, "resolvedMedium": 30, "resolvedHard": 3, "resolvedRate": 0.4375, "resolvedEasyRate": 0.8518518518518519, "resolvedMediumRate": 0.46153846153846156, "resolvedHardRate": 0.08333333333333333, "date": "2026-03-27", "path": "java/verified/20260327_CodeArts-Agent_CodeArts-MiniMax-M2.5", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "InfCode + GPT5.2", "site": "https://www.tokfinity.com/infcode", "oss": true, "verified": false, "orgIcon": "https://raw.githubusercontent.com/Tokfinity/InfCode/main/figures/Tokfinity.png", "resolved": 50, "resolvedEasy": 17, "resolvedMedium": 27, "resolvedHard": 6, "resolvedRate": 0.390625, "resolvedEasyRate": 0.6296296296296297, "resolvedMediumRate": 0.4153846153846154, "resolvedHardRate": 0.16666666666666666, "date": "2026-01-31", "path": "java/verified/20260131_InfCode_GPT-5.2", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "iSWE-Agent", "site": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20251201_iSWE_Agent/README.md", "oss": false, "verified": false, "orgIcon": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQPBss61ZSzmBM75JqdYCVqsfqJ4JijX0mt9g&s", "resolved": 43, "resolvedEasy": 16, "resolvedMedium": 26, "resolvedHard": 1, "resolvedRate": 0.3359375, "resolvedEasyRate": 0.5925925925925926, "resolvedMediumRate": 0.4, "resolvedHardRate": 0.027777777777777776, "date": "2025-12-01", "path": "java/verified/20251201_iSWE_Agent", "hasLogs": false, "hasTrajs": false, "hasReadme": true }, { "name": "iSWE-OpenModels", "site": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20251027_iSWE-OpenModels", "oss": true, "verified": false, "orgIcon": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQPBss61ZSzmBM75JqdYCVqsfqJ4JijX0mt9g&s", "resolved": 40, "resolvedEasy": 12, "resolvedMedium": 28, "resolvedHard": 0, "resolvedRate": 0.3125, "resolvedEasyRate": 0.4444444444444444, "resolvedMediumRate": 0.4307692307692308, "resolvedHardRate": 0, "date": "2025-10-27", "path": "java/verified/20251027_iSWE-OpenModels", "hasLogs": false, "hasTrajs": false, "hasReadme": true }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 37, "resolvedEasy": 16, "resolvedMedium": 21, "resolvedHard": 0, "resolvedRate": 0.2890625, "resolvedEasyRate": 0.5925925925925926, "resolvedMediumRate": 0.3230769230769231, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + CodeArts-MiniMax-M2.5", "site": "https://www.huaweicloud.com/intl/zh-cn/product/codearts/ai.html", "oss": false, "verified": false, "orgIcon": "https://res-static.hc-cdn.cn/cloudbu-site/china/zh-cn/pangudoer/logo_1@2x.png", "resolved": 35, "resolvedEasy": 16, "resolvedMedium": 19, "resolvedHard": 0, "resolvedRate": 0.2734375, "resolvedEasyRate": 0.5925925925925926, "resolvedMediumRate": 0.2923076923076923, "resolvedHardRate": 0, "date": "2026-03-27", "path": "java/verified/20260327_MSWE-Agent_CodeArts-MiniMax-M2.5", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 30, "resolvedEasy": 12, "resolvedMedium": 18, "resolvedHard": 0, "resolvedRate": 0.234375, "resolvedEasyRate": 0.4444444444444444, "resolvedMediumRate": 0.27692307692307694, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 29, "resolvedEasy": 14, "resolvedMedium": 15, "resolvedHard": 0, "resolvedRate": 0.2265625, "resolvedEasyRate": 0.5185185185185185, "resolvedMediumRate": 0.23076923076923078, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 28, "resolvedEasy": 13, "resolvedMedium": 15, "resolvedHard": 0, "resolvedRate": 0.21875, "resolvedEasyRate": 0.48148148148148145, "resolvedMediumRate": 0.23076923076923078, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 28, "resolvedEasy": 13, "resolvedMedium": 15, "resolvedHard": 0, "resolvedRate": 0.21875, "resolvedEasyRate": 0.48148148148148145, "resolvedMediumRate": 0.23076923076923078, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 28, "resolvedEasy": 14, "resolvedMedium": 14, "resolvedHard": 0, "resolvedRate": 0.21875, "resolvedEasyRate": 0.5185185185185185, "resolvedMediumRate": 0.2153846153846154, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 27, "resolvedEasy": 11, "resolvedMedium": 16, "resolvedHard": 0, "resolvedRate": 0.2109375, "resolvedEasyRate": 0.4074074074074074, "resolvedMediumRate": 0.24615384615384617, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 26, "resolvedEasy": 13, "resolvedMedium": 13, "resolvedHard": 0, "resolvedRate": 0.203125, "resolvedEasyRate": 0.48148148148148145, "resolvedMediumRate": 0.2, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 21, "resolvedEasy": 9, "resolvedMedium": 12, "resolvedHard": 0, "resolvedRate": 0.1640625, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.18461538461538463, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 19, "resolvedEasy": 10, "resolvedMedium": 9, "resolvedHard": 0, "resolvedRate": 0.1484375, "resolvedEasyRate": 0.37037037037037035, "resolvedMediumRate": 0.13846153846153847, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 19, "resolvedEasy": 7, "resolvedMedium": 12, "resolvedHard": 0, "resolvedRate": 0.1484375, "resolvedEasyRate": 0.25925925925925924, "resolvedMediumRate": 0.18461538461538463, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 19, "resolvedEasy": 11, "resolvedMedium": 8, "resolvedHard": 0, "resolvedRate": 0.1484375, "resolvedEasyRate": 0.4074074074074074, "resolvedMediumRate": 0.12307692307692308, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 18, "resolvedEasy": 9, "resolvedMedium": 9, "resolvedHard": 0, "resolvedRate": 0.140625, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.13846153846153847, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 17, "resolvedEasy": 9, "resolvedMedium": 8, "resolvedHard": 0, "resolvedRate": 0.1328125, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.12307692307692308, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 16, "resolvedEasy": 6, "resolvedMedium": 10, "resolvedHard": 0, "resolvedRate": 0.125, "resolvedEasyRate": 0.2222222222222222, "resolvedMediumRate": 0.15384615384615385, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 16, "resolvedEasy": 6, "resolvedMedium": 10, "resolvedHard": 0, "resolvedRate": 0.125, "resolvedEasyRate": 0.2222222222222222, "resolvedMediumRate": 0.15384615384615385, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 9, "resolvedMedium": 6, "resolvedHard": 0, "resolvedRate": 0.1171875, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.09230769230769231, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 15, "resolvedEasy": 6, "resolvedMedium": 9, "resolvedHard": 0, "resolvedRate": 0.1171875, "resolvedEasyRate": 0.2222222222222222, "resolvedMediumRate": 0.13846153846153847, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 6, "resolvedMedium": 9, "resolvedHard": 0, "resolvedRate": 0.1171875, "resolvedEasyRate": 0.2222222222222222, "resolvedMediumRate": 0.13846153846153847, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 14, "resolvedEasy": 9, "resolvedMedium": 5, "resolvedHard": 0, "resolvedRate": 0.109375, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.07692307692307693, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 14, "resolvedEasy": 9, "resolvedMedium": 5, "resolvedHard": 0, "resolvedRate": 0.109375, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.07692307692307693, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 13, "resolvedEasy": 6, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.1015625, "resolvedEasyRate": 0.2222222222222222, "resolvedMediumRate": 0.1076923076923077, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 12, "resolvedEasy": 4, "resolvedMedium": 8, "resolvedHard": 0, "resolvedRate": 0.09375, "resolvedEasyRate": 0.14814814814814814, "resolvedMediumRate": 0.12307692307692308, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 5, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.09375, "resolvedEasyRate": 0.18518518518518517, "resolvedMediumRate": 0.1076923076923077, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 8, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.09375, "resolvedEasyRate": 0.2962962962962963, "resolvedMediumRate": 0.06153846153846154, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 11, "resolvedEasy": 4, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.0859375, "resolvedEasyRate": 0.14814814814814814, "resolvedMediumRate": 0.1076923076923077, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 3, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.0703125, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.07692307692307693, "resolvedHardRate": 0.027777777777777776, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 9, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.0703125, "resolvedEasyRate": 0.18518518518518517, "resolvedMediumRate": 0.06153846153846154, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.0625, "resolvedEasyRate": 0.18518518518518517, "resolvedMediumRate": 0.046153846153846156, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.0625, "resolvedEasyRate": 0.18518518518518517, "resolvedMediumRate": 0.046153846153846156, "resolvedHardRate": 0, "date": "2025-04-26", "path": "java/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.0546875, "resolvedEasyRate": 0.14814814814814814, "resolvedMediumRate": 0.046153846153846156, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.0546875, "resolvedEasyRate": 0.14814814814814814, "resolvedMediumRate": 0.046153846153846156, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.0390625, "resolvedEasyRate": 0.07407407407407407, "resolvedMediumRate": 0.046153846153846156, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.07407407407407407, "resolvedMediumRate": 0.03076923076923077, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0234375, "resolvedEasyRate": 0.07407407407407407, "resolvedMediumRate": 0.015384615384615385, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 1, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0, "resolvedMediumRate": 0.015384615384615385, "resolvedHardRate": 0, "date": "2025-03-29", "path": "java/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "elastic__logstash-13825", "elastic__logstash-14571", "fasterxml__jackson-databind-4615", "googlecontainertools__jib-4035", "elastic__logstash-13930", "fasterxml__jackson-core-922", "elastic__logstash-16681", "elastic__logstash-16482", "fasterxml__jackson-core-183", "fasterxml__jackson-databind-3560", "apache__dubbo-7041", "alibaba__fastjson2-2097", "fasterxml__jackson-core-566", "elastic__logstash-14027", "elastic__logstash-17021", "fasterxml__jackson-core-729", "apache__dubbo-10638", "google__gson-1787", "googlecontainertools__jib-2542", "fasterxml__jackson-databind-4365", "fasterxml__jackson-core-1182", "elastic__logstash-16079", "mockito__mockito-3133", "alibaba__fastjson2-82", "elastic__logstash-15680", "elastic__logstash-17020", "elastic__logstash-14045", "fasterxml__jackson-databind-4186", "elastic__logstash-15925", "elastic__logstash-13880", "elastic__logstash-14970", "fasterxml__jackson-core-980", "googlecontainertools__jib-4144", "fasterxml__jackson-databind-4050", "fasterxml__jackson-databind-3509", "fasterxml__jackson-databind-4304", "google__gson-1703", "fasterxml__jackson-databind-3626", "fasterxml__jackson-core-1204", "fasterxml__jackson-databind-3860", "fasterxml__jackson-databind-2036", "fasterxml__jackson-databind-4486", "fasterxml__jackson-databind-4048", "mockito__mockito-3129", "fasterxml__jackson-databind-3851", "fasterxml__jackson-core-964", "fasterxml__jackson-dataformat-xml-531", "fasterxml__jackson-databind-4311", "fasterxml__jackson-databind-4360", "google__gson-1391", "elastic__logstash-16579", "fasterxml__jackson-core-1016", "fasterxml__jackson-databind-4230", "fasterxml__jackson-core-1208", "fasterxml__jackson-dataformat-xml-644", "fasterxml__jackson-databind-3621", "fasterxml__jackson-databind-4468", "mockito__mockito-3424", "google__gson-1093", "fasterxml__jackson-databind-3666", "fasterxml__jackson-databind-4189", "fasterxml__jackson-databind-4325", "fasterxml__jackson-databind-4469", "fasterxml__jackson-databind-4159", "apache__dubbo-11781", "fasterxml__jackson-databind-3716", "elastic__logstash-15008", "fasterxml__jackson-databind-4015", "elastic__logstash-15928", "elastic__logstash-14878", "elastic__logstash-15964", "elastic__logstash-14898", "fasterxml__jackson-databind-4228", "fasterxml__jackson-databind-4487", "elastic__logstash-14897", "fasterxml__jackson-core-1142", "elastic__logstash-16094", "fasterxml__jackson-core-174", "mockito__mockito-3220", "elastic__logstash-17019", "alibaba__fastjson2-2775", "elastic__logstash-16569", "alibaba__fastjson2-2285", "googlecontainertools__jib-2536", "mockito__mockito-3173", "fasterxml__jackson-databind-4013", "fasterxml__jackson-databind-4131", "elastic__logstash-15241", "alibaba__fastjson2-2559", "fasterxml__jackson-databind-4087", "fasterxml__jackson-databind-4132", "elastic__logstash-15969", "fasterxml__jackson-databind-3701", "fasterxml__jackson-core-891", "alibaba__fastjson2-1245", "google__gson-1555", "elastic__logstash-16968", "fasterxml__jackson-databind-3371", "googlecontainertools__jib-2688", "fasterxml__jackson-databind-1923", "fasterxml__jackson-dataformat-xml-544", "mockito__mockito-3167", "elastic__logstash-13902", "elastic__logstash-15233", "fasterxml__jackson-core-1172", "elastic__logstash-14981", "fasterxml__jackson-databind-3625", "fasterxml__jackson-databind-4072", "fasterxml__jackson-databind-4257", "elastic__logstash-14000", "fasterxml__jackson-dataformat-xml-638", "fasterxml__jackson-databind-4320", "fasterxml__jackson-databind-4338", "fasterxml__jackson-core-1263", "elastic__logstash-13931", "elastic__logstash-13914", "fasterxml__jackson-core-370", "fasterxml__jackson-databind-4219", "fasterxml__jackson-dataformat-xml-590", "fasterxml__jackson-core-1053", "elastic__logstash-16195", "elastic__logstash-15697", "fasterxml__jackson-core-1309", "elastic__logstash-15000", "elastic__logstash-14058", "elastic__logstash-13997", "fasterxml__jackson-databind-4641", "fasterxml__jackson-databind-4426" ], "easy_ids": [ "fasterxml__jackson-databind-4050", "fasterxml__jackson-databind-4304", "fasterxml__jackson-databind-4486", "mockito__mockito-3129", "fasterxml__jackson-databind-4257", "fasterxml__jackson-databind-4072", "fasterxml__jackson-databind-3625", "fasterxml__jackson-databind-4487", "fasterxml__jackson-databind-4360", "fasterxml__jackson-core-1142", "fasterxml__jackson-databind-4311", "fasterxml__jackson-core-183", "fasterxml__jackson-databind-4320", "fasterxml__jackson-databind-4338", "fasterxml__jackson-databind-4468", "fasterxml__jackson-databind-4189", "fasterxml__jackson-databind-4325", "fasterxml__jackson-databind-4469", "fasterxml__jackson-databind-4219", "elastic__logstash-16079", "alibaba__fastjson2-82", "fasterxml__jackson-databind-4186", "fasterxml__jackson-databind-3701", "alibaba__fastjson2-1245", "elastic__logstash-15964", "fasterxml__jackson-databind-4426", "fasterxml__jackson-databind-1923" ], "medium_ids": [ "google__gson-1703", "fasterxml__jackson-databind-3626", "fasterxml__jackson-core-1204", "fasterxml__jackson-databind-3860", "fasterxml__jackson-databind-4615", "elastic__logstash-15233", "fasterxml__jackson-core-1172", "fasterxml__jackson-databind-4048", "googlecontainertools__jib-4035", "fasterxml__jackson-databind-3851", "elastic__logstash-14981", "fasterxml__jackson-core-964", "fasterxml__jackson-core-922", "google__gson-1391", "elastic__logstash-16681", "fasterxml__jackson-core-1016", "apache__dubbo-7041", "fasterxml__jackson-core-174", "fasterxml__jackson-databind-3560", "fasterxml__jackson-databind-4230", "fasterxml__jackson-dataformat-xml-638", "mockito__mockito-3167", "alibaba__fastjson2-2097", "fasterxml__jackson-core-1208", "fasterxml__jackson-core-1263", "fasterxml__jackson-dataformat-xml-644", "fasterxml__jackson-core-566", "fasterxml__jackson-databind-3621", "googlecontainertools__jib-4144", "google__gson-1093", "apache__dubbo-10638", "fasterxml__jackson-databind-3666", "google__gson-1787", "googlecontainertools__jib-2542", "fasterxml__jackson-core-370", "elastic__logstash-13914", "alibaba__fastjson2-2775", "alibaba__fastjson2-2285", "fasterxml__jackson-databind-4159", "apache__dubbo-11781", "fasterxml__jackson-dataformat-xml-590", "fasterxml__jackson-databind-4365", "googlecontainertools__jib-2536", "mockito__mockito-3173", "fasterxml__jackson-core-1182", "fasterxml__jackson-databind-4131", "elastic__logstash-15241", "elastic__logstash-15680", "alibaba__fastjson2-2559", "fasterxml__jackson-databind-4015", "elastic__logstash-15928", "fasterxml__jackson-databind-4087", "fasterxml__jackson-databind-4132", "elastic__logstash-15925", "fasterxml__jackson-core-1053", "fasterxml__jackson-core-891", "google__gson-1555", "fasterxml__jackson-core-1309", "elastic__logstash-15697", "elastic__logstash-13880", "elastic__logstash-14970", "fasterxml__jackson-databind-4641", "googlecontainertools__jib-2688", "fasterxml__jackson-dataformat-xml-544", "fasterxml__jackson-core-980" ], "hard_ids": [ "elastic__logstash-13825", "fasterxml__jackson-databind-3509", "elastic__logstash-14571", "elastic__logstash-13902", "fasterxml__jackson-databind-2036", "elastic__logstash-13930", "fasterxml__jackson-dataformat-xml-531", "elastic__logstash-14897", "elastic__logstash-16579", "elastic__logstash-14000", "elastic__logstash-16094", "elastic__logstash-16482", "mockito__mockito-3220", "elastic__logstash-17019", "elastic__logstash-14027", "elastic__logstash-13931", "mockito__mockito-3424", "elastic__logstash-17021", "fasterxml__jackson-core-729", "elastic__logstash-16569", "fasterxml__jackson-databind-3716", "fasterxml__jackson-databind-4013", "elastic__logstash-15008", "mockito__mockito-3133", "elastic__logstash-14045", "elastic__logstash-17020", "elastic__logstash-15969", "elastic__logstash-14878", "elastic__logstash-16195", "elastic__logstash-15000", "elastic__logstash-16968", "fasterxml__jackson-databind-3371", "elastic__logstash-14058", "elastic__logstash-14898", "elastic__logstash-13997", "fasterxml__jackson-databind-4228" ] } } ] }, { "name": "TypeScript", "data": [ { "name": "Verified", "results": [ { "name": "RepoRepair + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/ZhongQiangDev/RepoRepair", "oss": true, "verified": true, "resolved": 61, "resolvedEasy": 30, "resolvedMedium": 23, "resolvedHard": 8, "resolvedRate": 0.27232142857142855, "resolvedEasyRate": 0.4166666666666667, "resolvedMediumRate": 0.26136363636363635, "resolvedHardRate": 0.125, "date": "2025-09-10", "path": "typescript/verified/20250910_RepoRepair_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 50, "resolvedEasy": 24, "resolvedMedium": 20, "resolvedHard": 6, "resolvedRate": 0.22321428571428573, "resolvedEasyRate": 0.3333333333333333, "resolvedMediumRate": 0.22727272727272727, "resolvedHardRate": 0.09375, "date": "2025-04-26", "path": "typescript/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 26, "resolvedEasy": 13, "resolvedMedium": 12, "resolvedHard": 1, "resolvedRate": 0.11607142857142858, "resolvedEasyRate": 0.18055555555555555, "resolvedMediumRate": 0.13636363636363635, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 26, "resolvedEasy": 13, "resolvedMedium": 12, "resolvedHard": 1, "resolvedRate": 0.11607142857142858, "resolvedEasyRate": 0.18055555555555555, "resolvedMediumRate": 0.13636363636363635, "resolvedHardRate": 0.015625, "date": "2025-04-26", "path": "typescript/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 25, "resolvedEasy": 15, "resolvedMedium": 8, "resolvedHard": 2, "resolvedRate": 0.11160714285714286, "resolvedEasyRate": 0.20833333333333334, "resolvedMediumRate": 0.09090909090909091, "resolvedHardRate": 0.03125, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 21, "resolvedEasy": 14, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.09375, "resolvedEasyRate": 0.19444444444444445, "resolvedMediumRate": 0.07954545454545454, "resolvedHardRate": 0, "date": "2025-04-26", "path": "typescript/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 20, "resolvedEasy": 14, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.08928571428571429, "resolvedEasyRate": 0.19444444444444445, "resolvedMediumRate": 0.056818181818181816, "resolvedHardRate": 0.015625, "date": "2025-04-26", "path": "typescript/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 18, "resolvedEasy": 11, "resolvedMedium": 5, "resolvedHard": 2, "resolvedRate": 0.08035714285714286, "resolvedEasyRate": 0.1527777777777778, "resolvedMediumRate": 0.056818181818181816, "resolvedHardRate": 0.03125, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 18, "resolvedEasy": 11, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.08035714285714286, "resolvedEasyRate": 0.1527777777777778, "resolvedMediumRate": 0.06818181818181818, "resolvedHardRate": 0.015625, "date": "2025-04-26", "path": "typescript/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 17, "resolvedEasy": 12, "resolvedMedium": 3, "resolvedHard": 2, "resolvedRate": 0.07589285714285714, "resolvedEasyRate": 0.16666666666666666, "resolvedMediumRate": 0.03409090909090909, "resolvedHardRate": 0.03125, "date": "2025-04-26", "path": "typescript/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 16, "resolvedEasy": 8, "resolvedMedium": 6, "resolvedHard": 2, "resolvedRate": 0.07142857142857142, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.06818181818181818, "resolvedHardRate": 0.03125, "date": "2025-04-26", "path": "typescript/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 15, "resolvedEasy": 8, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.06696428571428571, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.06818181818181818, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 14, "resolvedEasy": 8, "resolvedMedium": 6, "resolvedHard": 0, "resolvedRate": 0.0625, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.06818181818181818, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 13, "resolvedEasy": 7, "resolvedMedium": 6, "resolvedHard": 0, "resolvedRate": 0.05803571428571429, "resolvedEasyRate": 0.09722222222222222, "resolvedMediumRate": 0.06818181818181818, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 13, "resolvedEasy": 8, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.05803571428571429, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.045454545454545456, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 8, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.05357142857142857, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.045454545454545456, "resolvedHardRate": 0, "date": "2025-04-26", "path": "typescript/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 11, "resolvedEasy": 8, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.049107142857142856, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.03409090909090909, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 11, "resolvedEasy": 8, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.049107142857142856, "resolvedEasyRate": 0.1111111111111111, "resolvedMediumRate": 0.022727272727272728, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 10, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.044642857142857144, "resolvedEasyRate": 0.06944444444444445, "resolvedMediumRate": 0.045454545454545456, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 6, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.044642857142857144, "resolvedEasyRate": 0.08333333333333333, "resolvedMediumRate": 0.045454545454545456, "resolvedHardRate": 0, "date": "2025-04-26", "path": "typescript/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 3, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.04017857142857143, "resolvedEasyRate": 0.041666666666666664, "resolvedMediumRate": 0.045454545454545456, "resolvedHardRate": 0.03125, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 8, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.03571428571428571, "resolvedEasyRate": 0.05555555555555555, "resolvedMediumRate": 0.03409090909090909, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 6, "resolvedEasy": 4, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.026785714285714284, "resolvedEasyRate": 0.05555555555555555, "resolvedMediumRate": 0.022727272727272728, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 1, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.022321428571428572, "resolvedEasyRate": 0.013888888888888888, "resolvedMediumRate": 0.03409090909090909, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.022321428571428572, "resolvedEasyRate": 0.041666666666666664, "resolvedMediumRate": 0.011363636363636364, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.022321428571428572, "resolvedEasyRate": 0.027777777777777776, "resolvedMediumRate": 0.022727272727272728, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.017857142857142856, "resolvedEasyRate": 0.027777777777777776, "resolvedMediumRate": 0.022727272727272728, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 1, "resolvedRate": 0.013392857142857142, "resolvedEasyRate": 0.027777777777777776, "resolvedMediumRate": 0, "resolvedHardRate": 0.015625, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 1, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.004464285714285714, "resolvedEasyRate": 0, "resolvedMediumRate": 0.011363636363636364, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.004464285714285714, "resolvedEasyRate": 0.013888888888888888, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.004464285714285714, "resolvedEasyRate": 0.013888888888888888, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.004464285714285714, "resolvedEasyRate": 0.013888888888888888, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.004464285714285714, "resolvedEasyRate": 0.013888888888888888, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "typescript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "darkreader__darkreader-7241", "darkreader__darkreader-6747", "vuejs__core-11899", "vuejs__core-11854", "vuejs__core-11813", "vuejs__core-11761", "vuejs__core-11694", "vuejs__core-11680", "vuejs__core-11625", "vuejs__core-11517", "vuejs__core-11515", "vuejs__core-11502", "vuejs__core-11491", "vuejs__core-11338", "vuejs__core-11201", "vuejs__core-11165", "vuejs__core-11026", "vuejs__core-10989", "vuejs__core-10939", "vuejs__core-10891", "vuejs__core-10874", "vuejs__core-10869", "vuejs__core-10522", "vuejs__core-10416", "vuejs__core-10325", "vuejs__core-10289", "vuejs__core-10250", "vuejs__core-10218", "vuejs__core-10161", "vuejs__core-10141", "vuejs__core-10101", "vuejs__core-10027", "vuejs__core-10004", "vuejs__core-9751", "vuejs__core-9572", "vuejs__core-9532", "vuejs__core-9507", "vuejs__core-9492", "vuejs__core-9442", "vuejs__core-9213", "vuejs__core-8911", "vuejs__core-8824", "vuejs__core-8785", "vuejs__core-8538", "vuejs__core-8535", "vuejs__core-8511", "vuejs__core-8470", "vuejs__core-8402", "vuejs__core-8381", "vuejs__core-8365", "mui__material-ui-39962", "mui__material-ui-39775", "mui__material-ui-39688", "mui__material-ui-39679", "mui__material-ui-39536", "mui__material-ui-39465", "mui__material-ui-39457", "mui__material-ui-39353", "mui__material-ui-39196", "mui__material-ui-39108", "mui__material-ui-39071", "mui__material-ui-38916", "mui__material-ui-38801", "mui__material-ui-38788", "mui__material-ui-38544", "mui__material-ui-38247", "mui__material-ui-38188", "mui__material-ui-38182", "mui__material-ui-38169", "mui__material-ui-38167", "mui__material-ui-38100", "mui__material-ui-37855", "mui__material-ui-37850", "mui__material-ui-37716", "mui__material-ui-37692", "mui__material-ui-37667", "mui__material-ui-37465", "mui__material-ui-37141", "mui__material-ui-37118", "mui__material-ui-37108", "mui__material-ui-37070", "mui__material-ui-36971", "mui__material-ui-36853", "mui__material-ui-36839", "mui__material-ui-36659", "mui__material-ui-36426", "mui__material-ui-36403", "mui__material-ui-36401", "mui__material-ui-36400", "mui__material-ui-36399", "mui__material-ui-36353", "mui__material-ui-36116", "mui__material-ui-36088", "mui__material-ui-36056", "mui__material-ui-36024", "mui__material-ui-35994", "mui__material-ui-35963", "mui__material-ui-35946", "mui__material-ui-35872", "mui__material-ui-35838", "mui__material-ui-35804", "mui__material-ui-35477", "mui__material-ui-35410", "mui__material-ui-35364", "mui__material-ui-35360", "mui__material-ui-35178", "mui__material-ui-35088", "mui__material-ui-35086", "mui__material-ui-35047", "mui__material-ui-34966", "mui__material-ui-34829", "mui__material-ui-34610", "mui__material-ui-34548", "mui__material-ui-34478", "mui__material-ui-34455", "mui__material-ui-34445", "mui__material-ui-34437", "mui__material-ui-34401", "mui__material-ui-34380", "mui__material-ui-34337", "mui__material-ui-34258", "mui__material-ui-34247", "mui__material-ui-34245", "mui__material-ui-34229", "mui__material-ui-34216", "mui__material-ui-34207", "mui__material-ui-34159", "mui__material-ui-34158", "mui__material-ui-34131", "mui__material-ui-33957", "mui__material-ui-33934", "mui__material-ui-33880", "mui__material-ui-33873", "mui__material-ui-33863", "mui__material-ui-33820", "mui__material-ui-33812", "mui__material-ui-33801", "mui__material-ui-33797", "mui__material-ui-33777", "mui__material-ui-33692", "mui__material-ui-33691", "mui__material-ui-33582", "mui__material-ui-33415", "mui__material-ui-33312", "mui__material-ui-33283", "mui__material-ui-33145", "mui__material-ui-33040", "mui__material-ui-32987", "mui__material-ui-32962", "mui__material-ui-32746", "mui__material-ui-32713", "mui__material-ui-32626", "mui__material-ui-32401", "mui__material-ui-32185", "mui__material-ui-32182", "mui__material-ui-31998", "mui__material-ui-31313", "mui__material-ui-31172", "mui__material-ui-30899", "mui__material-ui-30788", "mui__material-ui-30561", "mui__material-ui-30524", "mui__material-ui-30515", "mui__material-ui-30374", "mui__material-ui-30345", "mui__material-ui-30333", "mui__material-ui-30126", "mui__material-ui-30118", "mui__material-ui-29988", "mui__material-ui-29954", "mui__material-ui-29902", "mui__material-ui-29901", "mui__material-ui-29898", "mui__material-ui-29880", "mui__material-ui-29824", "mui__material-ui-29756", "mui__material-ui-29613", "mui__material-ui-29579", "mui__material-ui-29488", "mui__material-ui-29317", "mui__material-ui-29297", "mui__material-ui-28645", "mui__material-ui-26480", "mui__material-ui-26470", "mui__material-ui-26460", "mui__material-ui-26413", "mui__material-ui-26323", "mui__material-ui-26186", "mui__material-ui-26173", "mui__material-ui-26170", "mui__material-ui-26098", "mui__material-ui-26061", "mui__material-ui-26026", "mui__material-ui-25974", "mui__material-ui-25874", "mui__material-ui-25825", "mui__material-ui-25815", "mui__material-ui-25784", "mui__material-ui-25637", "mui__material-ui-25431", "mui__material-ui-25428", "mui__material-ui-25390", "mui__material-ui-25276", "mui__material-ui-25259", "mui__material-ui-25245", "mui__material-ui-25149", "mui__material-ui-25105", "mui__material-ui-24968", "mui__material-ui-24945", "mui__material-ui-24857", "mui__material-ui-24794", "mui__material-ui-24782", "mui__material-ui-24621", "mui__material-ui-24290", "mui__material-ui-24213", "mui__material-ui-24205", "mui__material-ui-24202", "mui__material-ui-24010", "mui__material-ui-23884", "mui__material-ui-23806", "mui__material-ui-23786", "mui__material-ui-23778", "mui__material-ui-23487", "mui__material-ui-23073" ], "easy_ids": [ "darkreader__darkreader-7241", "darkreader__darkreader-6747", "vuejs__core-11813", "vuejs__core-11694", "vuejs__core-11680", "vuejs__core-11625", "vuejs__core-11515", "vuejs__core-11502", "vuejs__core-11491", "vuejs__core-11165", "vuejs__core-10939", "vuejs__core-10522", "vuejs__core-10289", "vuejs__core-10161", "vuejs__core-9751", "vuejs__core-9532", "vuejs__core-9492", "vuejs__core-8911", "vuejs__core-8824", "vuejs__core-8785", "vuejs__core-8538", "vuejs__core-8470", "vuejs__core-8365", "mui__material-ui-39775", "mui__material-ui-39465", "mui__material-ui-39457", "mui__material-ui-39108", "mui__material-ui-39071", "mui__material-ui-38544", "mui__material-ui-38247", "mui__material-ui-38188", "mui__material-ui-38100", "mui__material-ui-36353", "mui__material-ui-36116", "mui__material-ui-35838", "mui__material-ui-35410", "mui__material-ui-35360", "mui__material-ui-35047", "mui__material-ui-34966", "mui__material-ui-34610", "mui__material-ui-34548", "mui__material-ui-34478", "mui__material-ui-34455", "mui__material-ui-34445", "mui__material-ui-34437", "mui__material-ui-34337", "mui__material-ui-34159", "mui__material-ui-33957", "mui__material-ui-33880", "mui__material-ui-33820", "mui__material-ui-33777", "mui__material-ui-33692", "mui__material-ui-32713", "mui__material-ui-32626", "mui__material-ui-31313", "mui__material-ui-30899", "mui__material-ui-30561", "mui__material-ui-30374", "mui__material-ui-29613", "mui__material-ui-26170", "mui__material-ui-25637", "mui__material-ui-25431", "mui__material-ui-25276", "mui__material-ui-25105", "mui__material-ui-24290", "mui__material-ui-24213", "mui__material-ui-24205", "mui__material-ui-24202", "mui__material-ui-24010", "mui__material-ui-23884", "mui__material-ui-23786", "mui__material-ui-23778" ], "medium_ids": [ "vuejs__core-11899", "vuejs__core-11854", "vuejs__core-11517", "vuejs__core-11338", "vuejs__core-11201", "vuejs__core-11026", "vuejs__core-10989", "vuejs__core-10891", "vuejs__core-10874", "vuejs__core-10869", "vuejs__core-10416", "vuejs__core-10250", "vuejs__core-10218", "vuejs__core-10027", "vuejs__core-10004", "vuejs__core-9572", "vuejs__core-9507", "vuejs__core-9442", "vuejs__core-9213", "vuejs__core-8511", "vuejs__core-8402", "vuejs__core-8381", "mui__material-ui-39962", "mui__material-ui-39536", "mui__material-ui-39353", "mui__material-ui-39196", "mui__material-ui-38801", "mui__material-ui-38788", "mui__material-ui-38182", "mui__material-ui-37855", "mui__material-ui-37692", "mui__material-ui-37141", "mui__material-ui-37118", "mui__material-ui-37108", "mui__material-ui-37070", "mui__material-ui-36853", "mui__material-ui-36839", "mui__material-ui-36659", "mui__material-ui-36403", "mui__material-ui-36088", "mui__material-ui-36056", "mui__material-ui-35804", "mui__material-ui-35477", "mui__material-ui-35364", "mui__material-ui-35088", "mui__material-ui-34829", "mui__material-ui-34247", "mui__material-ui-34245", "mui__material-ui-34229", "mui__material-ui-34216", "mui__material-ui-33934", "mui__material-ui-33801", "mui__material-ui-33312", "mui__material-ui-33283", "mui__material-ui-33145", "mui__material-ui-33040", "mui__material-ui-32987", "mui__material-ui-32962", "mui__material-ui-32401", "mui__material-ui-32185", "mui__material-ui-32182", "mui__material-ui-30788", "mui__material-ui-30515", "mui__material-ui-30333", "mui__material-ui-30126", "mui__material-ui-30118", "mui__material-ui-29988", "mui__material-ui-29954", "mui__material-ui-29880", "mui__material-ui-29756", "mui__material-ui-29579", "mui__material-ui-29317", "mui__material-ui-28645", "mui__material-ui-26470", "mui__material-ui-26413", "mui__material-ui-26173", "mui__material-ui-26098", "mui__material-ui-26061", "mui__material-ui-25825", "mui__material-ui-25815", "mui__material-ui-25428", "mui__material-ui-25259", "mui__material-ui-24968", "mui__material-ui-24794", "mui__material-ui-24782", "mui__material-ui-24621", "mui__material-ui-23806", "mui__material-ui-23487" ], "hard_ids": [ "vuejs__core-11761", "vuejs__core-10325", "vuejs__core-10141", "vuejs__core-10101", "vuejs__core-8535", "mui__material-ui-39688", "mui__material-ui-39679", "mui__material-ui-38916", "mui__material-ui-38169", "mui__material-ui-38167", "mui__material-ui-37850", "mui__material-ui-37716", "mui__material-ui-37667", "mui__material-ui-37465", "mui__material-ui-36971", "mui__material-ui-36426", "mui__material-ui-36401", "mui__material-ui-36400", "mui__material-ui-36399", "mui__material-ui-36024", "mui__material-ui-35994", "mui__material-ui-35963", "mui__material-ui-35946", "mui__material-ui-35872", "mui__material-ui-35178", "mui__material-ui-35086", "mui__material-ui-34401", "mui__material-ui-34380", "mui__material-ui-34258", "mui__material-ui-34207", "mui__material-ui-34158", "mui__material-ui-34131", "mui__material-ui-33873", "mui__material-ui-33863", "mui__material-ui-33812", "mui__material-ui-33797", "mui__material-ui-33691", "mui__material-ui-33582", "mui__material-ui-33415", "mui__material-ui-32746", "mui__material-ui-31998", "mui__material-ui-31172", "mui__material-ui-30524", "mui__material-ui-30345", "mui__material-ui-29902", "mui__material-ui-29901", "mui__material-ui-29898", "mui__material-ui-29824", "mui__material-ui-29488", "mui__material-ui-29297", "mui__material-ui-26480", "mui__material-ui-26460", "mui__material-ui-26323", "mui__material-ui-26186", "mui__material-ui-26026", "mui__material-ui-25974", "mui__material-ui-25874", "mui__material-ui-25784", "mui__material-ui-25390", "mui__material-ui-25245", "mui__material-ui-25149", "mui__material-ui-24945", "mui__material-ui-24857", "mui__material-ui-23073" ] } } ] }, { "name": "JavaScript", "data": [ { "name": "Verified", "results": [ { "name": "RepoRepair + Claude-4.5-Sonnet", "site": "https://github.com/JXTZZ/M-RepoRepair", "oss": true, "verified": false, "resolved": 61, "resolvedEasy": 7, "resolvedMedium": 29, "resolvedHard": 25, "resolvedRate": 0.17134831460674158, "resolvedEasyRate": 0.7, "resolvedMediumRate": 0.2761904761904762, "resolvedHardRate": 0.1037344398340249, "date": "2026-02-02", "path": "javascript/verified/20260202_RepoRepair_Claude-4.5-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 58, "resolvedEasy": 4, "resolvedMedium": 35, "resolvedHard": 19, "resolvedRate": 0.16292134831460675, "resolvedEasyRate": 0.4, "resolvedMediumRate": 0.3333333333333333, "resolvedHardRate": 0.07883817427385892, "date": "2025-04-26", "path": "javascript/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 34, "resolvedEasy": 2, "resolvedMedium": 19, "resolvedHard": 13, "resolvedRate": 0.09550561797752809, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.18095238095238095, "resolvedHardRate": 0.05394190871369295, "date": "2025-04-26", "path": "javascript/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 31, "resolvedEasy": 3, "resolvedMedium": 15, "resolvedHard": 13, "resolvedRate": 0.08707865168539326, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.14285714285714285, "resolvedHardRate": 0.05394190871369295, "date": "2025-04-26", "path": "javascript/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 27, "resolvedEasy": 2, "resolvedMedium": 17, "resolvedHard": 8, "resolvedRate": 0.07584269662921349, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.1619047619047619, "resolvedHardRate": 0.03319502074688797, "date": "2025-04-26", "path": "javascript/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 20, "resolvedEasy": 3, "resolvedMedium": 10, "resolvedHard": 7, "resolvedRate": 0.056179775280898875, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.09523809523809523, "resolvedHardRate": 0.029045643153526972, "date": "2025-04-26", "path": "javascript/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 18, "resolvedEasy": 2, "resolvedMedium": 10, "resolvedHard": 6, "resolvedRate": 0.05056179775280899, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.09523809523809523, "resolvedHardRate": 0.024896265560165973, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 18, "resolvedEasy": 3, "resolvedMedium": 8, "resolvedHard": 7, "resolvedRate": 0.05056179775280899, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.0761904761904762, "resolvedHardRate": 0.029045643153526972, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 18, "resolvedEasy": 3, "resolvedMedium": 11, "resolvedHard": 4, "resolvedRate": 0.05056179775280899, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.10476190476190476, "resolvedHardRate": 0.016597510373443983, "date": "2025-04-26", "path": "javascript/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 17, "resolvedEasy": 0, "resolvedMedium": 8, "resolvedHard": 9, "resolvedRate": 0.047752808988764044, "resolvedEasyRate": 0, "resolvedMediumRate": 0.0761904761904762, "resolvedHardRate": 0.03734439834024896, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 17, "resolvedEasy": 3, "resolvedMedium": 7, "resolvedHard": 7, "resolvedRate": 0.047752808988764044, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.06666666666666667, "resolvedHardRate": 0.029045643153526972, "date": "2025-04-26", "path": "javascript/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 16, "resolvedEasy": 3, "resolvedMedium": 8, "resolvedHard": 5, "resolvedRate": 0.0449438202247191, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.0761904761904762, "resolvedHardRate": 0.02074688796680498, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 0, "resolvedMedium": 8, "resolvedHard": 7, "resolvedRate": 0.042134831460674156, "resolvedEasyRate": 0, "resolvedMediumRate": 0.0761904761904762, "resolvedHardRate": 0.029045643153526972, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 1, "resolvedMedium": 7, "resolvedHard": 7, "resolvedRate": 0.042134831460674156, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.06666666666666667, "resolvedHardRate": 0.029045643153526972, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 2, "resolvedMedium": 4, "resolvedHard": 9, "resolvedRate": 0.042134831460674156, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.0380952380952381, "resolvedHardRate": 0.03734439834024896, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 13, "resolvedEasy": 3, "resolvedMedium": 7, "resolvedHard": 3, "resolvedRate": 0.03651685393258427, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.06666666666666667, "resolvedHardRate": 0.012448132780082987, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 12, "resolvedEasy": 3, "resolvedMedium": 5, "resolvedHard": 4, "resolvedRate": 0.033707865168539325, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.047619047619047616, "resolvedHardRate": 0.016597510373443983, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 4, "resolvedMedium": 4, "resolvedHard": 4, "resolvedRate": 0.033707865168539325, "resolvedEasyRate": 0.4, "resolvedMediumRate": 0.0380952380952381, "resolvedHardRate": 0.016597510373443983, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 10, "resolvedEasy": 3, "resolvedMedium": 5, "resolvedHard": 2, "resolvedRate": 0.028089887640449437, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.047619047619047616, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 0, "resolvedMedium": 4, "resolvedHard": 5, "resolvedRate": 0.025280898876404494, "resolvedEasyRate": 0, "resolvedMediumRate": 0.0380952380952381, "resolvedHardRate": 0.02074688796680498, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 9, "resolvedEasy": 1, "resolvedMedium": 3, "resolvedHard": 5, "resolvedRate": 0.025280898876404494, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.02857142857142857, "resolvedHardRate": 0.02074688796680498, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 8, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 5, "resolvedRate": 0.02247191011235955, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.009523809523809525, "resolvedHardRate": 0.02074688796680498, "date": "2025-04-26", "path": "javascript/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 2, "resolvedRate": 0.019662921348314606, "resolvedEasyRate": 0.3, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 2, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.019662921348314606, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.0380952380952381, "resolvedHardRate": 0.004149377593360996, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 7, "resolvedEasy": 1, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.019662921348314606, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.0380952380952381, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 7, "resolvedEasy": 0, "resolvedMedium": 3, "resolvedHard": 4, "resolvedRate": 0.019662921348314606, "resolvedEasyRate": 0, "resolvedMediumRate": 0.02857142857142857, "resolvedHardRate": 0.016597510373443983, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 6, "resolvedEasy": 0, "resolvedMedium": 2, "resolvedHard": 4, "resolvedRate": 0.016853932584269662, "resolvedEasyRate": 0, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.016597510373443983, "date": "2025-04-26", "path": "javascript/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 1, "resolvedMedium": 2, "resolvedHard": 2, "resolvedRate": 0.014044943820224719, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 1, "resolvedMedium": 2, "resolvedHard": 2, "resolvedRate": 0.014044943820224719, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.014044943820224719, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.004149377593360996, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 4, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 3, "resolvedRate": 0.011235955056179775, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0, "resolvedHardRate": 0.012448132780082987, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 0, "resolvedMedium": 2, "resolvedHard": 2, "resolvedRate": 0.011235955056179775, "resolvedEasyRate": 0, "resolvedMediumRate": 0.01904761904761905, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 2, "resolvedRate": 0.011235955056179775, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.009523809523809525, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 2, "resolvedRate": 0.008426966292134831, "resolvedEasyRate": 0, "resolvedMediumRate": 0.009523809523809525, "resolvedHardRate": 0.008298755186721992, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 1, "resolvedRate": 0.008426966292134831, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0, "resolvedHardRate": 0.004149377593360996, "date": "2025-03-29", "path": "javascript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.008426966292134831, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.009523809523809525, "resolvedHardRate": 0, "date": "2025-03-29", "path": "javascript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 2, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.0056179775280898875, "resolvedEasyRate": 0, "resolvedMediumRate": 0.009523809523809525, "resolvedHardRate": 0.004149377593360996, "date": "2025-03-29", "path": "javascript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "axios__axios-5919", "axios__axios-5661", "axios__axios-5338", "axios__axios-5085", "iamkun__dayjs-2532", "iamkun__dayjs-2420", "iamkun__dayjs-2399", "iamkun__dayjs-2377", "iamkun__dayjs-2369", "iamkun__dayjs-2367", "iamkun__dayjs-2231", "iamkun__dayjs-2175", "iamkun__dayjs-1964", "iamkun__dayjs-1954", "iamkun__dayjs-1953", "iamkun__dayjs-1725", "iamkun__dayjs-1611", "iamkun__dayjs-1567", "iamkun__dayjs-1502", "iamkun__dayjs-1470", "iamkun__dayjs-1414", "iamkun__dayjs-1321", "iamkun__dayjs-1319", "iamkun__dayjs-1307", "iamkun__dayjs-1295", "iamkun__dayjs-1209", "iamkun__dayjs-1202", "iamkun__dayjs-1201", "iamkun__dayjs-1161", "iamkun__dayjs-1160", "iamkun__dayjs-1112", "iamkun__dayjs-1047", "iamkun__dayjs-1023", "iamkun__dayjs-1019", "iamkun__dayjs-1010", "iamkun__dayjs-1008", "iamkun__dayjs-973", "iamkun__dayjs-957", "iamkun__dayjs-950", "iamkun__dayjs-938", "iamkun__dayjs-918", "iamkun__dayjs-889", "iamkun__dayjs-882", "iamkun__dayjs-873", "iamkun__dayjs-872", "iamkun__dayjs-867", "iamkun__dayjs-858", "iamkun__dayjs-857", "iamkun__dayjs-745", "iamkun__dayjs-734", "iamkun__dayjs-668", "iamkun__dayjs-658", "iamkun__dayjs-602", "iamkun__dayjs-591", "iamkun__dayjs-589", "iamkun__dayjs-569", "iamkun__dayjs-556", "iamkun__dayjs-379", "iamkun__dayjs-337", "iamkun__dayjs-318", "Kong__insomnia-7734", "sveltejs__svelte-15115", "sveltejs__svelte-15086", "sveltejs__svelte-15083", "sveltejs__svelte-15057", "sveltejs__svelte-15020", "sveltejs__svelte-15013", "sveltejs__svelte-14993", "sveltejs__svelte-14942", "sveltejs__svelte-14937", "sveltejs__svelte-14935", "sveltejs__svelte-14907", "sveltejs__svelte-14811", "sveltejs__svelte-14699", "sveltejs__svelte-14691", "sveltejs__svelte-14677", "sveltejs__svelte-14640", "sveltejs__svelte-14629", "sveltejs__svelte-14626", "sveltejs__svelte-14620", "sveltejs__svelte-14607", "sveltejs__svelte-14597", "sveltejs__svelte-14577", "sveltejs__svelte-14575", "sveltejs__svelte-14562", "sveltejs__svelte-14554", "sveltejs__svelte-14546", "sveltejs__svelte-14494", "sveltejs__svelte-14468", "sveltejs__svelte-14456", "sveltejs__svelte-14448", "sveltejs__svelte-14445", "sveltejs__svelte-14440", "sveltejs__svelte-14378", "sveltejs__svelte-14353", "sveltejs__svelte-14349", "sveltejs__svelte-14317", "sveltejs__svelte-14289", "sveltejs__svelte-14276", "sveltejs__svelte-14273", "sveltejs__svelte-14260", "sveltejs__svelte-14236", "sveltejs__svelte-14224", "sveltejs__svelte-14222", "sveltejs__svelte-14216", "sveltejs__svelte-14213", "sveltejs__svelte-14210", "sveltejs__svelte-14195", "sveltejs__svelte-14153", "sveltejs__svelte-14143", "sveltejs__svelte-14134", "sveltejs__svelte-14101", "sveltejs__svelte-14007", "sveltejs__svelte-13866", "sveltejs__svelte-13850", "sveltejs__svelte-13849", "sveltejs__svelte-13824", "sveltejs__svelte-13774", "sveltejs__svelte-13763", "sveltejs__svelte-13760", "sveltejs__svelte-13750", "sveltejs__svelte-13746", "sveltejs__svelte-13734", "sveltejs__svelte-13698", "sveltejs__svelte-13690", "sveltejs__svelte-13679", "sveltejs__svelte-13668", "sveltejs__svelte-13656", "sveltejs__svelte-13646", "sveltejs__svelte-13632", "sveltejs__svelte-13630", "sveltejs__svelte-13617", "sveltejs__svelte-13589", "sveltejs__svelte-13586", "sveltejs__svelte-13568", "sveltejs__svelte-13559", "sveltejs__svelte-13553", "sveltejs__svelte-13540", "sveltejs__svelte-13526", "sveltejs__svelte-13473", "sveltejs__svelte-13461", "sveltejs__svelte-13456", "sveltejs__svelte-13445", "sveltejs__svelte-13437", "sveltejs__svelte-13427", "sveltejs__svelte-13416", "sveltejs__svelte-13411", "sveltejs__svelte-13362", "sveltejs__svelte-13333", "sveltejs__svelte-13316", "sveltejs__svelte-13291", "sveltejs__svelte-13265", "sveltejs__svelte-13262", "sveltejs__svelte-13247", "sveltejs__svelte-13209", "sveltejs__svelte-13198", "sveltejs__svelte-13175", "sveltejs__svelte-13171", "sveltejs__svelte-13151", "sveltejs__svelte-13145", "sveltejs__svelte-13104", "sveltejs__svelte-13097", "sveltejs__svelte-13073", "sveltejs__svelte-13057", "sveltejs__svelte-13035", "sveltejs__svelte-13026", "sveltejs__svelte-13018", "sveltejs__svelte-13008", "sveltejs__svelte-12963", "sveltejs__svelte-12949", "sveltejs__svelte-12948", "sveltejs__svelte-12938", "sveltejs__svelte-12899", "sveltejs__svelte-12869", "sveltejs__svelte-12845", "sveltejs__svelte-12799", "sveltejs__svelte-12734", "sveltejs__svelte-12715", "sveltejs__svelte-12692", "sveltejs__svelte-12681", "sveltejs__svelte-12679", "sveltejs__svelte-12660", "sveltejs__svelte-12649", "sveltejs__svelte-12638", "sveltejs__svelte-12618", "sveltejs__svelte-12562", "sveltejs__svelte-12560", "sveltejs__svelte-12545", "sveltejs__svelte-12524", "sveltejs__svelte-12509", "sveltejs__svelte-12493", "sveltejs__svelte-12475", "sveltejs__svelte-12474", "sveltejs__svelte-12469", "sveltejs__svelte-12460", "sveltejs__svelte-12445", "sveltejs__svelte-12413", "sveltejs__svelte-12376", "sveltejs__svelte-12375", "sveltejs__svelte-12373", "sveltejs__svelte-12370", "sveltejs__svelte-12365", "sveltejs__svelte-12345", "sveltejs__svelte-12342", "sveltejs__svelte-12329", "sveltejs__svelte-12321", "sveltejs__svelte-12316", "sveltejs__svelte-12304", "sveltejs__svelte-12298", "sveltejs__svelte-12290", "sveltejs__svelte-12252", "sveltejs__svelte-12215", "sveltejs__svelte-12176", "sveltejs__svelte-12123", "sveltejs__svelte-12119", "sveltejs__svelte-12098", "sveltejs__svelte-12056", "sveltejs__svelte-12047", "sveltejs__svelte-12019", "sveltejs__svelte-12007", "sveltejs__svelte-11947", "sveltejs__svelte-11946", "sveltejs__svelte-11937", "sveltejs__svelte-11913", "sveltejs__svelte-11835", "sveltejs__svelte-11784", "sveltejs__svelte-11768", "sveltejs__svelte-11766", "sveltejs__svelte-11736", "sveltejs__svelte-11720", "sveltejs__svelte-11676", "sveltejs__svelte-11669", "sveltejs__svelte-11652", "sveltejs__svelte-11632", "sveltejs__svelte-11496", "sveltejs__svelte-11495", "sveltejs__svelte-11490", "sveltejs__svelte-11487", "sveltejs__svelte-11470", "sveltejs__svelte-11463", "sveltejs__svelte-11454", "sveltejs__svelte-11421", "sveltejs__svelte-11378", "sveltejs__svelte-11371", "sveltejs__svelte-11367", "sveltejs__svelte-11357", "sveltejs__svelte-11344", "sveltejs__svelte-11334", "sveltejs__svelte-11326", "sveltejs__svelte-11325", "sveltejs__svelte-11319", "sveltejs__svelte-11295", "sveltejs__svelte-11290", "sveltejs__svelte-11287", "sveltejs__svelte-11276", "sveltejs__svelte-11254", "sveltejs__svelte-11251", "sveltejs__svelte-11219", "sveltejs__svelte-11209", "sveltejs__svelte-11203", "sveltejs__svelte-11184", "sveltejs__svelte-11183", "sveltejs__svelte-11114", "sveltejs__svelte-11104", "sveltejs__svelte-11096", "sveltejs__svelte-11059", "sveltejs__svelte-10996", "sveltejs__svelte-10917", "sveltejs__svelte-10797", "sveltejs__svelte-10728", "sveltejs__svelte-10727", "sveltejs__svelte-10721", "sveltejs__svelte-10700", "sveltejs__svelte-10691", "sveltejs__svelte-10673", "sveltejs__svelte-10671", "sveltejs__svelte-10669", "sveltejs__svelte-10656", "sveltejs__svelte-10655", "sveltejs__svelte-10642", "sveltejs__svelte-10617", "sveltejs__svelte-10608", "sveltejs__svelte-10590", "sveltejs__svelte-10553", "sveltejs__svelte-10430", "sveltejs__svelte-10413", "sveltejs__svelte-10398", "sveltejs__svelte-10395", "sveltejs__svelte-10391", "sveltejs__svelte-10378", "sveltejs__svelte-10368", "sveltejs__svelte-10297", "sveltejs__svelte-10291", "sveltejs__svelte-10287", "sveltejs__svelte-10267", "sveltejs__svelte-10266", "sveltejs__svelte-10259", "sveltejs__svelte-10235", "sveltejs__svelte-10209", "sveltejs__svelte-10187", "sveltejs__svelte-10181", "sveltejs__svelte-10114", "sveltejs__svelte-10077", "sveltejs__svelte-10071", "sveltejs__svelte-10054", "sveltejs__svelte-10046", "sveltejs__svelte-10027", "sveltejs__svelte-9988", "sveltejs__svelte-9973", "sveltejs__svelte-9970", "sveltejs__svelte-9962", "sveltejs__svelte-9904", "sveltejs__svelte-9838", "sveltejs__svelte-9824", "sveltejs__svelte-9811", "sveltejs__svelte-9789", "sveltejs__svelte-9784", "sveltejs__svelte-9754", "sveltejs__svelte-9731", "sveltejs__svelte-9723", "sveltejs__svelte-9718", "sveltejs__svelte-9717", "sveltejs__svelte-9713", "sveltejs__svelte-9665", "sveltejs__svelte-9664", "sveltejs__svelte-9626", "sveltejs__svelte-9550", "sveltejs__svelte-9500", "sveltejs__svelte-9497", "sveltejs__svelte-9471", "sveltejs__svelte-9462", "sveltejs__svelte-9456", "sveltejs__svelte-9328", "expressjs__express-5555", "expressjs__express-3870", "expressjs__express-3695", "expressjs__express-3495", "anuraghazra__github-readme-stats-3442", "anuraghazra__github-readme-stats-2844", "anuraghazra__github-readme-stats-2491", "anuraghazra__github-readme-stats-2228", "anuraghazra__github-readme-stats-2099", "anuraghazra__github-readme-stats-2075", "anuraghazra__github-readme-stats-1608", "anuraghazra__github-readme-stats-1378", "anuraghazra__github-readme-stats-1370", "anuraghazra__github-readme-stats-1314", "anuraghazra__github-readme-stats-1041", "anuraghazra__github-readme-stats-721", "anuraghazra__github-readme-stats-293", "anuraghazra__github-readme-stats-148", "anuraghazra__github-readme-stats-117", "anuraghazra__github-readme-stats-105", "anuraghazra__github-readme-stats-99", "anuraghazra__github-readme-stats-88", "anuraghazra__github-readme-stats-58" ], "easy_ids": [ "iamkun__dayjs-2532", "iamkun__dayjs-2369", "iamkun__dayjs-2175", "iamkun__dayjs-1953", "iamkun__dayjs-1414", "iamkun__dayjs-957", "iamkun__dayjs-889", "iamkun__dayjs-602", "sveltejs__svelte-12365", "expressjs__express-5555" ], "medium_ids": [ "axios__axios-5085", "iamkun__dayjs-2420", "iamkun__dayjs-2399", "iamkun__dayjs-2377", "iamkun__dayjs-1964", "iamkun__dayjs-1954", "iamkun__dayjs-1725", "iamkun__dayjs-1611", "iamkun__dayjs-1567", "iamkun__dayjs-1502", "iamkun__dayjs-1470", "iamkun__dayjs-1321", "iamkun__dayjs-1319", "iamkun__dayjs-1307", "iamkun__dayjs-1295", "iamkun__dayjs-1209", "iamkun__dayjs-1201", "iamkun__dayjs-1161", "iamkun__dayjs-1160", "iamkun__dayjs-1023", "iamkun__dayjs-1019", "iamkun__dayjs-1010", "iamkun__dayjs-1008", "iamkun__dayjs-973", "iamkun__dayjs-950", "iamkun__dayjs-938", "iamkun__dayjs-918", "iamkun__dayjs-867", "iamkun__dayjs-857", "iamkun__dayjs-745", "iamkun__dayjs-734", "iamkun__dayjs-668", "iamkun__dayjs-591", "iamkun__dayjs-589", "iamkun__dayjs-556", "iamkun__dayjs-379", "iamkun__dayjs-337", "sveltejs__svelte-15013", "sveltejs__svelte-14942", "sveltejs__svelte-14937", "sveltejs__svelte-14699", "sveltejs__svelte-14626", "sveltejs__svelte-14620", "sveltejs__svelte-14562", "sveltejs__svelte-14546", "sveltejs__svelte-14468", "sveltejs__svelte-14276", "sveltejs__svelte-14222", "sveltejs__svelte-14153", "sveltejs__svelte-14101", "sveltejs__svelte-13774", "sveltejs__svelte-13760", "sveltejs__svelte-13646", "sveltejs__svelte-13617", "sveltejs__svelte-13559", "sveltejs__svelte-13540", "sveltejs__svelte-13411", "sveltejs__svelte-13316", "sveltejs__svelte-13265", "sveltejs__svelte-13198", "sveltejs__svelte-13151", "sveltejs__svelte-13104", "sveltejs__svelte-13097", "sveltejs__svelte-13073", "sveltejs__svelte-12649", "sveltejs__svelte-12638", "sveltejs__svelte-12562", "sveltejs__svelte-12376", "sveltejs__svelte-12373", "sveltejs__svelte-12298", "sveltejs__svelte-12119", "sveltejs__svelte-12056", "sveltejs__svelte-12007", "sveltejs__svelte-11947", "sveltejs__svelte-11937", "sveltejs__svelte-11913", "sveltejs__svelte-11835", "sveltejs__svelte-11784", "sveltejs__svelte-11736", "sveltejs__svelte-11676", "sveltejs__svelte-11669", "sveltejs__svelte-11496", "sveltejs__svelte-11487", "sveltejs__svelte-11463", "sveltejs__svelte-11378", "sveltejs__svelte-11357", "sveltejs__svelte-11344", "sveltejs__svelte-11251", "sveltejs__svelte-11209", "sveltejs__svelte-11114", "sveltejs__svelte-10700", "sveltejs__svelte-10669", "sveltejs__svelte-10655", "sveltejs__svelte-10590", "sveltejs__svelte-10391", "sveltejs__svelte-10297", "sveltejs__svelte-10291", "sveltejs__svelte-10267", "sveltejs__svelte-10235", "sveltejs__svelte-9718", "sveltejs__svelte-9713", "sveltejs__svelte-9500", "sveltejs__svelte-9497", "sveltejs__svelte-9471", "anuraghazra__github-readme-stats-99" ], "hard_ids": [ "axios__axios-5919", "axios__axios-5661", "axios__axios-5338", "iamkun__dayjs-2367", "iamkun__dayjs-2231", "iamkun__dayjs-1202", "iamkun__dayjs-1112", "iamkun__dayjs-1047", "iamkun__dayjs-882", "iamkun__dayjs-873", "iamkun__dayjs-872", "iamkun__dayjs-858", "iamkun__dayjs-658", "iamkun__dayjs-569", "iamkun__dayjs-318", "Kong__insomnia-7734", "sveltejs__svelte-15115", "sveltejs__svelte-15086", "sveltejs__svelte-15083", "sveltejs__svelte-15057", "sveltejs__svelte-15020", "sveltejs__svelte-14993", "sveltejs__svelte-14935", "sveltejs__svelte-14907", "sveltejs__svelte-14811", "sveltejs__svelte-14691", "sveltejs__svelte-14677", "sveltejs__svelte-14640", "sveltejs__svelte-14629", "sveltejs__svelte-14607", "sveltejs__svelte-14597", "sveltejs__svelte-14577", "sveltejs__svelte-14575", "sveltejs__svelte-14554", "sveltejs__svelte-14494", "sveltejs__svelte-14456", "sveltejs__svelte-14448", "sveltejs__svelte-14445", "sveltejs__svelte-14440", "sveltejs__svelte-14378", "sveltejs__svelte-14353", "sveltejs__svelte-14349", "sveltejs__svelte-14317", "sveltejs__svelte-14289", "sveltejs__svelte-14273", "sveltejs__svelte-14260", "sveltejs__svelte-14236", "sveltejs__svelte-14224", "sveltejs__svelte-14216", "sveltejs__svelte-14213", "sveltejs__svelte-14210", "sveltejs__svelte-14195", "sveltejs__svelte-14143", "sveltejs__svelte-14134", "sveltejs__svelte-14007", "sveltejs__svelte-13866", "sveltejs__svelte-13850", "sveltejs__svelte-13849", "sveltejs__svelte-13824", "sveltejs__svelte-13763", "sveltejs__svelte-13750", "sveltejs__svelte-13746", "sveltejs__svelte-13734", "sveltejs__svelte-13698", "sveltejs__svelte-13690", "sveltejs__svelte-13679", "sveltejs__svelte-13668", "sveltejs__svelte-13656", "sveltejs__svelte-13632", "sveltejs__svelte-13630", "sveltejs__svelte-13589", "sveltejs__svelte-13586", "sveltejs__svelte-13568", "sveltejs__svelte-13553", "sveltejs__svelte-13526", "sveltejs__svelte-13473", "sveltejs__svelte-13461", "sveltejs__svelte-13456", "sveltejs__svelte-13445", "sveltejs__svelte-13437", "sveltejs__svelte-13427", "sveltejs__svelte-13416", "sveltejs__svelte-13362", "sveltejs__svelte-13333", "sveltejs__svelte-13291", "sveltejs__svelte-13262", "sveltejs__svelte-13247", "sveltejs__svelte-13209", "sveltejs__svelte-13175", "sveltejs__svelte-13171", "sveltejs__svelte-13145", "sveltejs__svelte-13057", "sveltejs__svelte-13035", "sveltejs__svelte-13026", "sveltejs__svelte-13018", "sveltejs__svelte-13008", "sveltejs__svelte-12963", "sveltejs__svelte-12949", "sveltejs__svelte-12948", "sveltejs__svelte-12938", "sveltejs__svelte-12899", "sveltejs__svelte-12869", "sveltejs__svelte-12845", "sveltejs__svelte-12799", "sveltejs__svelte-12734", "sveltejs__svelte-12715", "sveltejs__svelte-12692", "sveltejs__svelte-12681", "sveltejs__svelte-12679", "sveltejs__svelte-12660", "sveltejs__svelte-12618", "sveltejs__svelte-12560", "sveltejs__svelte-12545", "sveltejs__svelte-12524", "sveltejs__svelte-12509", "sveltejs__svelte-12493", "sveltejs__svelte-12475", "sveltejs__svelte-12474", "sveltejs__svelte-12469", "sveltejs__svelte-12460", "sveltejs__svelte-12445", "sveltejs__svelte-12413", "sveltejs__svelte-12375", "sveltejs__svelte-12370", "sveltejs__svelte-12345", "sveltejs__svelte-12342", "sveltejs__svelte-12329", "sveltejs__svelte-12321", "sveltejs__svelte-12316", "sveltejs__svelte-12304", "sveltejs__svelte-12290", "sveltejs__svelte-12252", "sveltejs__svelte-12215", "sveltejs__svelte-12176", "sveltejs__svelte-12123", "sveltejs__svelte-12098", "sveltejs__svelte-12047", "sveltejs__svelte-12019", "sveltejs__svelte-11946", "sveltejs__svelte-11768", "sveltejs__svelte-11766", "sveltejs__svelte-11720", "sveltejs__svelte-11652", "sveltejs__svelte-11632", "sveltejs__svelte-11495", "sveltejs__svelte-11490", "sveltejs__svelte-11470", "sveltejs__svelte-11454", "sveltejs__svelte-11421", "sveltejs__svelte-11371", "sveltejs__svelte-11367", "sveltejs__svelte-11334", "sveltejs__svelte-11326", "sveltejs__svelte-11325", "sveltejs__svelte-11319", "sveltejs__svelte-11295", "sveltejs__svelte-11290", "sveltejs__svelte-11287", "sveltejs__svelte-11276", "sveltejs__svelte-11254", "sveltejs__svelte-11219", "sveltejs__svelte-11203", "sveltejs__svelte-11184", "sveltejs__svelte-11183", "sveltejs__svelte-11104", "sveltejs__svelte-11096", "sveltejs__svelte-11059", "sveltejs__svelte-10996", "sveltejs__svelte-10917", "sveltejs__svelte-10797", "sveltejs__svelte-10728", "sveltejs__svelte-10727", "sveltejs__svelte-10721", "sveltejs__svelte-10691", "sveltejs__svelte-10673", "sveltejs__svelte-10671", "sveltejs__svelte-10656", "sveltejs__svelte-10642", "sveltejs__svelte-10617", "sveltejs__svelte-10608", "sveltejs__svelte-10553", "sveltejs__svelte-10430", "sveltejs__svelte-10413", "sveltejs__svelte-10398", "sveltejs__svelte-10395", "sveltejs__svelte-10378", "sveltejs__svelte-10368", "sveltejs__svelte-10287", "sveltejs__svelte-10266", "sveltejs__svelte-10259", "sveltejs__svelte-10209", "sveltejs__svelte-10187", "sveltejs__svelte-10181", "sveltejs__svelte-10114", "sveltejs__svelte-10077", "sveltejs__svelte-10071", "sveltejs__svelte-10054", "sveltejs__svelte-10046", "sveltejs__svelte-10027", "sveltejs__svelte-9988", "sveltejs__svelte-9973", "sveltejs__svelte-9970", "sveltejs__svelte-9962", "sveltejs__svelte-9904", "sveltejs__svelte-9838", "sveltejs__svelte-9824", "sveltejs__svelte-9811", "sveltejs__svelte-9789", "sveltejs__svelte-9784", "sveltejs__svelte-9754", "sveltejs__svelte-9731", "sveltejs__svelte-9723", "sveltejs__svelte-9717", "sveltejs__svelte-9665", "sveltejs__svelte-9664", "sveltejs__svelte-9626", "sveltejs__svelte-9550", "sveltejs__svelte-9462", "sveltejs__svelte-9456", "sveltejs__svelte-9328", "expressjs__express-3870", "expressjs__express-3695", "expressjs__express-3495", "anuraghazra__github-readme-stats-3442", "anuraghazra__github-readme-stats-2844", "anuraghazra__github-readme-stats-2491", "anuraghazra__github-readme-stats-2228", "anuraghazra__github-readme-stats-2099", "anuraghazra__github-readme-stats-2075", "anuraghazra__github-readme-stats-1608", "anuraghazra__github-readme-stats-1378", "anuraghazra__github-readme-stats-1370", "anuraghazra__github-readme-stats-1314", "anuraghazra__github-readme-stats-1041", "anuraghazra__github-readme-stats-721", "anuraghazra__github-readme-stats-293", "anuraghazra__github-readme-stats-148", "anuraghazra__github-readme-stats-117", "anuraghazra__github-readme-stats-105", "anuraghazra__github-readme-stats-88", "anuraghazra__github-readme-stats-58" ] } } ] }, { "name": "Go", "data": [ { "name": "Verified", "results": [ { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 54, "resolvedEasy": 36, "resolvedMedium": 14, "resolvedHard": 4, "resolvedRate": 0.1261682242990654, "resolvedEasyRate": 0.2553191489361702, "resolvedMediumRate": 0.0915032679738562, "resolvedHardRate": 0.029850746268656716, "date": "2025-04-26", "path": "go/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 42, "resolvedEasy": 27, "resolvedMedium": 13, "resolvedHard": 2, "resolvedRate": 0.09813084112149532, "resolvedEasyRate": 0.19148936170212766, "resolvedMediumRate": 0.08496732026143791, "resolvedHardRate": 0.014925373134328358, "date": "2025-04-26", "path": "go/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 32, "resolvedEasy": 16, "resolvedMedium": 14, "resolvedHard": 2, "resolvedRate": 0.07476635514018691, "resolvedEasyRate": 0.11347517730496454, "resolvedMediumRate": 0.0915032679738562, "resolvedHardRate": 0.014925373134328358, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 29, "resolvedEasy": 18, "resolvedMedium": 8, "resolvedHard": 3, "resolvedRate": 0.06775700934579439, "resolvedEasyRate": 0.1276595744680851, "resolvedMediumRate": 0.05228758169934641, "resolvedHardRate": 0.022388059701492536, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 26, "resolvedEasy": 20, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.06074766355140187, "resolvedEasyRate": 0.14184397163120568, "resolvedMediumRate": 0.032679738562091505, "resolvedHardRate": 0.007462686567164179, "date": "2025-04-26", "path": "go/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 26, "resolvedEasy": 21, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.06074766355140187, "resolvedEasyRate": 0.14893617021276595, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0.007462686567164179, "date": "2025-04-26", "path": "go/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 25, "resolvedEasy": 19, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.05841121495327103, "resolvedEasyRate": 0.1347517730496454, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0.014925373134328358, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 25, "resolvedEasy": 19, "resolvedMedium": 6, "resolvedHard": 0, "resolvedRate": 0.05841121495327103, "resolvedEasyRate": 0.1347517730496454, "resolvedMediumRate": 0.0392156862745098, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 24, "resolvedEasy": 17, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.056074766355140186, "resolvedEasyRate": 0.12056737588652482, "resolvedMediumRate": 0.0392156862745098, "resolvedHardRate": 0.007462686567164179, "date": "2025-04-26", "path": "go/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 23, "resolvedEasy": 15, "resolvedMedium": 7, "resolvedHard": 1, "resolvedRate": 0.053738317757009345, "resolvedEasyRate": 0.10638297872340426, "resolvedMediumRate": 0.0457516339869281, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 23, "resolvedEasy": 14, "resolvedMedium": 7, "resolvedHard": 2, "resolvedRate": 0.053738317757009345, "resolvedEasyRate": 0.09929078014184398, "resolvedMediumRate": 0.0457516339869281, "resolvedHardRate": 0.014925373134328358, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 22, "resolvedEasy": 16, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.0514018691588785, "resolvedEasyRate": 0.11347517730496454, "resolvedMediumRate": 0.032679738562091505, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 20, "resolvedEasy": 12, "resolvedMedium": 6, "resolvedHard": 2, "resolvedRate": 0.04672897196261682, "resolvedEasyRate": 0.0851063829787234, "resolvedMediumRate": 0.0392156862745098, "resolvedHardRate": 0.014925373134328358, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 19, "resolvedEasy": 13, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.04439252336448598, "resolvedEasyRate": 0.09219858156028368, "resolvedMediumRate": 0.032679738562091505, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 19, "resolvedEasy": 9, "resolvedMedium": 9, "resolvedHard": 1, "resolvedRate": 0.04439252336448598, "resolvedEasyRate": 0.06382978723404255, "resolvedMediumRate": 0.058823529411764705, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 19, "resolvedEasy": 13, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.04439252336448598, "resolvedEasyRate": 0.09219858156028368, "resolvedMediumRate": 0.032679738562091505, "resolvedHardRate": 0.007462686567164179, "date": "2025-04-26", "path": "go/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 18, "resolvedEasy": 14, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.04205607476635514, "resolvedEasyRate": 0.09929078014184398, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0, "date": "2025-04-26", "path": "go/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 17, "resolvedEasy": 13, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.0397196261682243, "resolvedEasyRate": 0.09219858156028368, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 17, "resolvedEasy": 14, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.0397196261682243, "resolvedEasyRate": 0.09929078014184398, "resolvedMediumRate": 0.013071895424836602, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 16, "resolvedEasy": 11, "resolvedMedium": 3, "resolvedHard": 2, "resolvedRate": 0.037383177570093455, "resolvedEasyRate": 0.07801418439716312, "resolvedMediumRate": 0.0196078431372549, "resolvedHardRate": 0.014925373134328358, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 16, "resolvedEasy": 12, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.037383177570093455, "resolvedEasyRate": 0.0851063829787234, "resolvedMediumRate": 0.0196078431372549, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 15, "resolvedEasy": 12, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.035046728971962614, "resolvedEasyRate": 0.0851063829787234, "resolvedMediumRate": 0.0196078431372549, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 15, "resolvedEasy": 11, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.035046728971962614, "resolvedEasyRate": 0.07801418439716312, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0, "date": "2025-04-26", "path": "go/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 12, "resolvedEasy": 9, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.028037383177570093, "resolvedEasyRate": 0.06382978723404255, "resolvedMediumRate": 0.0196078431372549, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 12, "resolvedEasy": 8, "resolvedMedium": 2, "resolvedHard": 2, "resolvedRate": 0.028037383177570093, "resolvedEasyRate": 0.05673758865248227, "resolvedMediumRate": 0.013071895424836602, "resolvedHardRate": 0.014925373134328358, "date": "2025-04-26", "path": "go/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 8, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.02336448598130841, "resolvedEasyRate": 0.05673758865248227, "resolvedMediumRate": 0.013071895424836602, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 10, "resolvedEasy": 7, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.02336448598130841, "resolvedEasyRate": 0.04964539007092199, "resolvedMediumRate": 0.0196078431372549, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 9, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.02102803738317757, "resolvedEasyRate": 0.06382978723404255, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 8, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.02102803738317757, "resolvedEasyRate": 0.05673758865248227, "resolvedMediumRate": 0.006535947712418301, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 9, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.02102803738317757, "resolvedEasyRate": 0.03546099290780142, "resolvedMediumRate": 0.026143790849673203, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 7, "resolvedEasy": 7, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.016355140186915886, "resolvedEasyRate": 0.04964539007092199, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 6, "resolvedEasy": 5, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.014018691588785047, "resolvedEasyRate": 0.03546099290780142, "resolvedMediumRate": 0.006535947712418301, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 6, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.014018691588785047, "resolvedEasyRate": 0.02127659574468085, "resolvedMediumRate": 0.013071895424836602, "resolvedHardRate": 0.007462686567164179, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 3, "resolvedEasy": 3, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.007009345794392523, "resolvedEasyRate": 0.02127659574468085, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 2, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.004672897196261682, "resolvedEasyRate": 0.014184397163120567, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "go/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "zeromicro__go-zero-2787", "zeromicro__go-zero-2537", "zeromicro__go-zero-2463", "zeromicro__go-zero-2363", "zeromicro__go-zero-2283", "zeromicro__go-zero-2116", "zeromicro__go-zero-2032", "zeromicro__go-zero-1969", "zeromicro__go-zero-1964", "zeromicro__go-zero-1907", "zeromicro__go-zero-1821", "zeromicro__go-zero-1783", "zeromicro__go-zero-1456", "zeromicro__go-zero-990", "zeromicro__go-zero-964", "grpc__grpc-go-3476", "grpc__grpc-go-3361", "grpc__grpc-go-3351", "grpc__grpc-go-3258", "grpc__grpc-go-3201", "grpc__grpc-go-3119", "grpc__grpc-go-2996", "grpc__grpc-go-2951", "grpc__grpc-go-2932", "grpc__grpc-go-2760", "grpc__grpc-go-2744", "grpc__grpc-go-2735", "grpc__grpc-go-2631", "grpc__grpc-go-2630", "grpc__grpc-go-2629", "grpc__grpc-go-2371", "cli__cli-10388", "cli__cli-10364", "cli__cli-10363", "cli__cli-10362", "cli__cli-10354", "cli__cli-10329", "cli__cli-10239", "cli__cli-10180", "cli__cli-10158", "cli__cli-10154", "cli__cli-10139", "cli__cli-10124", "cli__cli-10078", "cli__cli-10074", "cli__cli-10072", "cli__cli-10048", "cli__cli-10043", "cli__cli-10016", "cli__cli-10009", "cli__cli-9983", "cli__cli-9938", "cli__cli-9934", "cli__cli-9933", "cli__cli-9868", "cli__cli-9866", "cli__cli-9863", "cli__cli-9845", "cli__cli-9794", "cli__cli-9774", "cli__cli-9701", "cli__cli-9642", "cli__cli-9608", "cli__cli-9520", "cli__cli-9491", "cli__cli-9477", "cli__cli-9471", "cli__cli-9465", "cli__cli-9452", "cli__cli-9433", "cli__cli-9431", "cli__cli-9411", "cli__cli-9385", "cli__cli-9307", "cli__cli-9271", "cli__cli-9256", "cli__cli-9240", "cli__cli-9162", "cli__cli-9128", "cli__cli-9115", "cli__cli-9113", "cli__cli-9106", "cli__cli-9088", "cli__cli-9083", "cli__cli-9037", "cli__cli-9008", "cli__cli-8969", "cli__cli-8953", "cli__cli-8935", "cli__cli-8934", "cli__cli-8905", "cli__cli-8893", "cli__cli-8824", "cli__cli-8778", "cli__cli-8773", "cli__cli-8711", "cli__cli-8632", "cli__cli-8595", "cli__cli-8552", "cli__cli-8525", "cli__cli-8516", "cli__cli-8487", "cli__cli-8459", "cli__cli-8447", "cli__cli-8395", "cli__cli-8393", "cli__cli-8361", "cli__cli-8337", "cli__cli-8238", "cli__cli-8235", "cli__cli-8229", "cli__cli-8202", "cli__cli-8180", "cli__cli-8157", "cli__cli-8146", "cli__cli-8141", "cli__cli-8037", "cli__cli-8030", "cli__cli-7960", "cli__cli-7954", "cli__cli-7906", "cli__cli-7902", "cli__cli-7886", "cli__cli-7879", "cli__cli-7873", "cli__cli-7866", "cli__cli-7861", "cli__cli-7837", "cli__cli-7768", "cli__cli-7759", "cli__cli-7747", "cli__cli-7709", "cli__cli-7647", "cli__cli-7646", "cli__cli-7645", "cli__cli-7626", "cli__cli-7621", "cli__cli-7589", "cli__cli-7587", "cli__cli-7560", "cli__cli-7477", "cli__cli-7451", "cli__cli-7441", "cli__cli-7440", "cli__cli-7398", "cli__cli-7376", "cli__cli-7363", "cli__cli-7353", "cli__cli-7352", "cli__cli-7347", "cli__cli-7330", "cli__cli-7314", "cli__cli-7311", "cli__cli-7305", "cli__cli-7288", "cli__cli-7270", "cli__cli-7268", "cli__cli-7259", "cli__cli-7257", "cli__cli-7254", "cli__cli-7249", "cli__cli-7246", "cli__cli-7236", "cli__cli-7224", "cli__cli-7212", "cli__cli-7205", "cli__cli-7203", "cli__cli-7201", "cli__cli-7185", "cli__cli-7180", "cli__cli-7110", "cli__cli-7105", "cli__cli-7080", "cli__cli-7003", "cli__cli-6969", "cli__cli-6962", "cli__cli-6915", "cli__cli-6739", "cli__cli-6706", "cli__cli-6647", "cli__cli-6578", "cli__cli-6567", "cli__cli-6530", "cli__cli-6453", "cli__cli-6404", "cli__cli-6292", "cli__cli-6158", "cli__cli-6144", "cli__cli-6122", "cli__cli-6107", "cli__cli-6085", "cli__cli-6079", "cli__cli-6075", "cli__cli-6074", "cli__cli-6053", "cli__cli-6043", "cli__cli-5985", "cli__cli-5973", "cli__cli-5885", "cli__cli-5801", "cli__cli-5799", "cli__cli-5793", "cli__cli-5750", "cli__cli-5698", "cli__cli-5692", "cli__cli-5681", "cli__cli-5675", "cli__cli-5671", "cli__cli-5656", "cli__cli-5648", "cli__cli-5642", "cli__cli-5617", "cli__cli-5597", "cli__cli-5595", "cli__cli-5588", "cli__cli-5570", "cli__cli-5569", "cli__cli-5567", "cli__cli-5564", "cli__cli-5539", "cli__cli-5538", "cli__cli-5537", "cli__cli-5524", "cli__cli-5503", "cli__cli-5479", "cli__cli-5464", "cli__cli-5462", "cli__cli-5452", "cli__cli-5396", "cli__cli-5380", "cli__cli-5356", "cli__cli-5316", "cli__cli-5306", "cli__cli-5272", "cli__cli-5254", "cli__cli-5251", "cli__cli-5229", "cli__cli-5219", "cli__cli-5160", "cli__cli-5108", "cli__cli-5092", "cli__cli-5069", "cli__cli-5049", "cli__cli-5047", "cli__cli-5044", "cli__cli-5028", "cli__cli-5019", "cli__cli-4921", "cli__cli-4904", "cli__cli-4886", "cli__cli-4862", "cli__cli-4861", "cli__cli-4845", "cli__cli-4803", "cli__cli-4794", "cli__cli-4769", "cli__cli-4748", "cli__cli-4747", "cli__cli-4676", "cli__cli-4663", "cli__cli-4656", "cli__cli-4655", "cli__cli-4654", "cli__cli-4562", "cli__cli-4559", "cli__cli-4543", "cli__cli-4534", "cli__cli-4525", "cli__cli-4519", "cli__cli-4489", "cli__cli-4467", "cli__cli-4453", "cli__cli-4451", "cli__cli-4450", "cli__cli-4423", "cli__cli-4416", "cli__cli-4410", "cli__cli-4318", "cli__cli-4310", "cli__cli-4302", "cli__cli-4293", "cli__cli-4253", "cli__cli-4199", "cli__cli-4146", "cli__cli-4136", "cli__cli-4090", "cli__cli-4007", "cli__cli-4005", "cli__cli-4003", "cli__cli-3997", "cli__cli-3992", "cli__cli-3982", "cli__cli-3972", "cli__cli-3924", "cli__cli-3898", "cli__cli-3833", "cli__cli-3832", "cli__cli-3827", "cli__cli-3779", "cli__cli-3774", "cli__cli-3769", "cli__cli-3663", "cli__cli-3658", "cli__cli-3611", "cli__cli-3608", "cli__cli-3578", "cli__cli-3576", "cli__cli-3529", "cli__cli-3519", "cli__cli-3517", "cli__cli-3498", "cli__cli-3490", "cli__cli-3472", "cli__cli-3461", "cli__cli-3445", "cli__cli-3414", "cli__cli-3351", "cli__cli-3308", "cli__cli-3286", "cli__cli-3270", "cli__cli-3244", "cli__cli-3192", "cli__cli-2997", "cli__cli-2982", "cli__cli-2909", "cli__cli-2898", "cli__cli-2896", "cli__cli-2892", "cli__cli-2888", "cli__cli-2843", "cli__cli-2785", "cli__cli-2671", "cli__cli-2519", "cli__cli-2466", "cli__cli-2404", "cli__cli-2351", "cli__cli-2316", "cli__cli-2301", "cli__cli-2282", "cli__cli-2279", "cli__cli-2265", "cli__cli-2264", "cli__cli-2263", "cli__cli-2255", "cli__cli-2251", "cli__cli-2250", "cli__cli-2249", "cli__cli-2239", "cli__cli-2224", "cli__cli-2222", "cli__cli-2221", "cli__cli-2209", "cli__cli-2157", "cli__cli-2146", "cli__cli-2138", "cli__cli-2108", "cli__cli-2058", "cli__cli-2034", "cli__cli-2004", "cli__cli-1986", "cli__cli-1893", "cli__cli-1882", "cli__cli-1867", "cli__cli-1766", "cli__cli-1763", "cli__cli-1642", "cli__cli-1639", "cli__cli-1582", "cli__cli-1580", "cli__cli-1567", "cli__cli-1534", "cli__cli-1494", "cli__cli-1479", "cli__cli-1476", "cli__cli-1462", "cli__cli-1457", "cli__cli-1388", "cli__cli-1387", "cli__cli-1373", "cli__cli-1366", "cli__cli-1363", "cli__cli-1359", "cli__cli-1343", "cli__cli-1337", "cli__cli-1283", "cli__cli-1282", "cli__cli-1279", "cli__cli-1155", "cli__cli-1103", "cli__cli-1069", "cli__cli-992", "cli__cli-981", "cli__cli-970", "cli__cli-962", "cli__cli-885", "cli__cli-879", "cli__cli-869", "cli__cli-864", "cli__cli-857", "cli__cli-842", "cli__cli-810", "cli__cli-728", "cli__cli-727", "cli__cli-708", "cli__cli-696", "cli__cli-690", "cli__cli-683", "cli__cli-668", "cli__cli-667", "cli__cli-665", "cli__cli-638", "cli__cli-625", "cli__cli-624", "cli__cli-617", "cli__cli-590", "cli__cli-549", "cli__cli-545", "cli__cli-538", "cli__cli-513", "cli__cli-495", "cli__cli-477", "cli__cli-474", "cli__cli-405", "cli__cli-402", "cli__cli-362", "cli__cli-352", "cli__cli-302", "cli__cli-278" ], "easy_ids": [ "zeromicro__go-zero-2787", "grpc__grpc-go-3351", "grpc__grpc-go-3201", "grpc__grpc-go-2996", "grpc__grpc-go-2932", "grpc__grpc-go-2371", "cli__cli-10388", "cli__cli-10364", "cli__cli-10363", "cli__cli-10329", "cli__cli-10158", "cli__cli-10074", "cli__cli-10072", "cli__cli-10048", "cli__cli-10043", "cli__cli-10016", "cli__cli-9983", "cli__cli-9938", "cli__cli-9863", "cli__cli-9794", "cli__cli-9774", "cli__cli-9642", "cli__cli-9608", "cli__cli-9520", "cli__cli-9491", "cli__cli-9477", "cli__cli-9471", "cli__cli-9431", "cli__cli-9271", "cli__cli-9256", "cli__cli-9115", "cli__cli-9088", "cli__cli-9008", "cli__cli-8969", "cli__cli-8935", "cli__cli-8905", "cli__cli-8893", "cli__cli-8632", "cli__cli-8525", "cli__cli-8459", "cli__cli-8447", "cli__cli-8361", "cli__cli-8337", "cli__cli-8238", "cli__cli-8235", "cli__cli-8229", "cli__cli-8202", "cli__cli-8180", "cli__cli-8141", "cli__cli-8037", "cli__cli-8030", "cli__cli-7954", "cli__cli-7906", "cli__cli-7886", "cli__cli-7879", "cli__cli-7873", "cli__cli-7866", "cli__cli-7837", "cli__cli-7759", "cli__cli-7621", "cli__cli-7363", "cli__cli-7352", "cli__cli-7249", "cli__cli-7236", "cli__cli-7224", "cli__cli-7180", "cli__cli-7110", "cli__cli-6915", "cli__cli-6739", "cli__cli-6706", "cli__cli-6647", "cli__cli-6453", "cli__cli-6144", "cli__cli-6079", "cli__cli-5985", "cli__cli-5973", "cli__cli-5885", "cli__cli-5801", "cli__cli-5793", "cli__cli-5750", "cli__cli-5692", "cli__cli-5675", "cli__cli-5656", "cli__cli-5595", "cli__cli-5538", "cli__cli-5396", "cli__cli-5356", "cli__cli-5254", "cli__cli-5229", "cli__cli-5219", "cli__cli-5092", "cli__cli-5047", "cli__cli-5044", "cli__cli-5028", "cli__cli-4921", "cli__cli-4676", "cli__cli-4253", "cli__cli-4199", "cli__cli-4136", "cli__cli-4090", "cli__cli-4005", "cli__cli-4003", "cli__cli-3982", "cli__cli-3924", "cli__cli-3827", "cli__cli-3663", "cli__cli-3658", "cli__cli-3611", "cli__cli-3576", "cli__cli-3529", "cli__cli-3517", "cli__cli-3472", "cli__cli-3445", "cli__cli-2843", "cli__cli-2519", "cli__cli-2466", "cli__cli-2351", "cli__cli-2301", "cli__cli-2282", "cli__cli-2279", "cli__cli-2263", "cli__cli-2255", "cli__cli-2251", "cli__cli-2250", "cli__cli-2249", "cli__cli-2209", "cli__cli-1580", "cli__cli-879", "cli__cli-668", "cli__cli-638", "cli__cli-625", "cli__cli-624", "cli__cli-617", "cli__cli-513", "cli__cli-495", "cli__cli-477", "cli__cli-474", "cli__cli-405", "cli__cli-402", "cli__cli-362", "cli__cli-278" ], "medium_ids": [ "zeromicro__go-zero-2363", "zeromicro__go-zero-2283", "zeromicro__go-zero-2116", "zeromicro__go-zero-2032", "zeromicro__go-zero-1969", "zeromicro__go-zero-1907", "zeromicro__go-zero-1783", "grpc__grpc-go-3476", "grpc__grpc-go-3361", "grpc__grpc-go-3258", "grpc__grpc-go-3119", "grpc__grpc-go-2744", "grpc__grpc-go-2735", "grpc__grpc-go-2631", "grpc__grpc-go-2630", "cli__cli-10239", "cli__cli-10078", "cli__cli-9933", "cli__cli-9866", "cli__cli-9845", "cli__cli-9701", "cli__cli-9465", "cli__cli-9452", "cli__cli-9385", "cli__cli-9240", "cli__cli-9113", "cli__cli-8824", "cli__cli-8773", "cli__cli-8711", "cli__cli-8552", "cli__cli-8487", "cli__cli-8395", "cli__cli-8146", "cli__cli-7960", "cli__cli-7902", "cli__cli-7861", "cli__cli-7768", "cli__cli-7747", "cli__cli-7709", "cli__cli-7647", "cli__cli-7646", "cli__cli-7645", "cli__cli-7626", "cli__cli-7589", "cli__cli-7587", "cli__cli-7560", "cli__cli-7451", "cli__cli-7441", "cli__cli-7440", "cli__cli-7398", "cli__cli-7353", "cli__cli-7347", "cli__cli-7330", "cli__cli-7314", "cli__cli-7288", "cli__cli-7270", "cli__cli-7268", "cli__cli-7254", "cli__cli-7246", "cli__cli-7203", "cli__cli-6969", "cli__cli-6962", "cli__cli-6122", "cli__cli-6075", "cli__cli-6074", "cli__cli-6053", "cli__cli-5799", "cli__cli-5698", "cli__cli-5671", "cli__cli-5642", "cli__cli-5617", "cli__cli-5570", "cli__cli-5569", "cli__cli-5567", "cli__cli-5564", "cli__cli-5539", "cli__cli-5537", "cli__cli-5524", "cli__cli-5464", "cli__cli-5462", "cli__cli-5380", "cli__cli-5306", "cli__cli-5251", "cli__cli-5160", "cli__cli-5069", "cli__cli-5049", "cli__cli-5019", "cli__cli-4904", "cli__cli-4886", "cli__cli-4862", "cli__cli-4861", "cli__cli-4845", "cli__cli-4769", "cli__cli-4748", "cli__cli-4747", "cli__cli-4663", "cli__cli-4654", "cli__cli-4562", "cli__cli-4559", "cli__cli-4543", "cli__cli-4534", "cli__cli-4525", "cli__cli-4489", "cli__cli-4423", "cli__cli-4293", "cli__cli-4146", "cli__cli-4007", "cli__cli-3992", "cli__cli-3898", "cli__cli-3832", "cli__cli-3779", "cli__cli-3774", "cli__cli-3769", "cli__cli-3608", "cli__cli-3578", "cli__cli-3490", "cli__cli-3461", "cli__cli-3351", "cli__cli-3308", "cli__cli-3270", "cli__cli-2982", "cli__cli-2909", "cli__cli-2898", "cli__cli-2785", "cli__cli-2671", "cli__cli-2404", "cli__cli-2264", "cli__cli-2239", "cli__cli-2157", "cli__cli-2146", "cli__cli-2138", "cli__cli-2058", "cli__cli-2034", "cli__cli-1893", "cli__cli-1882", "cli__cli-1766", "cli__cli-1567", "cli__cli-1494", "cli__cli-1457", "cli__cli-1366", "cli__cli-1337", "cli__cli-1283", "cli__cli-1155", "cli__cli-857", "cli__cli-842", "cli__cli-727", "cli__cli-708", "cli__cli-683", "cli__cli-665", "cli__cli-545", "cli__cli-538", "cli__cli-352", "cli__cli-302" ], "hard_ids": [ "zeromicro__go-zero-2537", "zeromicro__go-zero-2463", "zeromicro__go-zero-1964", "zeromicro__go-zero-1821", "zeromicro__go-zero-1456", "zeromicro__go-zero-990", "zeromicro__go-zero-964", "grpc__grpc-go-2951", "grpc__grpc-go-2760", "grpc__grpc-go-2629", "cli__cli-10362", "cli__cli-10354", "cli__cli-10180", "cli__cli-10154", "cli__cli-10139", "cli__cli-10124", "cli__cli-10009", "cli__cli-9934", "cli__cli-9868", "cli__cli-9433", "cli__cli-9411", "cli__cli-9307", "cli__cli-9162", "cli__cli-9128", "cli__cli-9106", "cli__cli-9083", "cli__cli-9037", "cli__cli-8953", "cli__cli-8934", "cli__cli-8778", "cli__cli-8595", "cli__cli-8516", "cli__cli-8393", "cli__cli-8157", "cli__cli-7477", "cli__cli-7376", "cli__cli-7311", "cli__cli-7305", "cli__cli-7259", "cli__cli-7257", "cli__cli-7212", "cli__cli-7205", "cli__cli-7201", "cli__cli-7185", "cli__cli-7105", "cli__cli-7080", "cli__cli-7003", "cli__cli-6578", "cli__cli-6567", "cli__cli-6530", "cli__cli-6404", "cli__cli-6292", "cli__cli-6158", "cli__cli-6107", "cli__cli-6085", "cli__cli-6043", "cli__cli-5681", "cli__cli-5648", "cli__cli-5597", "cli__cli-5588", "cli__cli-5503", "cli__cli-5479", "cli__cli-5452", "cli__cli-5316", "cli__cli-5272", "cli__cli-5108", "cli__cli-4803", "cli__cli-4794", "cli__cli-4656", "cli__cli-4655", "cli__cli-4519", "cli__cli-4467", "cli__cli-4453", "cli__cli-4451", "cli__cli-4450", "cli__cli-4416", "cli__cli-4410", "cli__cli-4318", "cli__cli-4310", "cli__cli-4302", "cli__cli-3997", "cli__cli-3972", "cli__cli-3833", "cli__cli-3519", "cli__cli-3498", "cli__cli-3414", "cli__cli-3286", "cli__cli-3244", "cli__cli-3192", "cli__cli-2997", "cli__cli-2896", "cli__cli-2892", "cli__cli-2888", "cli__cli-2316", "cli__cli-2265", "cli__cli-2224", "cli__cli-2222", "cli__cli-2221", "cli__cli-2108", "cli__cli-2004", "cli__cli-1986", "cli__cli-1867", "cli__cli-1763", "cli__cli-1642", "cli__cli-1639", "cli__cli-1582", "cli__cli-1534", "cli__cli-1479", "cli__cli-1476", "cli__cli-1462", "cli__cli-1388", "cli__cli-1387", "cli__cli-1373", "cli__cli-1363", "cli__cli-1359", "cli__cli-1343", "cli__cli-1282", "cli__cli-1279", "cli__cli-1103", "cli__cli-1069", "cli__cli-992", "cli__cli-981", "cli__cli-970", "cli__cli-962", "cli__cli-885", "cli__cli-869", "cli__cli-864", "cli__cli-810", "cli__cli-728", "cli__cli-696", "cli__cli-690", "cli__cli-667", "cli__cli-590", "cli__cli-549" ] } } ] }, { "name": "Rust", "data": [ { "name": "Verified", "results": [ { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 38, "resolvedEasy": 14, "resolvedMedium": 17, "resolvedHard": 7, "resolvedRate": 0.1589958158995816, "resolvedEasyRate": 0.21212121212121213, "resolvedMediumRate": 0.1349206349206349, "resolvedHardRate": 0.14893617021276595, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 35, "resolvedEasy": 21, "resolvedMedium": 10, "resolvedHard": 4, "resolvedRate": 0.14644351464435146, "resolvedEasyRate": 0.3181818181818182, "resolvedMediumRate": 0.07936507936507936, "resolvedHardRate": 0.0851063829787234, "date": "2025-04-26", "path": "rust/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 29, "resolvedEasy": 16, "resolvedMedium": 9, "resolvedHard": 4, "resolvedRate": 0.12133891213389121, "resolvedEasyRate": 0.24242424242424243, "resolvedMediumRate": 0.07142857142857142, "resolvedHardRate": 0.0851063829787234, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 24, "resolvedEasy": 10, "resolvedMedium": 11, "resolvedHard": 3, "resolvedRate": 0.100418410041841, "resolvedEasyRate": 0.15151515151515152, "resolvedMediumRate": 0.0873015873015873, "resolvedHardRate": 0.06382978723404255, "date": "2025-04-26", "path": "rust/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 19, "resolvedEasy": 15, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.0794979079497908, "resolvedEasyRate": 0.22727272727272727, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 17, "resolvedEasy": 11, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.07112970711297072, "resolvedEasyRate": 0.16666666666666666, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 17, "resolvedEasy": 13, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.07112970711297072, "resolvedEasyRate": 0.19696969696969696, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0, "date": "2025-04-26", "path": "rust/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 16, "resolvedEasy": 9, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.06694560669456066, "resolvedEasyRate": 0.13636363636363635, "resolvedMediumRate": 0.047619047619047616, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 16, "resolvedEasy": 9, "resolvedMedium": 3, "resolvedHard": 4, "resolvedRate": 0.06694560669456066, "resolvedEasyRate": 0.13636363636363635, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0.0851063829787234, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 16, "resolvedEasy": 10, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.06694560669456066, "resolvedEasyRate": 0.15151515151515152, "resolvedMediumRate": 0.03968253968253968, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 15, "resolvedEasy": 8, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.06276150627615062, "resolvedEasyRate": 0.12121212121212122, "resolvedMediumRate": 0.047619047619047616, "resolvedHardRate": 0.02127659574468085, "date": "2025-04-26", "path": "rust/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 14, "resolvedEasy": 7, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.058577405857740586, "resolvedEasyRate": 0.10606060606060606, "resolvedMediumRate": 0.047619047619047616, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 14, "resolvedEasy": 9, "resolvedMedium": 2, "resolvedHard": 3, "resolvedRate": 0.058577405857740586, "resolvedEasyRate": 0.13636363636363635, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0.06382978723404255, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 13, "resolvedEasy": 7, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.05439330543933055, "resolvedEasyRate": 0.10606060606060606, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 13, "resolvedEasy": 8, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.05439330543933055, "resolvedEasyRate": 0.12121212121212122, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0.02127659574468085, "date": "2025-04-26", "path": "rust/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 12, "resolvedEasy": 8, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.0502092050209205, "resolvedEasyRate": 0.12121212121212122, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 12, "resolvedEasy": 6, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.0502092050209205, "resolvedEasyRate": 0.09090909090909091, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 12, "resolvedEasy": 10, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.0502092050209205, "resolvedEasyRate": 0.15151515151515152, "resolvedMediumRate": 0.007936507936507936, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 9, "resolvedMedium": 1, "resolvedHard": 2, "resolvedRate": 0.0502092050209205, "resolvedEasyRate": 0.13636363636363635, "resolvedMediumRate": 0.007936507936507936, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 3, "resolvedRate": 0.0502092050209205, "resolvedEasyRate": 0.07575757575757576, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0.06382978723404255, "date": "2025-04-26", "path": "rust/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 11, "resolvedEasy": 9, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.04602510460251046, "resolvedEasyRate": 0.13636363636363635, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 11, "resolvedEasy": 4, "resolvedMedium": 5, "resolvedHard": 2, "resolvedRate": 0.04602510460251046, "resolvedEasyRate": 0.06060606060606061, "resolvedMediumRate": 0.03968253968253968, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 4, "resolvedMedium": 5, "resolvedHard": 1, "resolvedRate": 0.04184100418410042, "resolvedEasyRate": 0.06060606060606061, "resolvedMediumRate": 0.03968253968253968, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 10, "resolvedEasy": 7, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.04184100418410042, "resolvedEasyRate": 0.10606060606060606, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 8, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.03347280334728033, "resolvedEasyRate": 0.06060606060606061, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.03347280334728033, "resolvedEasyRate": 0.07575757575757576, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0.02127659574468085, "date": "2025-04-26", "path": "rust/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.03347280334728033, "resolvedEasyRate": 0.07575757575757576, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0.02127659574468085, "date": "2025-04-26", "path": "rust/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 6, "resolvedEasy": 4, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.02510460251046025, "resolvedEasyRate": 0.06060606060606061, "resolvedMediumRate": 0.015873015873015872, "resolvedHardRate": 0, "date": "2025-03-29", "path": "rust/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 6, "resolvedEasy": 2, "resolvedMedium": 3, "resolvedHard": 1, "resolvedRate": 0.02510460251046025, "resolvedEasyRate": 0.030303030303030304, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 6, "resolvedEasy": 3, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.02510460251046025, "resolvedEasyRate": 0.045454545454545456, "resolvedMediumRate": 0.023809523809523808, "resolvedHardRate": 0, "date": "2025-04-26", "path": "rust/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.02092050209205021, "resolvedEasyRate": 0.045454545454545456, "resolvedMediumRate": 0.007936507936507936, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 1, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.02092050209205021, "resolvedEasyRate": 0.015151515151515152, "resolvedMediumRate": 0.031746031746031744, "resolvedHardRate": 0, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 4, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 2, "resolvedRate": 0.016736401673640166, "resolvedEasyRate": 0.015151515151515152, "resolvedMediumRate": 0.007936507936507936, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 2, "resolvedRate": 0.016736401673640166, "resolvedEasyRate": 0.015151515151515152, "resolvedMediumRate": 0.007936507936507936, "resolvedHardRate": 0.0425531914893617, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 2, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 1, "resolvedRate": 0.008368200836820083, "resolvedEasyRate": 0.015151515151515152, "resolvedMediumRate": 0, "resolvedHardRate": 0.02127659574468085, "date": "2025-03-29", "path": "rust/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.0041841004184100415, "resolvedEasyRate": 0.015151515151515152, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "rust/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "clap-rs__clap-5873", "clap-rs__clap-5527", "clap-rs__clap-5520", "clap-rs__clap-5489", "clap-rs__clap-5298", "clap-rs__clap-5228", "clap-rs__clap-5227", "clap-rs__clap-5206", "clap-rs__clap-5084", "clap-rs__clap-5080", "clap-rs__clap-5075", "clap-rs__clap-5025", "clap-rs__clap-5015", "clap-rs__clap-4909", "clap-rs__clap-4902", "clap-rs__clap-4667", "clap-rs__clap-4635", "clap-rs__clap-4601", "clap-rs__clap-4573", "clap-rs__clap-4523", "clap-rs__clap-4474", "clap-rs__clap-4465", "clap-rs__clap-4408", "clap-rs__clap-4248", "clap-rs__clap-4159", "clap-rs__clap-4151", "clap-rs__clap-4103", "clap-rs__clap-4101", "clap-rs__clap-4081", "clap-rs__clap-4080", "clap-rs__clap-4077", "clap-rs__clap-4072", "clap-rs__clap-4059", "clap-rs__clap-4056", "clap-rs__clap-4039", "clap-rs__clap-4032", "clap-rs__clap-4006", "clap-rs__clap-4005", "clap-rs__clap-3990", "clap-rs__clap-3989", "clap-rs__clap-3975", "clap-rs__clap-3974", "clap-rs__clap-3972", "clap-rs__clap-3968", "clap-rs__clap-3960", "clap-rs__clap-3874", "clap-rs__clap-3775", "clap-rs__clap-3732", "clap-rs__clap-3700", "clap-rs__clap-3684", "clap-rs__clap-3678", "clap-rs__clap-3670", "clap-rs__clap-3521", "clap-rs__clap-3455", "clap-rs__clap-3453", "clap-rs__clap-3423", "clap-rs__clap-3421", "clap-rs__clap-3420", "clap-rs__clap-3402", "clap-rs__clap-3394", "clap-rs__clap-3391", "clap-rs__clap-3334", "clap-rs__clap-3311", "clap-rs__clap-3225", "clap-rs__clap-3212", "clap-rs__clap-3196", "clap-rs__clap-3179", "clap-rs__clap-3169", "clap-rs__clap-3023", "clap-rs__clap-3020", "clap-rs__clap-2993", "clap-rs__clap-2990", "clap-rs__clap-2930", "clap-rs__clap-2895", "clap-rs__clap-2882", "clap-rs__clap-2878", "clap-rs__clap-2867", "clap-rs__clap-2804", "clap-rs__clap-2796", "clap-rs__clap-2773", "clap-rs__clap-2771", "clap-rs__clap-2758", "clap-rs__clap-2749", "clap-rs__clap-2730", "clap-rs__clap-2701", "clap-rs__clap-2696", "clap-rs__clap-2677", "clap-rs__clap-2669", "clap-rs__clap-2653", "clap-rs__clap-2648", "clap-rs__clap-2643", "clap-rs__clap-2642", "clap-rs__clap-2635", "clap-rs__clap-2633", "clap-rs__clap-2611", "clap-rs__clap-2609", "clap-rs__clap-2587", "clap-rs__clap-2543", "clap-rs__clap-2534", "clap-rs__clap-2529", "clap-rs__clap-2501", "clap-rs__clap-2400", "clap-rs__clap-2360", "clap-rs__clap-2358", "clap-rs__clap-2329", "clap-rs__clap-2310", "clap-rs__clap-2309", "clap-rs__clap-2297", "clap-rs__clap-2296", "clap-rs__clap-2253", "clap-rs__clap-2251", "clap-rs__clap-2203", "clap-rs__clap-2190", "clap-rs__clap-2182", "clap-rs__clap-2168", "clap-rs__clap-2166", "clap-rs__clap-2161", "clap-rs__clap-2154", "clap-rs__clap-2135", "clap-rs__clap-2093", "clap-rs__clap-2085", "clap-rs__clap-2075", "clap-rs__clap-2058", "clap-rs__clap-2027", "clap-rs__clap-2008", "clap-rs__clap-1975", "clap-rs__clap-1972", "clap-rs__clap-1958", "clap-rs__clap-1869", "clap-rs__clap-1710", "clap-rs__clap-1624", "clap-rs__clap-941", "serde-rs__serde-2798", "serde-rs__serde-2709", "BurntSushi__ripgrep-2626", "BurntSushi__ripgrep-2610", "BurntSushi__ripgrep-2576", "BurntSushi__ripgrep-2488", "BurntSushi__ripgrep-2295", "BurntSushi__ripgrep-2209", "BurntSushi__ripgrep-1980", "BurntSushi__ripgrep-1642", "BurntSushi__ripgrep-1367", "BurntSushi__ripgrep-1294", "BurntSushi__ripgrep-954", "BurntSushi__ripgrep-727", "BurntSushi__ripgrep-723", "BurntSushi__ripgrep-454", "tokio-rs__bytes-732", "tokio-rs__bytes-721", "tokio-rs__bytes-643", "tokio-rs__bytes-547", "tokio-rs__bytes-543", "tokio-rs__tracing-2897", "tokio-rs__tracing-2892", "tokio-rs__tracing-2883", "tokio-rs__tracing-2335", "tokio-rs__tracing-2090", "tokio-rs__tracing-2027", "tokio-rs__tracing-2008", "tokio-rs__tracing-2001", "tokio-rs__tracing-1983", "tokio-rs__tracing-1853", "tokio-rs__tracing-1619", "tokio-rs__tracing-1523", "tokio-rs__tracing-1297", "tokio-rs__tracing-1291", "tokio-rs__tracing-1252", "tokio-rs__tracing-1236", "tokio-rs__tracing-1233", "tokio-rs__tracing-1228", "tokio-rs__tracing-1045", "tokio-rs__tracing-1017", "tokio-rs__tracing-1015", "sharkdp__bat-3189", "sharkdp__bat-3108", "sharkdp__bat-3075", "sharkdp__bat-2896", "sharkdp__bat-2665", "sharkdp__bat-2544", "sharkdp__bat-1518", "sharkdp__bat-1402", "sharkdp__bat-1276", "sharkdp__bat-1197", "rayon-rs__rayon-986", "rayon-rs__rayon-863", "sharkdp__fd-1394", "sharkdp__fd-1162", "sharkdp__fd-1121", "sharkdp__fd-1079", "sharkdp__fd-986", "sharkdp__fd-866", "sharkdp__fd-813", "sharkdp__fd-658", "sharkdp__fd-590", "sharkdp__fd-569", "sharkdp__fd-558", "sharkdp__fd-555", "sharkdp__fd-546", "sharkdp__fd-497", "tokio-rs__tokio-7124", "tokio-rs__tokio-7111", "tokio-rs__tokio-6967", "tokio-rs__tokio-6742", "tokio-rs__tokio-6618", "tokio-rs__tokio-6462", "tokio-rs__tokio-6409", "tokio-rs__tokio-6345", "tokio-rs__tokio-6339", "tokio-rs__tokio-6290", "tokio-rs__tokio-6280", "tokio-rs__tokio-6252", "tokio-rs__tokio-6205", "tokio-rs__tokio-5781", "tokio-rs__tokio-5583", "tokio-rs__tokio-5566", "tokio-rs__tokio-5520", "tokio-rs__tokio-5343", "tokio-rs__tokio-5244", "tokio-rs__tokio-5200", "tokio-rs__tokio-5179", "tokio-rs__tokio-5144", "tokio-rs__tokio-4789", "tokio-rs__tokio-4613", "tokio-rs__tokio-4519", "nushell__nushell-13870", "nushell__nushell-13357", "nushell__nushell-12901", "nushell__nushell-12118", "nushell__nushell-11948", "nushell__nushell-11672", "nushell__nushell-11493", "nushell__nushell-11292", "nushell__nushell-11169", "nushell__nushell-10629", "nushell__nushell-10613", "nushell__nushell-10405", "nushell__nushell-10395", "nushell__nushell-10381" ], "easy_ids": [ "clap-rs__clap-5873", "clap-rs__clap-5527", "clap-rs__clap-5520", "clap-rs__clap-5084", "clap-rs__clap-4909", "clap-rs__clap-4902", "clap-rs__clap-4667", "clap-rs__clap-4601", "clap-rs__clap-4465", "clap-rs__clap-4101", "clap-rs__clap-4039", "clap-rs__clap-3974", "clap-rs__clap-3960", "clap-rs__clap-3421", "clap-rs__clap-3391", "clap-rs__clap-3334", "clap-rs__clap-3311", "clap-rs__clap-3196", "clap-rs__clap-3179", "clap-rs__clap-2993", "clap-rs__clap-2930", "clap-rs__clap-2878", "clap-rs__clap-2804", "clap-rs__clap-2773", "clap-rs__clap-2771", "clap-rs__clap-2730", "clap-rs__clap-2696", "clap-rs__clap-2669", "clap-rs__clap-2653", "clap-rs__clap-2648", "clap-rs__clap-2642", "clap-rs__clap-2529", "clap-rs__clap-2085", "clap-rs__clap-2027", "clap-rs__clap-1972", "clap-rs__clap-1710", "clap-rs__clap-941", "serde-rs__serde-2798", "BurntSushi__ripgrep-2626", "BurntSushi__ripgrep-2576", "BurntSushi__ripgrep-2488", "BurntSushi__ripgrep-954", "BurntSushi__ripgrep-727", "BurntSushi__ripgrep-454", "tokio-rs__bytes-543", "tokio-rs__tracing-2090", "tokio-rs__tracing-1619", "tokio-rs__tracing-1236", "tokio-rs__tracing-1233", "sharkdp__bat-3108", "sharkdp__bat-3075", "rayon-rs__rayon-986", "sharkdp__fd-569", "sharkdp__fd-555", "tokio-rs__tokio-6462", "tokio-rs__tokio-6339", "tokio-rs__tokio-6252", "tokio-rs__tokio-5244", "tokio-rs__tokio-5144", "tokio-rs__tokio-4519", "nushell__nushell-12901", "nushell__nushell-12118", "nushell__nushell-11948", "nushell__nushell-11493", "nushell__nushell-11292", "nushell__nushell-10629" ], "medium_ids": [ "clap-rs__clap-5489", "clap-rs__clap-5298", "clap-rs__clap-5228", "clap-rs__clap-5227", "clap-rs__clap-5206", "clap-rs__clap-5080", "clap-rs__clap-5075", "clap-rs__clap-5025", "clap-rs__clap-5015", "clap-rs__clap-4635", "clap-rs__clap-4573", "clap-rs__clap-4523", "clap-rs__clap-4474", "clap-rs__clap-4408", "clap-rs__clap-4248", "clap-rs__clap-4159", "clap-rs__clap-4081", "clap-rs__clap-4059", "clap-rs__clap-4006", "clap-rs__clap-4005", "clap-rs__clap-3990", "clap-rs__clap-3989", "clap-rs__clap-3975", "clap-rs__clap-3972", "clap-rs__clap-3968", "clap-rs__clap-3874", "clap-rs__clap-3775", "clap-rs__clap-3700", "clap-rs__clap-3684", "clap-rs__clap-3670", "clap-rs__clap-3521", "clap-rs__clap-3455", "clap-rs__clap-3453", "clap-rs__clap-3423", "clap-rs__clap-3420", "clap-rs__clap-3394", "clap-rs__clap-3225", "clap-rs__clap-3169", "clap-rs__clap-3023", "clap-rs__clap-3020", "clap-rs__clap-2990", "clap-rs__clap-2895", "clap-rs__clap-2882", "clap-rs__clap-2796", "clap-rs__clap-2749", "clap-rs__clap-2701", "clap-rs__clap-2643", "clap-rs__clap-2635", "clap-rs__clap-2633", "clap-rs__clap-2611", "clap-rs__clap-2609", "clap-rs__clap-2587", "clap-rs__clap-2534", "clap-rs__clap-2501", "clap-rs__clap-2310", "clap-rs__clap-2309", "clap-rs__clap-2297", "clap-rs__clap-2296", "clap-rs__clap-2253", "clap-rs__clap-2251", "clap-rs__clap-2203", "clap-rs__clap-2182", "clap-rs__clap-2168", "clap-rs__clap-2166", "clap-rs__clap-2154", "clap-rs__clap-2135", "clap-rs__clap-2093", "clap-rs__clap-2075", "clap-rs__clap-2058", "clap-rs__clap-2008", "clap-rs__clap-1975", "clap-rs__clap-1958", "clap-rs__clap-1869", "clap-rs__clap-1624", "serde-rs__serde-2709", "BurntSushi__ripgrep-2295", "BurntSushi__ripgrep-2209", "BurntSushi__ripgrep-1980", "BurntSushi__ripgrep-1642", "BurntSushi__ripgrep-1367", "BurntSushi__ripgrep-1294", "BurntSushi__ripgrep-723", "tokio-rs__bytes-732", "tokio-rs__bytes-721", "tokio-rs__tracing-2897", "tokio-rs__tracing-2892", "tokio-rs__tracing-2883", "tokio-rs__tracing-2001", "tokio-rs__tracing-1983", "tokio-rs__tracing-1853", "tokio-rs__tracing-1297", "tokio-rs__tracing-1252", "tokio-rs__tracing-1045", "sharkdp__bat-3189", "sharkdp__bat-2896", "sharkdp__bat-2665", "sharkdp__bat-2544", "sharkdp__bat-1518", "sharkdp__bat-1402", "sharkdp__bat-1276", "sharkdp__bat-1197", "rayon-rs__rayon-863", "sharkdp__fd-1394", "sharkdp__fd-1162", "sharkdp__fd-1121", "sharkdp__fd-986", "sharkdp__fd-813", "sharkdp__fd-658", "sharkdp__fd-590", "sharkdp__fd-558", "sharkdp__fd-546", "sharkdp__fd-497", "tokio-rs__tokio-6967", "tokio-rs__tokio-6345", "tokio-rs__tokio-5583", "tokio-rs__tokio-5566", "tokio-rs__tokio-5520", "tokio-rs__tokio-5343", "tokio-rs__tokio-4789", "tokio-rs__tokio-4613", "nushell__nushell-13870", "nushell__nushell-11672", "nushell__nushell-11169", "nushell__nushell-10613", "nushell__nushell-10405", "nushell__nushell-10395" ], "hard_ids": [ "clap-rs__clap-4151", "clap-rs__clap-4103", "clap-rs__clap-4080", "clap-rs__clap-4077", "clap-rs__clap-4072", "clap-rs__clap-4056", "clap-rs__clap-4032", "clap-rs__clap-3732", "clap-rs__clap-3678", "clap-rs__clap-3402", "clap-rs__clap-3212", "clap-rs__clap-2867", "clap-rs__clap-2758", "clap-rs__clap-2677", "clap-rs__clap-2543", "clap-rs__clap-2400", "clap-rs__clap-2360", "clap-rs__clap-2358", "clap-rs__clap-2329", "clap-rs__clap-2190", "clap-rs__clap-2161", "BurntSushi__ripgrep-2610", "tokio-rs__bytes-643", "tokio-rs__bytes-547", "tokio-rs__tracing-2335", "tokio-rs__tracing-2027", "tokio-rs__tracing-2008", "tokio-rs__tracing-1523", "tokio-rs__tracing-1291", "tokio-rs__tracing-1228", "tokio-rs__tracing-1017", "tokio-rs__tracing-1015", "sharkdp__fd-1079", "sharkdp__fd-866", "tokio-rs__tokio-7124", "tokio-rs__tokio-7111", "tokio-rs__tokio-6742", "tokio-rs__tokio-6618", "tokio-rs__tokio-6409", "tokio-rs__tokio-6290", "tokio-rs__tokio-6280", "tokio-rs__tokio-6205", "tokio-rs__tokio-5781", "tokio-rs__tokio-5200", "tokio-rs__tokio-5179", "nushell__nushell-13357", "nushell__nushell-10381" ] } } ] }, { "name": "C", "data": [ { "name": "Verified", "results": [ { "name": "RepoRepair + Claude-4-Sonnet", "site": "https://github.com/JXTZZ/M-RepoRepair", "oss": true, "verified": false, "resolved": 14, "resolvedEasy": 5, "resolvedMedium": 6, "resolvedHard": 3, "resolvedRate": 0.109375, "resolvedEasyRate": 0.16666666666666666, "resolvedMediumRate": 0.1111111111111111, "resolvedHardRate": 0.06818181818181818, "date": "2025-12-26", "path": "c/verified/20251226_RepoRepair_Claude-4-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 12, "resolvedEasy": 4, "resolvedMedium": 8, "resolvedHard": 0, "resolvedRate": 0.09375, "resolvedEasyRate": 0.13333333333333333, "resolvedMediumRate": 0.14814814814814814, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 12, "resolvedEasy": 6, "resolvedMedium": 4, "resolvedHard": 2, "resolvedRate": 0.09375, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.07407407407407407, "resolvedHardRate": 0.045454545454545456, "date": "2025-04-26", "path": "c/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 11, "resolvedEasy": 6, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.0859375, "resolvedEasyRate": 0.2, "resolvedMediumRate": 0.07407407407407407, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 11, "resolvedEasy": 4, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.0859375, "resolvedEasyRate": 0.13333333333333333, "resolvedMediumRate": 0.1111111111111111, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.0625, "resolvedEasyRate": 0.16666666666666666, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 7, "resolvedEasy": 3, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.0546875, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.07407407407407407, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 6, "resolvedEasy": 4, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.046875, "resolvedEasyRate": 0.13333333333333333, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 6, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.046875, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0.022727272727272728, "date": "2025-04-26", "path": "c/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.0390625, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.0390625, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 5, "resolvedEasy": 4, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0390625, "resolvedEasyRate": 0.13333333333333333, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.03125, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 4, "resolvedEasy": 3, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 3, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.03125, "resolvedEasyRate": 0.1, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 0, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.0234375, "resolvedEasyRate": 0, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 1, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.0234375, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 1, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.0234375, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 3, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 1, "resolvedRate": 0.0234375, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0.022727272727272728, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0234375, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 2, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.015625, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 2, "resolvedEasy": 0, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.015625, "resolvedEasyRate": 0, "resolvedMediumRate": 0.037037037037037035, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 2, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.015625, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 2, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.015625, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 2, "resolvedEasy": 2, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.015625, "resolvedEasyRate": 0.06666666666666667, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 1, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 1, "resolvedEasy": 0, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0, "resolvedMediumRate": 0.018518518518518517, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.0078125, "resolvedEasyRate": 0.03333333333333333, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "ponylang__ponyc-4595", "ponylang__ponyc-4593", "ponylang__ponyc-4588", "ponylang__ponyc-4583", "ponylang__ponyc-4506", "ponylang__ponyc-4505", "ponylang__ponyc-4458", "ponylang__ponyc-4299", "ponylang__ponyc-4294", "ponylang__ponyc-4288", "ponylang__ponyc-4263", "ponylang__ponyc-4182", "ponylang__ponyc-4132", "ponylang__ponyc-4087", "ponylang__ponyc-4067", "ponylang__ponyc-4061", "ponylang__ponyc-4057", "ponylang__ponyc-4034", "ponylang__ponyc-4024", "ponylang__ponyc-4017", "ponylang__ponyc-4006", "ponylang__ponyc-4005", "ponylang__ponyc-3991", "ponylang__ponyc-3973", "ponylang__ponyc-3971", "ponylang__ponyc-3962", "ponylang__ponyc-3953", "ponylang__ponyc-3819", "ponylang__ponyc-3781", "ponylang__ponyc-3777", "ponylang__ponyc-3763", "ponylang__ponyc-3758", "ponylang__ponyc-3746", "ponylang__ponyc-3739", "ponylang__ponyc-3725", "ponylang__ponyc-3723", "ponylang__ponyc-3675", "ponylang__ponyc-3650", "ponylang__ponyc-3647", "ponylang__ponyc-3603", "ponylang__ponyc-3586", "ponylang__ponyc-3442", "ponylang__ponyc-3293", "ponylang__ponyc-3006", "ponylang__ponyc-2950", "ponylang__ponyc-2865", "ponylang__ponyc-2853", "ponylang__ponyc-2664", "ponylang__ponyc-2660", "ponylang__ponyc-2586", "ponylang__ponyc-2532", "ponylang__ponyc-2504", "ponylang__ponyc-2342", "ponylang__ponyc-2272", "ponylang__ponyc-2261", "ponylang__ponyc-2247", "ponylang__ponyc-2221", "ponylang__ponyc-2214", "ponylang__ponyc-2205", "ponylang__ponyc-2203", "ponylang__ponyc-2201", "ponylang__ponyc-2176", "ponylang__ponyc-2168", "ponylang__ponyc-2126", "ponylang__ponyc-2007", "ponylang__ponyc-1981", "ponylang__ponyc-1867", "ponylang__ponyc-1858", "ponylang__ponyc-1852", "ponylang__ponyc-1726", "ponylang__ponyc-1536", "ponylang__ponyc-1516", "ponylang__ponyc-1486", "ponylang__ponyc-1411", "ponylang__ponyc-1339", "ponylang__ponyc-1260", "ponylang__ponyc-1124", "ponylang__ponyc-1095", "ponylang__ponyc-1057", "ponylang__ponyc-1051", "ponylang__ponyc-1035", "ponylang__ponyc-671", "facebook__zstd-3942", "facebook__zstd-3530", "facebook__zstd-3438", "facebook__zstd-3362", "facebook__zstd-3223", "facebook__zstd-2451", "facebook__zstd-2130", "facebook__zstd-2094", "facebook__zstd-1837", "facebook__zstd-1733", "facebook__zstd-1726", "facebook__zstd-1540", "facebook__zstd-1532", "facebook__zstd-1530", "facebook__zstd-1459", "facebook__zstd-1458", "facebook__zstd-1416", "facebook__zstd-1390", "facebook__zstd-1243", "facebook__zstd-1107", "facebook__zstd-1106", "facebook__zstd-1105", "facebook__zstd-1080", "facebook__zstd-1008", "facebook__zstd-983", "facebook__zstd-938", "facebook__zstd-836", "facebook__zstd-809", "facebook__zstd-637", "jqlang__jq-3238", "jqlang__jq-3165", "jqlang__jq-3161", "jqlang__jq-2919", "jqlang__jq-2840", "jqlang__jq-2839", "jqlang__jq-2824", "jqlang__jq-2821", "jqlang__jq-2728", "jqlang__jq-2674", "jqlang__jq-2658", "jqlang__jq-2654", "jqlang__jq-2646", "jqlang__jq-2624", "jqlang__jq-2157", "jqlang__jq-2133", "jqlang__jq-1793" ], "easy_ids": [ "ponylang__ponyc-3777", "ponylang__ponyc-3763", "ponylang__ponyc-3723", "ponylang__ponyc-3647", "ponylang__ponyc-3586", "ponylang__ponyc-3293", "ponylang__ponyc-3006", "ponylang__ponyc-2664", "ponylang__ponyc-2504", "ponylang__ponyc-2342", "ponylang__ponyc-2203", "ponylang__ponyc-2126", "ponylang__ponyc-1339", "ponylang__ponyc-1260", "ponylang__ponyc-1095", "facebook__zstd-2451", "facebook__zstd-2130", "facebook__zstd-1532", "facebook__zstd-1458", "facebook__zstd-1080", "jqlang__jq-3238", "jqlang__jq-2919", "jqlang__jq-2840", "jqlang__jq-2839", "jqlang__jq-2824", "jqlang__jq-2728", "jqlang__jq-2674", "jqlang__jq-2654", "jqlang__jq-2157", "jqlang__jq-1793" ], "medium_ids": [ "ponylang__ponyc-4595", "ponylang__ponyc-4458", "ponylang__ponyc-4299", "ponylang__ponyc-4294", "ponylang__ponyc-4288", "ponylang__ponyc-4263", "ponylang__ponyc-4132", "ponylang__ponyc-4087", "ponylang__ponyc-4067", "ponylang__ponyc-4061", "ponylang__ponyc-4057", "ponylang__ponyc-4005", "ponylang__ponyc-3991", "ponylang__ponyc-3973", "ponylang__ponyc-3971", "ponylang__ponyc-3953", "ponylang__ponyc-3758", "ponylang__ponyc-3725", "ponylang__ponyc-3675", "ponylang__ponyc-3650", "ponylang__ponyc-3603", "ponylang__ponyc-3442", "ponylang__ponyc-2853", "ponylang__ponyc-2586", "ponylang__ponyc-2261", "ponylang__ponyc-2247", "ponylang__ponyc-2214", "ponylang__ponyc-2007", "ponylang__ponyc-1867", "ponylang__ponyc-1852", "ponylang__ponyc-1536", "ponylang__ponyc-1516", "ponylang__ponyc-1051", "ponylang__ponyc-1035", "facebook__zstd-3942", "facebook__zstd-3530", "facebook__zstd-3438", "facebook__zstd-3362", "facebook__zstd-1837", "facebook__zstd-1733", "facebook__zstd-1726", "facebook__zstd-1540", "facebook__zstd-1459", "facebook__zstd-1106", "facebook__zstd-983", "facebook__zstd-938", "facebook__zstd-809", "facebook__zstd-637", "jqlang__jq-3165", "jqlang__jq-3161", "jqlang__jq-2658", "jqlang__jq-2646", "jqlang__jq-2624", "jqlang__jq-2133" ], "hard_ids": [ "ponylang__ponyc-4593", "ponylang__ponyc-4588", "ponylang__ponyc-4583", "ponylang__ponyc-4506", "ponylang__ponyc-4505", "ponylang__ponyc-4182", "ponylang__ponyc-4034", "ponylang__ponyc-4024", "ponylang__ponyc-4017", "ponylang__ponyc-4006", "ponylang__ponyc-3962", "ponylang__ponyc-3819", "ponylang__ponyc-3781", "ponylang__ponyc-3746", "ponylang__ponyc-3739", "ponylang__ponyc-2950", "ponylang__ponyc-2865", "ponylang__ponyc-2660", "ponylang__ponyc-2532", "ponylang__ponyc-2272", "ponylang__ponyc-2221", "ponylang__ponyc-2205", "ponylang__ponyc-2201", "ponylang__ponyc-2176", "ponylang__ponyc-2168", "ponylang__ponyc-1981", "ponylang__ponyc-1858", "ponylang__ponyc-1726", "ponylang__ponyc-1486", "ponylang__ponyc-1411", "ponylang__ponyc-1124", "ponylang__ponyc-1057", "ponylang__ponyc-671", "facebook__zstd-3223", "facebook__zstd-2094", "facebook__zstd-1530", "facebook__zstd-1416", "facebook__zstd-1390", "facebook__zstd-1243", "facebook__zstd-1107", "facebook__zstd-1105", "facebook__zstd-1008", "facebook__zstd-836", "jqlang__jq-2821" ] } } ] }, { "name": "C++", "data": [ { "name": "Verified", "results": [ { "name": "InfCode + GPT5(Sep)", "site": "https://www.tokfinity.com/infcode", "oss": true, "verified": false, "orgIcon": "https://raw.githubusercontent.com/Tokfinity/InfCode/main/figures/Tokfinity.png", "resolved": 33, "resolvedEasy": 14, "resolvedMedium": 15, "resolvedHard": 4, "resolvedRate": 0.2558139534883721, "resolvedEasyRate": 0.5, "resolvedMediumRate": 0.2542372881355932, "resolvedHardRate": 0.09523809523809523, "date": "2025-11-10", "path": "c++/verified/20251110_InfCode_GPT-5", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 19, "resolvedEasy": 9, "resolvedMedium": 9, "resolvedHard": 1, "resolvedRate": 0.14728682170542637, "resolvedEasyRate": 0.32142857142857145, "resolvedMediumRate": 0.15254237288135594, "resolvedHardRate": 0.023809523809523808, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 16, "resolvedEasy": 9, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.12403100775193798, "resolvedEasyRate": 0.32142857142857145, "resolvedMediumRate": 0.1016949152542373, "resolvedHardRate": 0.023809523809523808, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.7-Sonnet", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 15, "resolvedEasy": 8, "resolvedMedium": 7, "resolvedHard": 0, "resolvedRate": 0.11627906976744186, "resolvedEasyRate": 0.2857142857142857, "resolvedMediumRate": 0.11864406779661017, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 12, "resolvedEasy": 4, "resolvedMedium": 7, "resolvedHard": 1, "resolvedRate": 0.09302325581395349, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.11864406779661017, "resolvedHardRate": 0.023809523809523808, "date": "2025-04-26", "path": "c++/verified/20250426_MopenHands_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 11, "resolvedEasy": 6, "resolvedMedium": 5, "resolvedHard": 0, "resolvedRate": 0.08527131782945736, "resolvedEasyRate": 0.21428571428571427, "resolvedMediumRate": 0.0847457627118644, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-V3", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 10, "resolvedEasy": 3, "resolvedMedium": 6, "resolvedHard": 1, "resolvedRate": 0.07751937984496124, "resolvedEasyRate": 0.10714285714285714, "resolvedMediumRate": 0.1016949152542373, "resolvedHardRate": 0.023809523809523808, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 10, "resolvedEasy": 5, "resolvedMedium": 5, "resolvedHard": 0, "resolvedRate": 0.07751937984496124, "resolvedEasyRate": 0.17857142857142858, "resolvedMediumRate": 0.0847457627118644, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Claude-3.5-Sonnet(Oct)", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.06976744186046512, "resolvedEasyRate": 0.17857142857142858, "resolvedMediumRate": 0.06779661016949153, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 9, "resolvedEasy": 5, "resolvedMedium": 4, "resolvedHard": 0, "resolvedRate": 0.06976744186046512, "resolvedEasyRate": 0.17857142857142858, "resolvedMediumRate": 0.06779661016949153, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 9, "resolvedEasy": 4, "resolvedMedium": 4, "resolvedHard": 1, "resolvedRate": 0.06976744186046512, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.06779661016949153, "resolvedHardRate": 0.023809523809523808, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 9, "resolvedEasy": 6, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.06976744186046512, "resolvedEasyRate": 0.21428571428571427, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MSWE-agent_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + DeepSeek-R1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.06201550387596899, "resolvedEasyRate": 0.17857142857142858, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-pro", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 8, "resolvedEasy": 5, "resolvedMedium": 2, "resolvedHard": 1, "resolvedRate": 0.06201550387596899, "resolvedEasyRate": 0.17857142857142858, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0.023809523809523808, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o3-mini-high", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 7, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.05426356589147287, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.05426356589147287, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 7, "resolvedEasy": 4, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.05426356589147287, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MagentLess_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 7, "resolvedEasy": 6, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.05426356589147287, "resolvedEasyRate": 0.21428571428571427, "resolvedMediumRate": 0.01694915254237288, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MopenHands_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 6, "resolvedEasy": 4, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.046511627906976744, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 6, "resolvedEasy": 3, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.046511627906976744, "resolvedEasyRate": 0.10714285714285714, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + OpenAI-o1", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.03875968992248062, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.5-Sonnet(Oct)", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03875968992248062, "resolvedEasyRate": 0.10714285714285714, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + GPT-4o-1120", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 5, "resolvedEasy": 3, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.03875968992248062, "resolvedEasyRate": 0.10714285714285714, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + OpenAI-o1", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.03875968992248062, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_OpenAI-o1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-thinking", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 5, "resolvedEasy": 2, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.03875968992248062, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MagentLess_Doubao-1.5-thinking", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Claude-3.7-Sonnet", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.031007751937984496, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_Claude-3.7-Sonnet", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-R1", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 4, "resolvedEasy": 1, "resolvedMedium": 3, "resolvedHard": 0, "resolvedRate": 0.031007751937984496, "resolvedEasyRate": 0.03571428571428571, "resolvedMediumRate": 0.05084745762711865, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_DeepSeek-R1", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 4, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.031007751937984496, "resolvedEasyRate": 0.14285714285714285, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Llama-4-Maverick", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 4, "resolvedEasy": 2, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.031007751937984496, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MopenHands_Llama-4-Maverick", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + GPT-4o-1120", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.023255813953488372, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.01694915254237288, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_GPT-4o-1120", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MopenHands + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MopenHands", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/169105795?s=200&v=4", "resolved": 3, "resolvedEasy": 3, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.023255813953488372, "resolvedEasyRate": 0.10714285714285714, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Gemini-2.5-Pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 3, "resolvedEasy": 2, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.023255813953488372, "resolvedEasyRate": 0.07142857142857142, "resolvedMediumRate": 0.01694915254237288, "resolvedHardRate": 0, "date": "2025-04-26", "path": "c++/verified/20250426_MagentLess_Gemini-2.5-Pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + DeepSeek-V3", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 2, "resolvedEasy": 0, "resolvedMedium": 2, "resolvedHard": 0, "resolvedRate": 0.015503875968992248, "resolvedEasyRate": 0, "resolvedMediumRate": 0.03389830508474576, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_DeepSeek-V3", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + OpenAI-o3-mini-high", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 2, "resolvedEasy": 1, "resolvedMedium": 1, "resolvedHard": 0, "resolvedRate": 0.015503875968992248, "resolvedEasyRate": 0.03571428571428571, "resolvedMediumRate": 0.01694915254237288, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_OpenAI-o3-mini-high", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Qwen2.5-72B-Instruct", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 1, "resolvedEasy": 1, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0.007751937984496124, "resolvedEasyRate": 0.03571428571428571, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MSWE-agent + Qwen2.5-72B-Instruct", "site": " https://github.com/multi-swe-bench/MSWE-agent", "oss": true, "verified": true, "orgIcon": "https://avatars.githubusercontent.com/u/166046056?s=200&v=4", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "hasLogs": false, "hasTrajs": false, "hasReadme": false }, { "name": "MagentLess + Doubao-1.5-pro", "site": "https://github.com/multi-swe-bench/MagentLess", "oss": true, "verified": true, "orgIcon": "https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png", "resolved": 0, "resolvedEasy": 0, "resolvedMedium": 0, "resolvedHard": 0, "resolvedRate": 0, "resolvedEasyRate": 0, "resolvedMediumRate": 0, "resolvedHardRate": 0, "date": "2025-03-29", "path": "c++/verified/20250329_MagentLess_Doubao-1.5-pro", "hasLogs": false, "hasTrajs": false, "hasReadme": false } ], "data": { "all_ids": [ "catchorg__Catch2-2849", "catchorg__Catch2-2723", "catchorg__Catch2-2394", "catchorg__Catch2-2288", "catchorg__Catch2-2187", "catchorg__Catch2-2128", "catchorg__Catch2-1616", "catchorg__Catch2-1614", "catchorg__Catch2-1609", "catchorg__Catch2-1608", "catchorg__Catch2-1448", "catchorg__Catch2-1422", "yhirose__cpp-httplib-1765", "simdjson__simdjson-2178", "simdjson__simdjson-2150", "simdjson__simdjson-2026", "simdjson__simdjson-2016", "simdjson__simdjson-1899", "simdjson__simdjson-1896", "simdjson__simdjson-1712", "simdjson__simdjson-1695", "simdjson__simdjson-1667", "simdjson__simdjson-1624", "simdjson__simdjson-1615", "simdjson__simdjson-958", "simdjson__simdjson-954", "simdjson__simdjson-949", "simdjson__simdjson-644", "simdjson__simdjson-559", "simdjson__simdjson-545", "simdjson__simdjson-543", "simdjson__simdjson-524", "simdjson__simdjson-485", "nlohmann__json-4537", "nlohmann__json-4536", "nlohmann__json-4525", "nlohmann__json-4512", "nlohmann__json-3664", "nlohmann__json-3663", "nlohmann__json-3605", "nlohmann__json-3601", "nlohmann__json-3590", "nlohmann__json-3564", "nlohmann__json-3543", "nlohmann__json-3523", "nlohmann__json-3514", "nlohmann__json-3463", "nlohmann__json-3446", "nlohmann__json-3415", "nlohmann__json-3332", "nlohmann__json-3109", "nlohmann__json-3073", "nlohmann__json-3037", "nlohmann__json-3008", "nlohmann__json-2989", "nlohmann__json-2963", "nlohmann__json-2866", "nlohmann__json-2576", "nlohmann__json-2352", "nlohmann__json-2332", "nlohmann__json-2319", "nlohmann__json-2297", "nlohmann__json-2273", "nlohmann__json-2258", "nlohmann__json-2225", "nlohmann__json-2212", "nlohmann__json-2099", "nlohmann__json-2019", "nlohmann__json-1821", "nlohmann__json-1769", "nlohmann__json-1555", "nlohmann__json-1446", "nlohmann__json-1343", "nlohmann__json-1323", "nlohmann__json-1314", "nlohmann__json-1303", "nlohmann__json-1301", "nlohmann__json-1221", "nlohmann__json-1138", "nlohmann__json-986", "nlohmann__json-969", "nlohmann__json-944", "nlohmann__json-807", "nlohmann__json-708", "nlohmann__json-609", "nlohmann__json-545", "nlohmann__json-508", "nlohmann__json-18", "fmtlib__fmt-4310", "fmtlib__fmt-4286", "fmtlib__fmt-4057", "fmtlib__fmt-4055", "fmtlib__fmt-3913", "fmtlib__fmt-3912", "fmtlib__fmt-3863", "fmtlib__fmt-3824", "fmtlib__fmt-3819", "fmtlib__fmt-3750", "fmtlib__fmt-3734", "fmtlib__fmt-3732", "fmtlib__fmt-3729", "fmtlib__fmt-3727", "fmtlib__fmt-3713", "fmtlib__fmt-3569", "fmtlib__fmt-3555", "fmtlib__fmt-3537", "fmtlib__fmt-3279", "fmtlib__fmt-3272", "fmtlib__fmt-3271", "fmtlib__fmt-3260", "fmtlib__fmt-3248", "fmtlib__fmt-3158", "fmtlib__fmt-2940", "fmtlib__fmt-2610", "fmtlib__fmt-2394", "fmtlib__fmt-2317", "fmtlib__fmt-2310", "fmtlib__fmt-2292", "fmtlib__fmt-2158", "fmtlib__fmt-2070", "fmtlib__fmt-1837", "fmtlib__fmt-1663", "fmtlib__fmt-1407", "fmtlib__fmt-1390", "fmtlib__fmt-1361", "fmtlib__fmt-1171", "fmtlib__fmt-974", "fmtlib__fmt-967", "fmtlib__fmt-493" ], "easy_ids": [ "catchorg__Catch2-2723", "catchorg__Catch2-2288", "catchorg__Catch2-1614", "catchorg__Catch2-1608", "yhirose__cpp-httplib-1765", "simdjson__simdjson-1896", "simdjson__simdjson-1624", "nlohmann__json-4537", "nlohmann__json-4525", "nlohmann__json-4512", "nlohmann__json-3332", "nlohmann__json-1821", "nlohmann__json-1769", "nlohmann__json-1446", "nlohmann__json-1343", "nlohmann__json-1221", "nlohmann__json-969", "nlohmann__json-807", "nlohmann__json-708", "nlohmann__json-545", "fmtlib__fmt-4310", "fmtlib__fmt-4057", "fmtlib__fmt-3734", "fmtlib__fmt-3713", "fmtlib__fmt-3272", "fmtlib__fmt-3248", "fmtlib__fmt-3158", "fmtlib__fmt-1407" ], "medium_ids": [ "catchorg__Catch2-2849", "catchorg__Catch2-2394", "catchorg__Catch2-2187", "catchorg__Catch2-2128", "catchorg__Catch2-1609", "catchorg__Catch2-1448", "catchorg__Catch2-1422", "simdjson__simdjson-2178", "simdjson__simdjson-2150", "simdjson__simdjson-2026", "simdjson__simdjson-2016", "simdjson__simdjson-958", "simdjson__simdjson-644", "simdjson__simdjson-524", "nlohmann__json-4536", "nlohmann__json-3663", "nlohmann__json-3601", "nlohmann__json-3543", "nlohmann__json-3523", "nlohmann__json-3514", "nlohmann__json-3463", "nlohmann__json-3037", "nlohmann__json-3008", "nlohmann__json-2989", "nlohmann__json-2352", "nlohmann__json-2332", "nlohmann__json-2019", "nlohmann__json-1323", "nlohmann__json-986", "nlohmann__json-609", "nlohmann__json-18", "fmtlib__fmt-4286", "fmtlib__fmt-4055", "fmtlib__fmt-3913", "fmtlib__fmt-3912", "fmtlib__fmt-3863", "fmtlib__fmt-3824", "fmtlib__fmt-3819", "fmtlib__fmt-3750", "fmtlib__fmt-3732", "fmtlib__fmt-3729", "fmtlib__fmt-3727", "fmtlib__fmt-3569", "fmtlib__fmt-3555", "fmtlib__fmt-3537", "fmtlib__fmt-3279", "fmtlib__fmt-3260", "fmtlib__fmt-2940", "fmtlib__fmt-2610", "fmtlib__fmt-2394", "fmtlib__fmt-2317", "fmtlib__fmt-2310", "fmtlib__fmt-2158", "fmtlib__fmt-2070", "fmtlib__fmt-1837", "fmtlib__fmt-1390", "fmtlib__fmt-1171", "fmtlib__fmt-967", "fmtlib__fmt-493" ], "hard_ids": [ "catchorg__Catch2-1616", "simdjson__simdjson-1899", "simdjson__simdjson-1712", "simdjson__simdjson-1695", "simdjson__simdjson-1667", "simdjson__simdjson-1615", "simdjson__simdjson-954", "simdjson__simdjson-949", "simdjson__simdjson-559", "simdjson__simdjson-545", "simdjson__simdjson-543", "simdjson__simdjson-485", "nlohmann__json-3664", "nlohmann__json-3605", "nlohmann__json-3590", "nlohmann__json-3564", "nlohmann__json-3446", "nlohmann__json-3415", "nlohmann__json-3109", "nlohmann__json-3073", "nlohmann__json-2963", "nlohmann__json-2866", "nlohmann__json-2576", "nlohmann__json-2319", "nlohmann__json-2297", "nlohmann__json-2273", "nlohmann__json-2258", "nlohmann__json-2225", "nlohmann__json-2212", "nlohmann__json-2099", "nlohmann__json-1555", "nlohmann__json-1314", "nlohmann__json-1303", "nlohmann__json-1301", "nlohmann__json-1138", "nlohmann__json-944", "nlohmann__json-508", "fmtlib__fmt-3271", "fmtlib__fmt-2292", "fmtlib__fmt-1663", "fmtlib__fmt-1361", "fmtlib__fmt-974" ] } } ] } ]