{ "dataset_revision": "02e2c3dea15d9fdfd1cd7a0f65f5f8ae2ed4c1ac", "task_name": "LoCoMo", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.4, "ndcg_at_3": 0.48754, "ndcg_at_5": 0.5165, "ndcg_at_10": 0.53975, "ndcg_at_20": 0.55703, "ndcg_at_25": 0.56205, "ndcg_at_50": 0.57066, "ndcg_at_100": 0.57979, "ndcg_at_1000": 0.59575, "map_at_1": 0.39067, "map_at_3": 0.46151, "map_at_5": 0.47791, "map_at_10": 0.48798, "map_at_20": 0.49281, "map_at_25": 0.49387, "map_at_50": 0.49521, "map_at_100": 0.49603, "map_at_1000": 0.49669, "recall_at_1": 0.39067, "recall_at_3": 0.55139, "recall_at_5": 0.62103, "recall_at_10": 0.69087, "recall_at_20": 0.75873, "recall_at_25": 0.78135, "recall_at_50": 0.8252, "recall_at_100": 0.87996, "recall_at_1000": 1.0, "accuracy": 0.39067, "precision_at_1": 0.4, "precision_at_3": 0.19087, "precision_at_5": 0.12952, "precision_at_10": 0.07214, "precision_at_20": 0.03958, "precision_at_25": 0.03262, "precision_at_50": 0.01726, "precision_at_100": 0.00926, "precision_at_1000": 0.00106, "mrr_at_1": 0.4, "mrr_at_3": 0.472024, "mrr_at_5": 0.488452, "mrr_at_10": 0.497972, "mrr_at_20": 0.502566, "mrr_at_25": 0.50352, "mrr_at_50": 0.50466, "mrr_at_100": 0.505399, "mrr_at_1000": 0.505972, "nauc_ndcg_at_1_max": 0.111678, "nauc_ndcg_at_1_std": -0.048548, "nauc_ndcg_at_1_diff1": 0.468445, "nauc_ndcg_at_3_max": 0.116904, "nauc_ndcg_at_3_std": -0.034701, "nauc_ndcg_at_3_diff1": 0.42025, "nauc_ndcg_at_5_max": 0.117714, "nauc_ndcg_at_5_std": -0.035773, "nauc_ndcg_at_5_diff1": 0.415612, "nauc_ndcg_at_10_max": 0.12167, "nauc_ndcg_at_10_std": -0.04411, "nauc_ndcg_at_10_diff1": 0.408122, "nauc_ndcg_at_20_max": 0.122311, "nauc_ndcg_at_20_std": -0.047679, "nauc_ndcg_at_20_diff1": 0.406153, "nauc_ndcg_at_25_max": 0.124501, "nauc_ndcg_at_25_std": -0.043202, "nauc_ndcg_at_25_diff1": 0.408324, "nauc_ndcg_at_50_max": 0.129852, "nauc_ndcg_at_50_std": -0.037974, "nauc_ndcg_at_50_diff1": 0.409532, "nauc_ndcg_at_100_max": 0.130727, "nauc_ndcg_at_100_std": -0.036527, "nauc_ndcg_at_100_diff1": 0.408888, "nauc_ndcg_at_1000_max": 0.12371, "nauc_ndcg_at_1000_std": -0.038655, "nauc_ndcg_at_1000_diff1": 0.417014, "nauc_map_at_1_max": 0.105739, "nauc_map_at_1_std": -0.054519, "nauc_map_at_1_diff1": 0.464888, "nauc_map_at_3_max": 0.115593, "nauc_map_at_3_std": -0.041276, "nauc_map_at_3_diff1": 0.432301, "nauc_map_at_5_max": 0.114989, "nauc_map_at_5_std": -0.041038, "nauc_map_at_5_diff1": 0.429121, "nauc_map_at_10_max": 0.116479, "nauc_map_at_10_std": -0.043725, "nauc_map_at_10_diff1": 0.426452, "nauc_map_at_20_max": 0.116759, "nauc_map_at_20_std": -0.044796, "nauc_map_at_20_diff1": 0.426528, "nauc_map_at_25_max": 0.117206, "nauc_map_at_25_std": -0.043853, "nauc_map_at_25_diff1": 0.426971, "nauc_map_at_50_max": 0.117903, "nauc_map_at_50_std": -0.043139, "nauc_map_at_50_diff1": 0.427193, "nauc_map_at_100_max": 0.117938, "nauc_map_at_100_std": -0.043084, "nauc_map_at_100_diff1": 0.427143, "nauc_map_at_1000_max": 0.117733, "nauc_map_at_1000_std": -0.043092, "nauc_map_at_1000_diff1": 0.427396, "nauc_recall_at_1_max": 0.105739, "nauc_recall_at_1_std": -0.054519, "nauc_recall_at_1_diff1": 0.464888, "nauc_recall_at_3_max": 0.117216, "nauc_recall_at_3_std": -0.029391, "nauc_recall_at_3_diff1": 0.384591, "nauc_recall_at_5_max": 0.117205, "nauc_recall_at_5_std": -0.029017, "nauc_recall_at_5_diff1": 0.364597, "nauc_recall_at_10_max": 0.132851, "nauc_recall_at_10_std": -0.061013, "nauc_recall_at_10_diff1": 0.330567, "nauc_recall_at_20_max": 0.137403, "nauc_recall_at_20_std": -0.082296, "nauc_recall_at_20_diff1": 0.297567, "nauc_recall_at_25_max": 0.153349, "nauc_recall_at_25_std": -0.055966, "nauc_recall_at_25_diff1": 0.304618, "nauc_recall_at_50_max": 0.208285, "nauc_recall_at_50_std": -0.010494, "nauc_recall_at_50_diff1": 0.292513, "nauc_recall_at_100_max": 0.255276, "nauc_recall_at_100_std": 0.024344, "nauc_recall_at_100_diff1": 0.243835, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.111678, "nauc_precision_at_1_std": -0.048548, "nauc_precision_at_1_diff1": 0.468445, "nauc_precision_at_3_max": 0.118279, "nauc_precision_at_3_std": 0.005115, "nauc_precision_at_3_diff1": 0.374583, "nauc_precision_at_5_max": 0.121583, "nauc_precision_at_5_std": 0.005665, "nauc_precision_at_5_diff1": 0.352511, "nauc_precision_at_10_max": 0.13609, "nauc_precision_at_10_std": -0.00761, "nauc_precision_at_10_diff1": 0.302595, "nauc_precision_at_20_max": 0.137308, "nauc_precision_at_20_std": -0.018692, "nauc_precision_at_20_diff1": 0.263673, "nauc_precision_at_25_max": 0.150361, "nauc_precision_at_25_std": 0.013211, "nauc_precision_at_25_diff1": 0.259848, "nauc_precision_at_50_max": 0.19022, "nauc_precision_at_50_std": 0.066363, "nauc_precision_at_50_diff1": 0.219311, "nauc_precision_at_100_max": 0.198885, "nauc_precision_at_100_std": 0.097435, "nauc_precision_at_100_diff1": 0.119165, "nauc_precision_at_1000_max": 0.032319, "nauc_precision_at_1000_std": 0.245572, "nauc_precision_at_1000_diff1": -0.109082, "nauc_mrr_at_1_max": 0.111678, "nauc_mrr_at_1_std": -0.048548, "nauc_mrr_at_1_diff1": 0.468445, "nauc_mrr_at_3_max": 0.118144, "nauc_mrr_at_3_std": -0.035922, "nauc_mrr_at_3_diff1": 0.430162, "nauc_mrr_at_5_max": 0.121709, "nauc_mrr_at_5_std": -0.034371, "nauc_mrr_at_5_diff1": 0.429471, "nauc_mrr_at_10_max": 0.122753, "nauc_mrr_at_10_std": -0.038764, "nauc_mrr_at_10_diff1": 0.426398, "nauc_mrr_at_20_max": 0.12275, "nauc_mrr_at_20_std": -0.039623, "nauc_mrr_at_20_diff1": 0.425641, "nauc_mrr_at_25_max": 0.123088, "nauc_mrr_at_25_std": -0.038872, "nauc_mrr_at_25_diff1": 0.426162, "nauc_mrr_at_50_max": 0.12362, "nauc_mrr_at_50_std": -0.038313, "nauc_mrr_at_50_diff1": 0.426466, "nauc_mrr_at_100_max": 0.123509, "nauc_mrr_at_100_std": -0.03814, "nauc_mrr_at_100_diff1": 0.426469, "nauc_mrr_at_1000_max": 0.123325, "nauc_mrr_at_1000_std": -0.038306, "nauc_mrr_at_1000_diff1": 0.42679, "hit_rate_at_1": 0.4, "hit_rate_at_3": 0.56667, "hit_rate_at_5": 0.6381, "hit_rate_at_10": 0.70714, "hit_rate_at_20": 0.77381, "hit_rate_at_25": 0.79524, "hit_rate_at_50": 0.83571, "hit_rate_at_100": 0.8881, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.4, "R_cap_at_3": 0.551786, "R_cap_at_5": 0.621032, "R_cap_at_10": 0.690873, "R_cap_at_20": 0.75873, "R_cap_at_25": 0.781349, "R_cap_at_50": 0.825198, "R_cap_at_100": 0.87996, "R_cap_at_1000": 1.0, "main_score": 0.53975, "hf_subset": "single_hop", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.22857, "ndcg_at_3": 0.21412, "ndcg_at_5": 0.24357, "ndcg_at_10": 0.28487, "ndcg_at_20": 0.32003, "ndcg_at_25": 0.32985, "ndcg_at_50": 0.35502, "ndcg_at_100": 0.37224, "ndcg_at_1000": 0.43268, "map_at_1": 0.08983, "map_at_3": 0.15227, "map_at_5": 0.17899, "map_at_10": 0.20319, "map_at_20": 0.21663, "map_at_25": 0.21981, "map_at_50": 0.22543, "map_at_100": 0.22774, "map_at_1000": 0.23165, "recall_at_1": 0.08983, "recall_at_3": 0.19154, "recall_at_5": 0.26645, "recall_at_10": 0.36647, "recall_at_20": 0.46615, "recall_at_25": 0.49724, "recall_at_50": 0.58891, "recall_at_100": 0.66211, "recall_at_1000": 0.9983, "accuracy": 0.08983, "precision_at_1": 0.22857, "precision_at_3": 0.17024, "precision_at_5": 0.14429, "precision_at_10": 0.10429, "precision_at_20": 0.06804, "precision_at_25": 0.05871, "precision_at_50": 0.03536, "precision_at_100": 0.02029, "precision_at_1000": 0.00312, "mrr_at_1": 0.228571, "mrr_at_3": 0.299405, "mrr_at_5": 0.322798, "mrr_at_10": 0.339192, "mrr_at_20": 0.346389, "mrr_at_25": 0.347514, "mrr_at_50": 0.349405, "mrr_at_100": 0.350381, "mrr_at_1000": 0.351035, "nauc_ndcg_at_1_max": 0.069158, "nauc_ndcg_at_1_std": 0.002966, "nauc_ndcg_at_1_diff1": 0.254834, "nauc_ndcg_at_3_max": 0.087455, "nauc_ndcg_at_3_std": 0.018091, "nauc_ndcg_at_3_diff1": 0.209282, "nauc_ndcg_at_5_max": 0.085059, "nauc_ndcg_at_5_std": 0.004012, "nauc_ndcg_at_5_diff1": 0.212285, "nauc_ndcg_at_10_max": 0.075994, "nauc_ndcg_at_10_std": -0.009755, "nauc_ndcg_at_10_diff1": 0.222428, "nauc_ndcg_at_20_max": 0.068329, "nauc_ndcg_at_20_std": -0.00675, "nauc_ndcg_at_20_diff1": 0.204662, "nauc_ndcg_at_25_max": 0.076066, "nauc_ndcg_at_25_std": 0.004463, "nauc_ndcg_at_25_diff1": 0.209555, "nauc_ndcg_at_50_max": 0.105148, "nauc_ndcg_at_50_std": 0.03814, "nauc_ndcg_at_50_diff1": 0.217746, "nauc_ndcg_at_100_max": 0.106819, "nauc_ndcg_at_100_std": 0.053349, "nauc_ndcg_at_100_diff1": 0.217466, "nauc_ndcg_at_1000_max": 0.095453, "nauc_ndcg_at_1000_std": 0.035647, "nauc_ndcg_at_1000_diff1": 0.212294, "nauc_map_at_1_max": 0.068762, "nauc_map_at_1_std": -0.003695, "nauc_map_at_1_diff1": 0.260599, "nauc_map_at_3_max": 0.088854, "nauc_map_at_3_std": -0.013028, "nauc_map_at_3_diff1": 0.237479, "nauc_map_at_5_max": 0.08982, "nauc_map_at_5_std": -0.007335, "nauc_map_at_5_diff1": 0.21947, "nauc_map_at_10_max": 0.081273, "nauc_map_at_10_std": -0.010387, "nauc_map_at_10_diff1": 0.213097, "nauc_map_at_20_max": 0.074977, "nauc_map_at_20_std": -0.011312, "nauc_map_at_20_diff1": 0.199277, "nauc_map_at_25_max": 0.078147, "nauc_map_at_25_std": -0.005471, "nauc_map_at_25_diff1": 0.199075, "nauc_map_at_50_max": 0.087945, "nauc_map_at_50_std": 0.004703, "nauc_map_at_50_diff1": 0.203196, "nauc_map_at_100_max": 0.088498, "nauc_map_at_100_std": 0.006464, "nauc_map_at_100_diff1": 0.20407, "nauc_map_at_1000_max": 0.087681, "nauc_map_at_1000_std": 0.006465, "nauc_map_at_1000_diff1": 0.203883, "nauc_recall_at_1_max": 0.068762, "nauc_recall_at_1_std": -0.003695, "nauc_recall_at_1_diff1": 0.260599, "nauc_recall_at_3_max": 0.086998, "nauc_recall_at_3_std": -0.022618, "nauc_recall_at_3_diff1": 0.221732, "nauc_recall_at_5_max": 0.069886, "nauc_recall_at_5_std": -0.025922, "nauc_recall_at_5_diff1": 0.196372, "nauc_recall_at_10_max": 0.048594, "nauc_recall_at_10_std": -0.044101, "nauc_recall_at_10_diff1": 0.207803, "nauc_recall_at_20_max": 0.030444, "nauc_recall_at_20_std": -0.043468, "nauc_recall_at_20_diff1": 0.165689, "nauc_recall_at_25_max": 0.047655, "nauc_recall_at_25_std": -0.018582, "nauc_recall_at_25_diff1": 0.179347, "nauc_recall_at_50_max": 0.128337, "nauc_recall_at_50_std": 0.082789, "nauc_recall_at_50_diff1": 0.194341, "nauc_recall_at_100_max": 0.139381, "nauc_recall_at_100_std": 0.147619, "nauc_recall_at_100_diff1": 0.187595, "nauc_recall_at_1000_max": 0.04239, "nauc_recall_at_1000_std": 0.866246, "nauc_recall_at_1000_diff1": 0.557096, "nauc_precision_at_1_max": 0.069158, "nauc_precision_at_1_std": 0.002966, "nauc_precision_at_1_diff1": 0.254834, "nauc_precision_at_3_max": 0.089742, "nauc_precision_at_3_std": 0.050356, "nauc_precision_at_3_diff1": 0.160829, "nauc_precision_at_5_max": 0.078038, "nauc_precision_at_5_std": 0.040887, "nauc_precision_at_5_diff1": 0.130143, "nauc_precision_at_10_max": 0.061107, "nauc_precision_at_10_std": 0.03306, "nauc_precision_at_10_diff1": 0.110745, "nauc_precision_at_20_max": 0.042791, "nauc_precision_at_20_std": 0.057645, "nauc_precision_at_20_diff1": 0.053327, "nauc_precision_at_25_max": 0.065468, "nauc_precision_at_25_std": 0.091231, "nauc_precision_at_25_diff1": 0.058458, "nauc_precision_at_50_max": 0.142236, "nauc_precision_at_50_std": 0.175508, "nauc_precision_at_50_diff1": 0.060606, "nauc_precision_at_100_max": 0.136034, "nauc_precision_at_100_std": 0.211633, "nauc_precision_at_100_diff1": 0.041118, "nauc_precision_at_1000_max": 0.060315, "nauc_precision_at_1000_std": 0.103099, "nauc_precision_at_1000_diff1": -0.071989, "nauc_mrr_at_1_max": 0.069158, "nauc_mrr_at_1_std": 0.002966, "nauc_mrr_at_1_diff1": 0.254834, "nauc_mrr_at_3_max": 0.077909, "nauc_mrr_at_3_std": 0.012967, "nauc_mrr_at_3_diff1": 0.224929, "nauc_mrr_at_5_max": 0.074636, "nauc_mrr_at_5_std": 0.011523, "nauc_mrr_at_5_diff1": 0.23483, "nauc_mrr_at_10_max": 0.082572, "nauc_mrr_at_10_std": 0.017956, "nauc_mrr_at_10_diff1": 0.237807, "nauc_mrr_at_20_max": 0.083894, "nauc_mrr_at_20_std": 0.018893, "nauc_mrr_at_20_diff1": 0.236785, "nauc_mrr_at_25_max": 0.084182, "nauc_mrr_at_25_std": 0.019471, "nauc_mrr_at_25_diff1": 0.237954, "nauc_mrr_at_50_max": 0.084336, "nauc_mrr_at_50_std": 0.020254, "nauc_mrr_at_50_diff1": 0.236556, "nauc_mrr_at_100_max": 0.083888, "nauc_mrr_at_100_std": 0.02091, "nauc_mrr_at_100_diff1": 0.236384, "nauc_mrr_at_1000_max": 0.083732, "nauc_mrr_at_1000_std": 0.020413, "nauc_mrr_at_1000_diff1": 0.236235, "hit_rate_at_1": 0.22857, "hit_rate_at_3": 0.38929, "hit_rate_at_5": 0.49286, "hit_rate_at_10": 0.61429, "hit_rate_at_20": 0.71429, "hit_rate_at_25": 0.73929, "hit_rate_at_50": 0.80714, "hit_rate_at_100": 0.87857, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.228571, "R_cap_at_3": 0.214881, "R_cap_at_5": 0.273214, "R_cap_at_10": 0.366869, "R_cap_at_20": 0.466153, "R_cap_at_25": 0.497239, "R_cap_at_50": 0.588908, "R_cap_at_100": 0.662113, "R_cap_at_1000": 0.998299, "main_score": 0.28487, "hf_subset": "multi_hop", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.46729, "ndcg_at_3": 0.54802, "ndcg_at_5": 0.57429, "ndcg_at_10": 0.59726, "ndcg_at_20": 0.61191, "ndcg_at_25": 0.61383, "ndcg_at_50": 0.62613, "ndcg_at_100": 0.63595, "ndcg_at_1000": 0.64688, "map_at_1": 0.43224, "map_at_3": 0.51826, "map_at_5": 0.53337, "map_at_10": 0.54385, "map_at_20": 0.54806, "map_at_25": 0.54843, "map_at_50": 0.55054, "map_at_100": 0.55148, "map_at_1000": 0.55194, "recall_at_1": 0.43224, "recall_at_3": 0.60566, "recall_at_5": 0.66952, "recall_at_10": 0.73546, "recall_at_20": 0.79102, "recall_at_25": 0.79907, "recall_at_50": 0.85826, "recall_at_100": 0.91589, "recall_at_1000": 0.99688, "accuracy": 0.43224, "precision_at_1": 0.46729, "precision_at_3": 0.22118, "precision_at_5": 0.14642, "precision_at_10": 0.08162, "precision_at_20": 0.04439, "precision_at_25": 0.03601, "precision_at_50": 0.01956, "precision_at_100": 0.01053, "precision_at_1000": 0.00117, "mrr_at_1": 0.46729, "mrr_at_3": 0.547248, "mrr_at_5": 0.561111, "mrr_at_10": 0.570301, "mrr_at_20": 0.573778, "mrr_at_25": 0.574168, "mrr_at_50": 0.575692, "mrr_at_100": 0.576424, "mrr_at_1000": 0.576687, "nauc_ndcg_at_1_max": 0.285372, "nauc_ndcg_at_1_std": -0.018086, "nauc_ndcg_at_1_diff1": 0.614157, "nauc_ndcg_at_3_max": 0.264857, "nauc_ndcg_at_3_std": -0.033652, "nauc_ndcg_at_3_diff1": 0.558116, "nauc_ndcg_at_5_max": 0.247753, "nauc_ndcg_at_5_std": -0.054446, "nauc_ndcg_at_5_diff1": 0.538243, "nauc_ndcg_at_10_max": 0.242155, "nauc_ndcg_at_10_std": -0.044631, "nauc_ndcg_at_10_diff1": 0.534375, "nauc_ndcg_at_20_max": 0.234469, "nauc_ndcg_at_20_std": -0.051766, "nauc_ndcg_at_20_diff1": 0.542499, "nauc_ndcg_at_25_max": 0.234345, "nauc_ndcg_at_25_std": -0.053691, "nauc_ndcg_at_25_diff1": 0.542782, "nauc_ndcg_at_50_max": 0.232806, "nauc_ndcg_at_50_std": -0.045693, "nauc_ndcg_at_50_diff1": 0.542379, "nauc_ndcg_at_100_max": 0.242971, "nauc_ndcg_at_100_std": -0.030575, "nauc_ndcg_at_100_diff1": 0.542972, "nauc_ndcg_at_1000_max": 0.248261, "nauc_ndcg_at_1000_std": -0.038685, "nauc_ndcg_at_1000_diff1": 0.549424, "nauc_map_at_1_max": 0.275129, "nauc_map_at_1_std": 0.019027, "nauc_map_at_1_diff1": 0.58794, "nauc_map_at_3_max": 0.261912, "nauc_map_at_3_std": -0.022083, "nauc_map_at_3_diff1": 0.562058, "nauc_map_at_5_max": 0.25157, "nauc_map_at_5_std": -0.034566, "nauc_map_at_5_diff1": 0.552324, "nauc_map_at_10_max": 0.251697, "nauc_map_at_10_std": -0.031714, "nauc_map_at_10_diff1": 0.551838, "nauc_map_at_20_max": 0.250422, "nauc_map_at_20_std": -0.033608, "nauc_map_at_20_diff1": 0.554916, "nauc_map_at_25_max": 0.250374, "nauc_map_at_25_std": -0.033932, "nauc_map_at_25_diff1": 0.554983, "nauc_map_at_50_max": 0.250041, "nauc_map_at_50_std": -0.032563, "nauc_map_at_50_diff1": 0.555283, "nauc_map_at_100_max": 0.250875, "nauc_map_at_100_std": -0.031311, "nauc_map_at_100_diff1": 0.555244, "nauc_map_at_1000_max": 0.251146, "nauc_map_at_1000_std": -0.031562, "nauc_map_at_1000_diff1": 0.555411, "nauc_recall_at_1_max": 0.275129, "nauc_recall_at_1_std": 0.019027, "nauc_recall_at_1_diff1": 0.58794, "nauc_recall_at_3_max": 0.254332, "nauc_recall_at_3_std": -0.041035, "nauc_recall_at_3_diff1": 0.512365, "nauc_recall_at_5_max": 0.202265, "nauc_recall_at_5_std": -0.098523, "nauc_recall_at_5_diff1": 0.447762, "nauc_recall_at_10_max": 0.164132, "nauc_recall_at_10_std": -0.069821, "nauc_recall_at_10_diff1": 0.412017, "nauc_recall_at_20_max": 0.09832, "nauc_recall_at_20_std": -0.117103, "nauc_recall_at_20_diff1": 0.425104, "nauc_recall_at_25_max": 0.096135, "nauc_recall_at_25_std": -0.128679, "nauc_recall_at_25_diff1": 0.423139, "nauc_recall_at_50_max": 0.045151, "nauc_recall_at_50_std": -0.072622, "nauc_recall_at_50_diff1": 0.375728, "nauc_recall_at_100_max": 0.109902, "nauc_recall_at_100_std": 0.180924, "nauc_recall_at_100_diff1": 0.300234, "nauc_recall_at_1000_max": 0.358867, "nauc_recall_at_1000_std": 0.869326, "nauc_recall_at_1000_diff1": -0.564116, "nauc_precision_at_1_max": 0.285372, "nauc_precision_at_1_std": -0.018086, "nauc_precision_at_1_diff1": 0.614157, "nauc_precision_at_3_max": 0.261235, "nauc_precision_at_3_std": -0.097601, "nauc_precision_at_3_diff1": 0.490571, "nauc_precision_at_5_max": 0.197098, "nauc_precision_at_5_std": -0.161802, "nauc_precision_at_5_diff1": 0.411432, "nauc_precision_at_10_max": 0.182946, "nauc_precision_at_10_std": -0.105712, "nauc_precision_at_10_diff1": 0.313224, "nauc_precision_at_20_max": 0.115996, "nauc_precision_at_20_std": -0.137825, "nauc_precision_at_20_diff1": 0.302682, "nauc_precision_at_25_max": 0.100058, "nauc_precision_at_25_std": -0.157312, "nauc_precision_at_25_diff1": 0.286413, "nauc_precision_at_50_max": 0.026456, "nauc_precision_at_50_std": -0.111477, "nauc_precision_at_50_diff1": 0.175425, "nauc_precision_at_100_max": 0.054889, "nauc_precision_at_100_std": 0.018479, "nauc_precision_at_100_diff1": 0.045603, "nauc_precision_at_1000_max": 0.002108, "nauc_precision_at_1000_std": -0.066592, "nauc_precision_at_1000_diff1": -0.086792, "nauc_mrr_at_1_max": 0.285372, "nauc_mrr_at_1_std": -0.018086, "nauc_mrr_at_1_diff1": 0.614157, "nauc_mrr_at_3_max": 0.27596, "nauc_mrr_at_3_std": -0.046731, "nauc_mrr_at_3_diff1": 0.585628, "nauc_mrr_at_5_max": 0.269576, "nauc_mrr_at_5_std": -0.05606, "nauc_mrr_at_5_diff1": 0.574904, "nauc_mrr_at_10_max": 0.263484, "nauc_mrr_at_10_std": -0.050813, "nauc_mrr_at_10_diff1": 0.572615, "nauc_mrr_at_20_max": 0.262414, "nauc_mrr_at_20_std": -0.051924, "nauc_mrr_at_20_diff1": 0.574125, "nauc_mrr_at_25_max": 0.262425, "nauc_mrr_at_25_std": -0.05226, "nauc_mrr_at_25_diff1": 0.574158, "nauc_mrr_at_50_max": 0.262534, "nauc_mrr_at_50_std": -0.051246, "nauc_mrr_at_50_diff1": 0.574137, "nauc_mrr_at_100_max": 0.263279, "nauc_mrr_at_100_std": -0.050494, "nauc_mrr_at_100_diff1": 0.574327, "nauc_mrr_at_1000_max": 0.263476, "nauc_mrr_at_1000_std": -0.05065, "nauc_mrr_at_1000_diff1": 0.574512, "hit_rate_at_1": 0.46729, "hit_rate_at_3": 0.63863, "hit_rate_at_5": 0.70093, "hit_rate_at_10": 0.76636, "hit_rate_at_20": 0.81931, "hit_rate_at_25": 0.82866, "hit_rate_at_50": 0.88162, "hit_rate_at_100": 0.93458, "hit_rate_at_1000": 0.99688, "R_cap_at_1": 0.46729, "R_cap_at_3": 0.605919, "R_cap_at_5": 0.669522, "R_cap_at_10": 0.735462, "R_cap_at_20": 0.791018, "R_cap_at_25": 0.799065, "R_cap_at_50": 0.858255, "R_cap_at_100": 0.915888, "R_cap_at_1000": 0.996885, "main_score": 0.59726, "hf_subset": "temporal_reasoning", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.22472, "ndcg_at_3": 0.23012, "ndcg_at_5": 0.25148, "ndcg_at_10": 0.28108, "ndcg_at_20": 0.30588, "ndcg_at_25": 0.30875, "ndcg_at_50": 0.32466, "ndcg_at_100": 0.34434, "ndcg_at_1000": 0.40428, "map_at_1": 0.15824, "map_at_3": 0.19835, "map_at_5": 0.21071, "map_at_10": 0.22479, "map_at_20": 0.23238, "map_at_25": 0.23338, "map_at_50": 0.23704, "map_at_100": 0.23901, "map_at_1000": 0.24198, "recall_at_1": 0.15824, "recall_at_3": 0.24251, "recall_at_5": 0.29307, "recall_at_10": 0.37247, "recall_at_20": 0.45244, "recall_at_25": 0.46153, "recall_at_50": 0.51884, "recall_at_100": 0.62334, "recall_at_1000": 1.0, "accuracy": 0.15824, "precision_at_1": 0.22472, "precision_at_3": 0.11985, "precision_at_5": 0.08764, "precision_at_10": 0.05843, "precision_at_20": 0.03764, "precision_at_25": 0.03146, "precision_at_50": 0.01978, "precision_at_100": 0.01191, "precision_at_1000": 0.00221, "mrr_at_1": 0.224719, "mrr_at_3": 0.275281, "mrr_at_5": 0.290449, "mrr_at_10": 0.301355, "mrr_at_20": 0.309223, "mrr_at_25": 0.309223, "mrr_at_50": 0.310143, "mrr_at_100": 0.311563, "mrr_at_1000": 0.312747, "nauc_ndcg_at_1_max": 0.106164, "nauc_ndcg_at_1_std": 0.050787, "nauc_ndcg_at_1_diff1": 0.455944, "nauc_ndcg_at_3_max": 0.071457, "nauc_ndcg_at_3_std": -0.043425, "nauc_ndcg_at_3_diff1": 0.448206, "nauc_ndcg_at_5_max": 0.082637, "nauc_ndcg_at_5_std": -0.020865, "nauc_ndcg_at_5_diff1": 0.40203, "nauc_ndcg_at_10_max": 0.061128, "nauc_ndcg_at_10_std": -0.021102, "nauc_ndcg_at_10_diff1": 0.34041, "nauc_ndcg_at_20_max": 0.097684, "nauc_ndcg_at_20_std": -0.003022, "nauc_ndcg_at_20_diff1": 0.352249, "nauc_ndcg_at_25_max": 0.104169, "nauc_ndcg_at_25_std": 0.00453, "nauc_ndcg_at_25_diff1": 0.352907, "nauc_ndcg_at_50_max": 0.104153, "nauc_ndcg_at_50_std": 0.003691, "nauc_ndcg_at_50_diff1": 0.352196, "nauc_ndcg_at_100_max": 0.100114, "nauc_ndcg_at_100_std": 0.020389, "nauc_ndcg_at_100_diff1": 0.339951, "nauc_ndcg_at_1000_max": 0.114039, "nauc_ndcg_at_1000_std": 0.03044, "nauc_ndcg_at_1000_diff1": 0.379267, "nauc_map_at_1_max": 0.138716, "nauc_map_at_1_std": 0.09075, "nauc_map_at_1_diff1": 0.488718, "nauc_map_at_3_max": 0.09292, "nauc_map_at_3_std": -0.011885, "nauc_map_at_3_diff1": 0.466052, "nauc_map_at_5_max": 0.094427, "nauc_map_at_5_std": 0.006418, "nauc_map_at_5_diff1": 0.436034, "nauc_map_at_10_max": 0.080618, "nauc_map_at_10_std": 0.007325, "nauc_map_at_10_diff1": 0.398212, "nauc_map_at_20_max": 0.093367, "nauc_map_at_20_std": 0.012546, "nauc_map_at_20_diff1": 0.404171, "nauc_map_at_25_max": 0.095551, "nauc_map_at_25_std": 0.015475, "nauc_map_at_25_diff1": 0.40424, "nauc_map_at_50_max": 0.094126, "nauc_map_at_50_std": 0.015777, "nauc_map_at_50_diff1": 0.402951, "nauc_map_at_100_max": 0.093765, "nauc_map_at_100_std": 0.017703, "nauc_map_at_100_diff1": 0.402137, "nauc_map_at_1000_max": 0.096297, "nauc_map_at_1000_std": 0.020242, "nauc_map_at_1000_diff1": 0.404939, "nauc_recall_at_1_max": 0.138716, "nauc_recall_at_1_std": 0.09075, "nauc_recall_at_1_diff1": 0.488718, "nauc_recall_at_3_max": 0.053121, "nauc_recall_at_3_std": -0.117973, "nauc_recall_at_3_diff1": 0.426754, "nauc_recall_at_5_max": 0.060311, "nauc_recall_at_5_std": -0.059914, "nauc_recall_at_5_diff1": 0.303217, "nauc_recall_at_10_max": 0.002979, "nauc_recall_at_10_std": -0.058499, "nauc_recall_at_10_diff1": 0.142987, "nauc_recall_at_20_max": 0.098108, "nauc_recall_at_20_std": -0.014722, "nauc_recall_at_20_diff1": 0.17964, "nauc_recall_at_25_max": 0.116761, "nauc_recall_at_25_std": 0.004776, "nauc_recall_at_25_diff1": 0.185333, "nauc_recall_at_50_max": 0.110134, "nauc_recall_at_50_std": -0.004535, "nauc_recall_at_50_diff1": 0.194129, "nauc_recall_at_100_max": 0.09487, "nauc_recall_at_100_std": 0.062293, "nauc_recall_at_100_diff1": 0.106288, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.106164, "nauc_precision_at_1_std": 0.050787, "nauc_precision_at_1_diff1": 0.455944, "nauc_precision_at_3_max": 0.00267, "nauc_precision_at_3_std": -0.111846, "nauc_precision_at_3_diff1": 0.38116, "nauc_precision_at_5_max": 0.041146, "nauc_precision_at_5_std": -0.039605, "nauc_precision_at_5_diff1": 0.266284, "nauc_precision_at_10_max": 0.013129, "nauc_precision_at_10_std": -0.049807, "nauc_precision_at_10_diff1": 0.126512, "nauc_precision_at_20_max": 0.177389, "nauc_precision_at_20_std": 0.03832, "nauc_precision_at_20_diff1": 0.131882, "nauc_precision_at_25_max": 0.207692, "nauc_precision_at_25_std": 0.075515, "nauc_precision_at_25_diff1": 0.147495, "nauc_precision_at_50_max": 0.226233, "nauc_precision_at_50_std": 0.082431, "nauc_precision_at_50_diff1": 0.158138, "nauc_precision_at_100_max": 0.183934, "nauc_precision_at_100_std": 0.166738, "nauc_precision_at_100_diff1": 0.077924, "nauc_precision_at_1000_max": 0.27725, "nauc_precision_at_1000_std": 0.211516, "nauc_precision_at_1000_diff1": 0.10415, "nauc_mrr_at_1_max": 0.106164, "nauc_mrr_at_1_std": 0.050787, "nauc_mrr_at_1_diff1": 0.455944, "nauc_mrr_at_3_max": 0.06583, "nauc_mrr_at_3_std": -0.036862, "nauc_mrr_at_3_diff1": 0.439839, "nauc_mrr_at_5_max": 0.078627, "nauc_mrr_at_5_std": -0.026158, "nauc_mrr_at_5_diff1": 0.409126, "nauc_mrr_at_10_max": 0.079854, "nauc_mrr_at_10_std": -0.02458, "nauc_mrr_at_10_diff1": 0.401687, "nauc_mrr_at_20_max": 0.087287, "nauc_mrr_at_20_std": -0.017872, "nauc_mrr_at_20_diff1": 0.402341, "nauc_mrr_at_25_max": 0.087287, "nauc_mrr_at_25_std": -0.017872, "nauc_mrr_at_25_diff1": 0.402341, "nauc_mrr_at_50_max": 0.088609, "nauc_mrr_at_50_std": -0.018593, "nauc_mrr_at_50_diff1": 0.402966, "nauc_mrr_at_100_max": 0.088195, "nauc_mrr_at_100_std": -0.01683, "nauc_mrr_at_100_diff1": 0.40183, "nauc_mrr_at_1000_max": 0.087154, "nauc_mrr_at_1000_std": -0.016418, "nauc_mrr_at_1000_diff1": 0.402316, "hit_rate_at_1": 0.22472, "hit_rate_at_3": 0.33708, "hit_rate_at_5": 0.40449, "hit_rate_at_10": 0.49438, "hit_rate_at_20": 0.59551, "hit_rate_at_25": 0.59551, "hit_rate_at_50": 0.62921, "hit_rate_at_100": 0.73034, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.224719, "R_cap_at_3": 0.245318, "R_cap_at_5": 0.293071, "R_cap_at_10": 0.373034, "R_cap_at_20": 0.452442, "R_cap_at_25": 0.46153, "R_cap_at_50": 0.518842, "R_cap_at_100": 0.623336, "R_cap_at_1000": 1.0, "main_score": 0.28108, "hf_subset": "open_domain", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.22422, "ndcg_at_3": 0.29993, "ndcg_at_5": 0.32948, "ndcg_at_10": 0.35639, "ndcg_at_20": 0.37585, "ndcg_at_25": 0.38058, "ndcg_at_50": 0.39169, "ndcg_at_100": 0.40109, "ndcg_at_1000": 0.43539, "map_at_1": 0.22197, "map_at_3": 0.28064, "map_at_5": 0.29718, "map_at_10": 0.30802, "map_at_20": 0.31346, "map_at_25": 0.31445, "map_at_50": 0.31608, "map_at_100": 0.31695, "map_at_1000": 0.31793, "recall_at_1": 0.22197, "recall_at_3": 0.35314, "recall_at_5": 0.42377, "recall_at_10": 0.50673, "recall_at_20": 0.58296, "recall_at_25": 0.60426, "recall_at_50": 0.66143, "recall_at_100": 0.71861, "recall_at_1000": 1.0, "accuracy": 0.22197, "precision_at_1": 0.22422, "precision_at_3": 0.11883, "precision_at_5": 0.08565, "precision_at_10": 0.05157, "precision_at_20": 0.0296, "precision_at_25": 0.02457, "precision_at_50": 0.0135, "precision_at_100": 0.00733, "precision_at_1000": 0.00103, "mrr_at_1": 0.224215, "mrr_at_3": 0.283259, "mrr_at_5": 0.300075, "mrr_at_10": 0.311506, "mrr_at_20": 0.31695, "mrr_at_25": 0.317846, "mrr_at_50": 0.319474, "mrr_at_100": 0.320306, "mrr_at_1000": 0.321232, "nauc_ndcg_at_1_max": 0.080607, "nauc_ndcg_at_1_std": -0.059698, "nauc_ndcg_at_1_diff1": 0.33603, "nauc_ndcg_at_3_max": 0.076694, "nauc_ndcg_at_3_std": -0.100871, "nauc_ndcg_at_3_diff1": 0.316058, "nauc_ndcg_at_5_max": 0.091841, "nauc_ndcg_at_5_std": -0.105897, "nauc_ndcg_at_5_diff1": 0.320655, "nauc_ndcg_at_10_max": 0.08837, "nauc_ndcg_at_10_std": -0.110147, "nauc_ndcg_at_10_diff1": 0.306672, "nauc_ndcg_at_20_max": 0.081707, "nauc_ndcg_at_20_std": -0.128685, "nauc_ndcg_at_20_diff1": 0.304801, "nauc_ndcg_at_25_max": 0.08558, "nauc_ndcg_at_25_std": -0.126928, "nauc_ndcg_at_25_diff1": 0.302917, "nauc_ndcg_at_50_max": 0.097849, "nauc_ndcg_at_50_std": -0.11451, "nauc_ndcg_at_50_diff1": 0.300543, "nauc_ndcg_at_100_max": 0.101154, "nauc_ndcg_at_100_std": -0.107657, "nauc_ndcg_at_100_diff1": 0.290256, "nauc_ndcg_at_1000_max": 0.094035, "nauc_ndcg_at_1000_std": -0.104851, "nauc_ndcg_at_1000_diff1": 0.306917, "nauc_map_at_1_max": 0.082733, "nauc_map_at_1_std": -0.065071, "nauc_map_at_1_diff1": 0.342807, "nauc_map_at_3_max": 0.0811, "nauc_map_at_3_std": -0.091722, "nauc_map_at_3_diff1": 0.324012, "nauc_map_at_5_max": 0.08942, "nauc_map_at_5_std": -0.094162, "nauc_map_at_5_diff1": 0.32668, "nauc_map_at_10_max": 0.088989, "nauc_map_at_10_std": -0.095799, "nauc_map_at_10_diff1": 0.321372, "nauc_map_at_20_max": 0.087227, "nauc_map_at_20_std": -0.10095, "nauc_map_at_20_diff1": 0.321223, "nauc_map_at_25_max": 0.08824, "nauc_map_at_25_std": -0.10036, "nauc_map_at_25_diff1": 0.320865, "nauc_map_at_50_max": 0.090061, "nauc_map_at_50_std": -0.098551, "nauc_map_at_50_diff1": 0.320572, "nauc_map_at_100_max": 0.090449, "nauc_map_at_100_std": -0.097924, "nauc_map_at_100_diff1": 0.319716, "nauc_map_at_1000_max": 0.090203, "nauc_map_at_1000_std": -0.097779, "nauc_map_at_1000_diff1": 0.320097, "nauc_recall_at_1_max": 0.082733, "nauc_recall_at_1_std": -0.065071, "nauc_recall_at_1_diff1": 0.342807, "nauc_recall_at_3_max": 0.064545, "nauc_recall_at_3_std": -0.129078, "nauc_recall_at_3_diff1": 0.299179, "nauc_recall_at_5_max": 0.097418, "nauc_recall_at_5_std": -0.141027, "nauc_recall_at_5_diff1": 0.306242, "nauc_recall_at_10_max": 0.076917, "nauc_recall_at_10_std": -0.159114, "nauc_recall_at_10_diff1": 0.260748, "nauc_recall_at_20_max": 0.048466, "nauc_recall_at_20_std": -0.236489, "nauc_recall_at_20_diff1": 0.246993, "nauc_recall_at_25_max": 0.063871, "nauc_recall_at_25_std": -0.232352, "nauc_recall_at_25_diff1": 0.235629, "nauc_recall_at_50_max": 0.13413, "nauc_recall_at_50_std": -0.168856, "nauc_recall_at_50_diff1": 0.214558, "nauc_recall_at_100_max": 0.161662, "nauc_recall_at_100_std": -0.123286, "nauc_recall_at_100_diff1": 0.121854, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.080607, "nauc_precision_at_1_std": -0.059698, "nauc_precision_at_1_diff1": 0.33603, "nauc_precision_at_3_max": 0.0647, "nauc_precision_at_3_std": -0.12627, "nauc_precision_at_3_diff1": 0.293749, "nauc_precision_at_5_max": 0.103244, "nauc_precision_at_5_std": -0.13979, "nauc_precision_at_5_diff1": 0.307239, "nauc_precision_at_10_max": 0.096183, "nauc_precision_at_10_std": -0.147001, "nauc_precision_at_10_diff1": 0.262598, "nauc_precision_at_20_max": 0.069166, "nauc_precision_at_20_std": -0.224249, "nauc_precision_at_20_diff1": 0.24908, "nauc_precision_at_25_max": 0.090682, "nauc_precision_at_25_std": -0.21123, "nauc_precision_at_25_diff1": 0.233934, "nauc_precision_at_50_max": 0.165883, "nauc_precision_at_50_std": -0.139818, "nauc_precision_at_50_diff1": 0.204083, "nauc_precision_at_100_max": 0.197561, "nauc_precision_at_100_std": -0.092006, "nauc_precision_at_100_diff1": 0.109004, "nauc_precision_at_1000_max": 0.141147, "nauc_precision_at_1000_std": 0.27903, "nauc_precision_at_1000_diff1": -0.019728, "nauc_mrr_at_1_max": 0.080607, "nauc_mrr_at_1_std": -0.059698, "nauc_mrr_at_1_diff1": 0.33603, "nauc_mrr_at_3_max": 0.079768, "nauc_mrr_at_3_std": -0.087271, "nauc_mrr_at_3_diff1": 0.317621, "nauc_mrr_at_5_max": 0.089563, "nauc_mrr_at_5_std": -0.090047, "nauc_mrr_at_5_diff1": 0.321868, "nauc_mrr_at_10_max": 0.091044, "nauc_mrr_at_10_std": -0.0902, "nauc_mrr_at_10_diff1": 0.316755, "nauc_mrr_at_20_max": 0.0893, "nauc_mrr_at_20_std": -0.0953, "nauc_mrr_at_20_diff1": 0.316579, "nauc_mrr_at_25_max": 0.089804, "nauc_mrr_at_25_std": -0.095194, "nauc_mrr_at_25_diff1": 0.316288, "nauc_mrr_at_50_max": 0.091194, "nauc_mrr_at_50_std": -0.093801, "nauc_mrr_at_50_diff1": 0.316054, "nauc_mrr_at_100_max": 0.091524, "nauc_mrr_at_100_std": -0.093111, "nauc_mrr_at_100_diff1": 0.315202, "nauc_mrr_at_1000_max": 0.091279, "nauc_mrr_at_1000_std": -0.092981, "nauc_mrr_at_1000_diff1": 0.315588, "hit_rate_at_1": 0.22422, "hit_rate_at_3": 0.3565, "hit_rate_at_5": 0.42825, "hit_rate_at_10": 0.5157, "hit_rate_at_20": 0.59193, "hit_rate_at_25": 0.61211, "hit_rate_at_50": 0.6704, "hit_rate_at_100": 0.72646, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.224215, "R_cap_at_3": 0.353139, "R_cap_at_5": 0.423767, "R_cap_at_10": 0.506726, "R_cap_at_20": 0.58296, "R_cap_at_25": 0.60426, "R_cap_at_50": 0.661435, "R_cap_at_100": 0.71861, "R_cap_at_1000": 1.0, "main_score": 0.35639, "hf_subset": "adversarial", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 494.32412338256836, "kg_co2_emissions": null, "date": 1785924330.148546, "evaluation_phases": [ { "name": "Data loading", "start": 6.77257776260376e-06, "end": 54.77692121267319, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 54.77693569660187, "end": 54.77693808823824, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 54.80562189593911, "end": 63.324501134455204, "split": "test", "subset": "adversarial" }, { "name": "Scoring", "start": 63.324548833072186, "end": 63.61993822827935, "split": "test", "subset": "adversarial" } ] }