{"id": "eval-garden-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.148447, "router_ms": 0.011967}
{"id": "eval-garden-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [9.829537391662598, 8.905885696411133, -11.418170928955078, -11.429336547851562, -11.416326522827148, -11.383506774902344, -11.41354751586914, -11.452909469604492, -11.40498161315918, -11.423681259155273, -11.392634391784668, -11.419623374938965], "ranking": ["garden", "soil", "lantern-charge", "archive", "lantern", "keel", "junction", "quartz", "rivet-calibration", "rivet", "iris", "iris-network", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.697869, "router_ms": 0.007366}
{"id": "eval-garden-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.13962, "router_ms": 0.016673}
{"id": "eval-garden-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098399, "router_ms": 0.009784}
{"id": "eval-garden-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094618, "router_ms": 0.009192}
{"id": "eval-garden-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.093837, "router_ms": 0.008837}
{"id": "eval-garden-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103471, "router_ms": 0.008641}
{"id": "eval-garden-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.0952, "router_ms": 0.008644}
{"id": "eval-garden-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.952169895172119, -10.063556671142578, -5.017200469970703, -11.34083080291748, -11.358210563659668, -11.33510971069336, -10.223990440368652, -11.156427383422852, -11.10240650177002, -9.888599395751953, -10.790337562561035, -10.740549087524414], "ranking": ["garden", "soil", "lantern", "quartz", "lantern-charge", "archive", "rivet", "keel", "junction", "iris", "iris-network", "quartz-batch", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 106.74444, "router_ms": 0.007355}
{"id": "eval-garden-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.200594, "router_ms": 0.020052}
{"id": "eval-garden-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.137494, "router_ms": 0.011218}
{"id": "eval-garden-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.151841, "router_ms": 0.010573}
{"id": "eval-garden-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.139826, "router_ms": 0.010138}
{"id": "eval-garden-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.139017, "router_ms": 0.009833}
{"id": "eval-garden-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.139225, "router_ms": 0.009939}
{"id": "eval-garden-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.134734, "router_ms": 0.006619}
{"id": "eval-harbor-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105343, "router_ms": 0.009621}
{"id": "eval-harbor-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.099325, "router_ms": 0.009437}
{"id": "eval-harbor-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.097051, "router_ms": 0.008615}
{"id": "eval-harbor-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.095513, "router_ms": 0.008383}
{"id": "eval-harbor-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.096155, "router_ms": 0.0083}
{"id": "eval-harbor-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.096072, "router_ms": 0.008226}
{"id": "eval-harbor-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091942, "router_ms": 0.005158}
{"id": "eval-harbor-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [9.691278457641602, 5.049304962158203, -11.02155590057373, -11.323006629943848, -10.970941543579102, -11.106721878051758, -11.357669830322266, -9.4385404586792, -8.870769500732422, -10.29703426361084, -10.645711898803711, -11.19121265411377], "ranking": ["harbor", "engine", "orbit", "mirror", "lantern", "lantern-charge", "cargo", "frost", "garden", "soil", "iris-network", "iris", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 87.155941, "router_ms": 0.005002}
{"id": "eval-harbor-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.143606, "router_ms": 0.018978}
{"id": "eval-harbor-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122772, "router_ms": 0.010305}
{"id": "eval-harbor-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.088716, "router_ms": 0.009121}
{"id": "eval-harbor-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.097557, "router_ms": 0.008926}
{"id": "eval-harbor-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.087869, "router_ms": 0.00837}
{"id": "eval-harbor-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.083044, "router_ms": 0.005195}
{"id": "eval-harbor-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.506385803222656, 5.286466598510742, -11.294589042663574, -11.288810729980469, -11.326292037963867, -11.278151512145996, -11.393287658691406, -11.377354621887207, -11.325961112976074, -11.262049674987793, -11.386184692382812, -11.274985313415527], "ranking": ["harbor", "engine", "lantern-charge", "mirror", "beacon", "frost", "quartz", "lantern", "drone", "iris-network", "archive", "export", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 88.473922, "router_ms": 0.005071}
{"id": "eval-harbor-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.139671, "router_ms": 0.017276}
{"id": "eval-iris-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.146423, "router_ms": 0.011045}
{"id": "eval-iris-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.126131, "router_ms": 0.0111}
{"id": "eval-iris-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.122978, "router_ms": 0.009908}
{"id": "eval-iris-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.121308, "router_ms": 0.009352}
{"id": "eval-iris-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.117519, "router_ms": 0.006281}
{"id": "eval-iris-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.322504043579102, 7.934057235717773, -11.388988494873047, -11.355183601379395, -11.403718948364258, -11.435961723327637, -11.38909912109375, -11.349196434020996, -11.307306289672852, -11.441774368286133, -11.316682815551758, -11.352814674377441], "ranking": ["iris", "iris-network", "junction", "garden", "mirror", "lantern", "beacon", "frost", "lantern-charge", "drone", "quartz-batch", "keel", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 95.321057, "router_ms": 0.005917}
{"id": "eval-iris-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.18626, "router_ms": 0.017714}
{"id": "eval-iris-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.120243, "router_ms": 0.010237}
{"id": "eval-iris-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11511, "router_ms": 0.009824}
{"id": "eval-iris-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111923, "router_ms": 0.01002}
{"id": "eval-iris-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107399, "router_ms": 0.008603}
{"id": "eval-iris-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.104259, "router_ms": 0.005795}
{"id": "eval-iris-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [9.597367286682129, -7.179789066314697, -11.409870147705078, -10.546661376953125, -11.468208312988281, -11.472970962524414, -11.443428993225098, -11.378433227539062, -11.446844100952148, -11.410990715026855, -11.459843635559082, -11.413043022155762], "ranking": ["iris", "iris-network", "prism", "frost", "garden", "lantern", "lantern-charge", "harbor", "cargo", "orbit", "archive", "export", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 103.47792, "router_ms": 0.00561}
{"id": "eval-iris-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.155543, "router_ms": 0.017222}
{"id": "eval-iris-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11324, "router_ms": 0.010127}
{"id": "eval-iris-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11068, "router_ms": 0.009597}
{"id": "eval-junction-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10918, "router_ms": 0.00939}
{"id": "eval-junction-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106599, "router_ms": 0.009008}
{"id": "eval-junction-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.099462, "router_ms": 0.005754}
{"id": "eval-junction-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [6.33122444152832, -10.568521499633789, -11.287206649780273, -11.336028099060059, -11.451650619506836, -11.436971664428711, -10.63071346282959, -10.962664604187012, -10.813403129577637, -11.352925300598145, -10.614968299865723, -11.220610618591309], "ranking": ["junction", "lantern-charge", "mirror", "cargo", "orbit", "lantern", "keel", "frost", "garden", "soil", "iris", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.09943, "router_ms": 0.005319}
{"id": "eval-junction-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.156448, "router_ms": 0.017173}
{"id": "eval-junction-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111895, "router_ms": 0.010169}
{"id": "eval-junction-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109618, "router_ms": 0.009342}
{"id": "eval-junction-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109648, "router_ms": 0.008729}
{"id": "eval-junction-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129604, "router_ms": 0.009772}
{"id": "eval-junction-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109076, "router_ms": 0.006357}
{"id": "eval-junction-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.817267417907715, -11.398323059082031, -11.459901809692383, -11.405369758605957, -11.39989185333252, -11.405868530273438, -11.431496620178223, -11.44167423248291, -11.441703796386719, -11.483541488647461, -11.41031551361084, -11.399264335632324], "ranking": ["junction", "beacon", "lantern-charge", "soil", "orbit", "engine", "lantern", "frost", "drone", "harbor", "export", "iris-network", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.916623, "router_ms": 0.005793}
{"id": "eval-junction-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.173062, "router_ms": 0.017931}
{"id": "eval-junction-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12939, "router_ms": 0.011147}
{"id": "eval-junction-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11685, "router_ms": 0.009816}
{"id": "eval-junction-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115141, "router_ms": 0.008936}
{"id": "eval-junction-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114998, "router_ms": 0.009159}
{"id": "eval-keel-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098032, "router_ms": 0.006315}
{"id": "eval-keel-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.503390312194824, -7.309561252593994, -10.551677703857422, -11.386517524719238, -10.565544128417969, -10.63076400756836, -11.392157554626465, -11.005291938781738, -11.023717880249023, -10.225462913513184, -11.327591896057129, -10.9865083694458], "ranking": ["keel", "rivet-calibration", "frost", "engine", "archive", "beacon", "harbor", "drone", "export", "garden", "prism", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 96.153711, "router_ms": 0.005527}
{"id": "eval-keel-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.138645, "router_ms": 0.016386}
{"id": "eval-keel-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094611, "router_ms": 0.009144}
{"id": "eval-keel-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.092947, "router_ms": 0.008556}
{"id": "eval-keel-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091526, "router_ms": 0.008207}
{"id": "eval-keel-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091015, "router_ms": 0.008327}
{"id": "eval-keel-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.09023, "router_ms": 0.007755}
{"id": "eval-keel-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.379083633422852, -11.161659240722656, -11.338848114013672, -11.390254974365234, -10.53793716430664, -11.310693740844727, -11.470420837402344, -11.427215576171875, -11.357369422912598, -11.317784309387207, -11.438980102539062, -11.386247634887695], "ranking": ["keel", "archive", "rivet-calibration", "beacon", "frost", "engine", "export", "harbor", "prism", "drone", "garden", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.428499, "router_ms": 0.005512}
{"id": "eval-keel-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.148825, "router_ms": 0.016531}
{"id": "eval-keel-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.095311, "router_ms": 0.009403}
{"id": "eval-keel-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.092665, "router_ms": 0.008823}
{"id": "eval-keel-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091032, "router_ms": 0.008081}
{"id": "eval-keel-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.090259, "router_ms": 0.008077}
{"id": "eval-keel-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.089603, "router_ms": 0.007657}
{"id": "eval-keel-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.086192, "router_ms": 0.004876}
{"id": "eval-lantern-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113148, "router_ms": 0.009489}
{"id": "eval-lantern-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103368, "router_ms": 0.00878}
{"id": "eval-lantern-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103534, "router_ms": 0.008718}
{"id": "eval-lantern-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.102704, "router_ms": 0.00845}
{"id": "eval-lantern-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.101464, "router_ms": 0.00828}
{"id": "eval-lantern-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.101043, "router_ms": 0.007947}
{"id": "eval-lantern-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098694, "router_ms": 0.005334}
{"id": "eval-lantern-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.077771186828613, -10.966012954711914, -11.201990127563477, -11.393867492675781, -11.288263320922852, -11.190375328063965, -11.290112495422363, -11.279410362243652, -10.673552513122559, -11.278423309326172, -11.345672607421875, -11.221471786499023], "ranking": ["lantern", "lantern-charge", "iris-network", "iris", "frost", "beacon", "soil", "orbit", "garden", "mirror", "drone", "cargo", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 91.435423, "router_ms": 0.005196}
{"id": "eval-lantern-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.145299, "router_ms": 0.016676}
{"id": "eval-lantern-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.102766, "router_ms": 0.010149}
{"id": "eval-lantern-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.108913, "router_ms": 0.009199}
{"id": "eval-lantern-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.093948, "router_ms": 0.009119}
{"id": "eval-lantern-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.092697, "router_ms": 0.008231}
{"id": "eval-lantern-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.089965, "router_ms": 0.005304}
{"id": "eval-lantern-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.703226089477539, 7.691359043121338, -11.438968658447266, -10.154081344604492, -11.016915321350098, -10.71131420135498, -11.458416938781738, -11.044757843017578, -11.049427032470703, -11.024188995361328, -10.528362274169922, -10.827560424804688], "ranking": ["lantern-charge", "lantern", "junction", "beacon", "garden", "nova", "keel", "rivet-calibration", "rivet", "archive", "iris", "iris-network", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 105.857165, "router_ms": 0.004874}
{"id": "eval-lantern-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.153531, "router_ms": 0.017378}
{"id": "eval-mirror-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.150227, "router_ms": 0.0123}
{"id": "eval-mirror-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12659, "router_ms": 0.009769}
{"id": "eval-mirror-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125858, "router_ms": 0.010176}
{"id": "eval-mirror-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125382, "router_ms": 0.009131}
{"id": "eval-mirror-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119907, "router_ms": 0.005856}
{"id": "eval-mirror-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.31332540512085, -11.464130401611328, -11.357179641723633, -11.377193450927734, -11.460406303405762, -11.458881378173828, -11.47458267211914, -11.455571174621582, -11.463949203491211, -11.457032203674316, -11.473447799682617, -11.469704627990723], "ranking": ["mirror", "prism", "rivet", "lantern-charge", "frost", "harbor", "cargo", "junction", "nova", "beacon", "drone", "lantern", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 98.592954, "router_ms": 0.005949}
{"id": "eval-mirror-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.185233, "router_ms": 0.017718}
{"id": "eval-mirror-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.127855, "router_ms": 0.010366}
{"id": "eval-mirror-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108014, "router_ms": 0.010678}
{"id": "eval-mirror-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10229, "router_ms": 0.009314}
{"id": "eval-mirror-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103076, "router_ms": 0.009309}
{"id": "eval-mirror-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.095508, "router_ms": 0.005559}
{"id": "eval-mirror-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.968134880065918, -9.22532844543457, -11.435161590576172, -11.465716361999512, -11.459169387817383, -11.448808670043945, -11.410764694213867, -11.483901977539062, -11.453641891479492, -11.447628021240234, -11.434931755065918, -11.456716537475586], "ranking": ["mirror", "quartz", "harbor", "archive", "frost", "lantern-charge", "export", "lantern", "orbit", "beacon", "drone", "iris-network", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 92.355189, "router_ms": 0.005173}
{"id": "eval-mirror-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.163738, "router_ms": 0.017206}
{"id": "eval-mirror-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107399, "router_ms": 0.009792}
{"id": "eval-mirror-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.102363, "router_ms": 0.009246}
{"id": "eval-nova-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.089115, "router_ms": 0.008566}
{"id": "eval-nova-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.071273, "router_ms": 0.007327}
{"id": "eval-nova-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.071006, "router_ms": 0.004957}
{"id": "eval-nova-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [5.650143623352051, -11.470599174499512, -11.343707084655762, -11.446039199829102, -11.457576751708984, -11.356437683105469, -11.46668815612793, -11.40383529663086, -11.37934398651123, -11.419157981872559, -11.388242721557617, -11.457596778869629], "ranking": ["nova", "beacon", "engine", "garden", "iris", "frost", "harbor", "cargo", "drone", "iris-network", "export", "archive", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 92.327257, "router_ms": 0.004432}
{"id": "eval-nova-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112264, "router_ms": 0.014343}
{"id": "eval-nova-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.072816, "router_ms": 0.007944}
{"id": "eval-nova-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.0702, "router_ms": 0.00734}
{"id": "eval-nova-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.069646, "router_ms": 0.00703}
{"id": "eval-nova-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.130042, "router_ms": 0.009912}
{"id": "eval-nova-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105676, "router_ms": 0.006233}
{"id": "eval-nova-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.619958877563477, -11.372095108032227, -10.647622108459473, -10.998038291931152, -10.915349960327148, -11.330671310424805, -10.63299560546875, -11.234281539916992, -11.126276969909668, -11.4449462890625, -11.010820388793945, -11.30985164642334], "ranking": ["nova", "beacon", "rivet", "frost", "cargo", "lantern", "harbor", "export", "lantern-charge", "drone", "prism", "iris-network", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 99.270004, "router_ms": 0.006067}
{"id": "eval-nova-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.160874, "router_ms": 0.016326}
{"id": "eval-nova-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.130439, "router_ms": 0.010431}
{"id": "eval-nova-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119622, "router_ms": 0.009326}
{"id": "eval-nova-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108542, "router_ms": 0.008802}
{"id": "eval-nova-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107503, "router_ms": 0.008702}
{"id": "eval-orbit-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125221, "router_ms": 0.006562}
{"id": "eval-orbit-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.4112772941589355, -11.332706451416016, -10.062492370605469, -10.203400611877441, -9.598102569580078, -10.79186725616455, -11.121228218078613, -10.526987075805664, -6.598150730133057, -7.125789642333984, -10.53573989868164, -11.27670669555664], "ranking": ["orbit", "engine", "export", "harbor", "quartz", "mirror", "drone", "frost", "archive", "beacon", "garden", "cargo", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 102.702371, "router_ms": 0.005828}
{"id": "eval-orbit-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.180819, "router_ms": 0.017762}
{"id": "eval-orbit-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.223142, "router_ms": 0.010965}
{"id": "eval-orbit-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121553, "router_ms": 0.00963}
{"id": "eval-orbit-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.14613, "router_ms": 0.009559}
{"id": "eval-orbit-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119714, "router_ms": 0.008935}
{"id": "eval-orbit-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.131218, "router_ms": 0.008903}
{"id": "eval-orbit-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [6.3033576011657715, -11.41659164428711, -11.214947700500488, -11.41208267211914, -11.312108039855957, -11.461721420288086, -11.430337905883789, -11.417841911315918, -11.446528434753418, -11.4389066696167, -11.460297584533691, -11.443344116210938], "ranking": ["orbit", "iris", "lantern", "rivet-calibration", "lantern-charge", "frost", "nova", "iris-network", "prism", "harbor", "cargo", "quartz-batch", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 142.617891, "router_ms": 0.007206}
{"id": "eval-orbit-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.173575, "router_ms": 0.018007}
{"id": "eval-orbit-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.294072, "router_ms": 0.055568}
{"id": "eval-orbit-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.209038, "router_ms": 0.017161}
{"id": "eval-orbit-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.199235, "router_ms": 0.015236}
{"id": "eval-orbit-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.230075, "router_ms": 0.026611}
{"id": "eval-orbit-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.133962, "router_ms": 0.011557}
{"id": "eval-orbit-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119747, "router_ms": 0.006801}
{"id": "eval-prism-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107652, "router_ms": 0.010231}
{"id": "eval-prism-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.100968, "router_ms": 0.00881}
{"id": "eval-prism-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098867, "router_ms": 0.008313}
{"id": "eval-prism-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098901, "router_ms": 0.008353}
{"id": "eval-prism-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.097738, "router_ms": 0.007935}
{"id": "eval-prism-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.097026, "router_ms": 0.007669}
{"id": "eval-prism-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094497, "router_ms": 0.004963}
{"id": "eval-prism-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [10.058465003967285, -11.35808277130127, -11.395358085632324, -11.298918724060059, -11.376766204833984, -11.360922813415527, -11.384462356567383, -11.3346529006958, -11.421213150024414, -11.38614273071289, -11.231120109558105, -11.291454315185547], "ranking": ["prism", "frost", "garden", "iris", "beacon", "rivet-calibration", "harbor", "drone", "archive", "export", "engine", "cargo", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 105.170797, "router_ms": 0.005098}
{"id": "eval-prism-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.17048, "router_ms": 0.017207}
{"id": "eval-prism-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126215, "router_ms": 0.011226}
{"id": "eval-prism-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125378, "router_ms": 0.009703}
{"id": "eval-prism-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113744, "router_ms": 0.009434}
{"id": "eval-prism-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112928, "router_ms": 0.008935}
{"id": "eval-prism-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108005, "router_ms": 0.006053}
{"id": "eval-prism-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [7.783456325531006, -11.350871086120605, -11.440305709838867, -11.468302726745605, -11.444717407226562, -11.468291282653809, -11.438386917114258, -11.472097396850586, -11.480454444885254, -11.463541984558105, -11.490352630615234, -11.461370468139648], "ranking": ["prism", "rivet", "cargo", "quartz", "frost", "lantern", "harbor", "drone", "nova", "beacon", "export", "iris-network", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 133.570185, "router_ms": 0.005806}
{"id": "eval-prism-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.259129, "router_ms": 0.031563}
{"id": "eval-quartz-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.136339, "router_ms": 0.01133}
{"id": "eval-quartz-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117763, "router_ms": 0.009862}
{"id": "eval-quartz-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115849, "router_ms": 0.009463}
{"id": "eval-quartz-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114759, "router_ms": 0.008857}
{"id": "eval-quartz-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11124, "router_ms": 0.005926}
{"id": "eval-quartz-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [9.253325462341309, -0.6516895294189453, -7.4880571365356445, -11.421065330505371, -11.176493644714355, -11.252103805541992, -11.449766159057617, -11.421207427978516, -11.464003562927246, -11.286712646484375, -11.034472465515137, -11.463125228881836], "ranking": ["quartz", "quartz-batch", "mirror", "lantern-charge", "drone", "beacon", "lantern", "frost", "harbor", "export", "archive", "iris-network", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 109.79965, "router_ms": 0.005765}
{"id": "eval-quartz-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.159955, "router_ms": 0.015812}
{"id": "eval-quartz-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118785, "router_ms": 0.010177}
{"id": "eval-quartz-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.130258, "router_ms": 0.010209}
{"id": "eval-quartz-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.123174, "router_ms": 0.009553}
{"id": "eval-quartz-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122421, "router_ms": 0.009041}
{"id": "eval-quartz-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119923, "router_ms": 0.006125}
{"id": "eval-quartz-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [6.443126201629639, -4.9894328117370605, -11.329298973083496, -10.995671272277832, -10.901998519897461, -8.954572677612305, -10.973638534545898, -9.095609664916992, -10.860000610351562, -10.853055000305176, -10.665283203125, -10.9835786819458], "ranking": ["quartz", "quartz-batch", "orbit", "junction", "drone", "frost", "engine", "export", "soil", "harbor", "beacon", "cargo", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 113.934846, "router_ms": 0.006011}
{"id": "eval-quartz-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.170231, "router_ms": 0.016262}
{"id": "eval-quartz-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125497, "router_ms": 0.010205}
{"id": "eval-quartz-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.124366, "router_ms": 0.00985}
{"id": "eval-rivet-1", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115912, "router_ms": 0.010024}
{"id": "eval-rivet-1", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109017, "router_ms": 0.008655}
{"id": "eval-rivet-1", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125012, "router_ms": 0.006036}
{"id": "eval-rivet-1", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.032523155212402, 5.0930986404418945, -11.045774459838867, -11.111373901367188, -11.299398422241211, -11.02957820892334, -10.90796947479248, -11.426010131835938, -10.650565147399902, -10.10616683959961, -10.731083869934082, -10.548370361328125], "ranking": ["rivet", "rivet-calibration", "lantern-charge", "mirror", "lantern", "archive", "harbor", "export", "frost", "drone", "beacon", "iris-network", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 109.228003, "router_ms": 0.005986}
{"id": "eval-rivet-1", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.158396, "router_ms": 0.017425}
{"id": "eval-rivet-1", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113189, "router_ms": 0.010087}
{"id": "eval-rivet-1", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108608, "router_ms": 0.00914}
{"id": "eval-rivet-1", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107032, "router_ms": 0.00922}
{"id": "eval-rivet-2", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122727, "router_ms": 0.010929}
{"id": "eval-rivet-2", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11202, "router_ms": 0.006777}
{"id": "eval-rivet-2", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [8.027447700500488, -11.266072273254395, -11.40830135345459, 0.2866683900356293, -11.423832893371582, -11.390559196472168, -11.425186157226562, -11.415412902832031, -11.410115242004395, -11.452159881591797, -11.419374465942383, -11.453349113464355], "ranking": ["rivet", "rivet-calibration", "prism", "frost", "quartz", "beacon", "cargo", "harbor", "nova", "drone", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 105.423177, "router_ms": 0.006162}
{"id": "eval-rivet-2", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.168936, "router_ms": 0.018355}
{"id": "eval-rivet-2", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.13753, "router_ms": 0.010037}
{"id": "eval-rivet-2", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109619, "router_ms": 0.00933}
{"id": "eval-rivet-2", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107969, "router_ms": 0.008774}
{"id": "eval-rivet-2", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126458, "router_ms": 0.008959}
{"id": "eval-unknown-weather", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.092624, "router_ms": 0.00599}
{"id": "eval-unknown-weather", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [-11.279678344726562, -11.37423038482666, -11.425868034362793, -10.790202140808105, -11.403650283813477, -11.357444763183594, -11.363469123840332, -11.213229179382324, -11.39967155456543, -11.318207740783691, -11.420628547668457, -11.412744522094727], "ranking": ["prism", "garden", "keel", "lantern", "junction", "lantern-charge", "nova", "iris-network", "iris", "rivet-calibration", "archive", "rivet", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 93.28497, "router_ms": 0.005063}
{"id": "eval-unknown-weather", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.145494, "router_ms": 0.015942}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.100863, "router_ms": 0.010464}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.103125, "router_ms": 0.008435}
{"id": "eval-unknown-weather", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.091274, "router_ms": 0.008206}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.0885, "router_ms": 0.008047}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.087646, "router_ms": 0.007873}
{"id": "eval-unknown-captain", "policy": "always-12", "repetition": 0, "depth": 12, "logits": [-11.368772506713867, -11.419684410095215, -11.30733871459961, -11.103347778320312, -11.180445671081543, -11.462408065795898, -11.390913009643555, -11.450128555297852, -11.37920093536377, -11.34627914428711, -11.41447925567627, -11.448532104492188], "ranking": ["iris-network", "iris", "garden", "soil", "keel", "lantern-charge", "lantern", "mirror", "frost", "junction", "orbit", "cargo", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 99.69092, "router_ms": 0.006074}
{"id": "eval-unknown-captain", "policy": "router-b4-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.169908, "router_ms": 0.016785}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.110355, "router_ms": 0.009916}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.10623, "router_ms": 0.009096}
{"id": "eval-unknown-captain", "policy": "router-b12-p0", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.107074, "router_ms": 0.008409}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.01", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.10395, "router_ms": 0.008272}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.04", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.103241, "router_ms": 0.007961}
{"id": "eval-unknown-captain", "policy": "never", "repetition": 0, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.100071, "router_ms": 0.005341}
{"id": "eval-garden-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [9.829537391662598, 8.905885696411133, -11.418170928955078, -11.429336547851562, -11.416326522827148, -11.383506774902344, -11.41354751586914, -11.452909469604492, -11.40498161315918, -11.423681259155273, -11.392634391784668, -11.419623374938965], "ranking": ["garden", "soil", "lantern-charge", "archive", "lantern", "keel", "junction", "quartz", "rivet-calibration", "rivet", "iris", "iris-network", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 103.805215, "router_ms": 0.006542}
{"id": "eval-garden-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.156398, "router_ms": 0.01651}
{"id": "eval-garden-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112697, "router_ms": 0.010466}
{"id": "eval-garden-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113568, "router_ms": 0.009026}
{"id": "eval-garden-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098789, "router_ms": 0.008474}
{"id": "eval-garden-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098786, "router_ms": 0.008119}
{"id": "eval-garden-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098865, "router_ms": 0.008175}
{"id": "eval-garden-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.093897, "router_ms": 0.005184}
{"id": "eval-garden-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.148803, "router_ms": 0.01028}
{"id": "eval-garden-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.203963, "router_ms": 0.05629}
{"id": "eval-garden-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.171752, "router_ms": 0.013322}
{"id": "eval-garden-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.145454, "router_ms": 0.010652}
{"id": "eval-garden-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.140788, "router_ms": 0.010009}
{"id": "eval-garden-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.140807, "router_ms": 0.009667}
{"id": "eval-garden-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.137554, "router_ms": 0.006891}
{"id": "eval-garden-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.952169895172119, -10.063556671142578, -5.017200469970703, -11.34083080291748, -11.358210563659668, -11.33510971069336, -10.223990440368652, -11.156427383422852, -11.10240650177002, -9.888599395751953, -10.790337562561035, -10.740549087524414], "ranking": ["garden", "soil", "lantern", "quartz", "lantern-charge", "archive", "rivet", "keel", "junction", "iris", "iris-network", "quartz-batch", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 112.169007, "router_ms": 0.006682}
{"id": "eval-harbor-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.144167, "router_ms": 0.015768}
{"id": "eval-harbor-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.101089, "router_ms": 0.009466}
{"id": "eval-harbor-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.100143, "router_ms": 0.00852}
{"id": "eval-harbor-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098321, "router_ms": 0.008581}
{"id": "eval-harbor-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.096968, "router_ms": 0.008182}
{"id": "eval-harbor-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094766, "router_ms": 0.00536}
{"id": "eval-harbor-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [9.691278457641602, 5.049304962158203, -11.02155590057373, -11.323006629943848, -10.970941543579102, -11.106721878051758, -11.357669830322266, -9.4385404586792, -8.870769500732422, -10.29703426361084, -10.645711898803711, -11.19121265411377], "ranking": ["harbor", "engine", "orbit", "mirror", "lantern", "lantern-charge", "cargo", "frost", "garden", "soil", "iris-network", "iris", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 101.782766, "router_ms": 0.004987}
{"id": "eval-harbor-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.151498, "router_ms": 0.016821}
{"id": "eval-harbor-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103894, "router_ms": 0.010968}
{"id": "eval-harbor-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105525, "router_ms": 0.008919}
{"id": "eval-harbor-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.086434, "router_ms": 0.008447}
{"id": "eval-harbor-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.086481, "router_ms": 0.007994}
{"id": "eval-harbor-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.083863, "router_ms": 0.00531}
{"id": "eval-harbor-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.506385803222656, 5.286466598510742, -11.294589042663574, -11.288810729980469, -11.326292037963867, -11.278151512145996, -11.393287658691406, -11.377354621887207, -11.325961112976074, -11.262049674987793, -11.386184692382812, -11.274985313415527], "ranking": ["harbor", "engine", "lantern-charge", "mirror", "beacon", "frost", "quartz", "lantern", "drone", "iris-network", "archive", "export", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 93.15996, "router_ms": 0.004831}
{"id": "eval-harbor-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.153671, "router_ms": 0.017029}
{"id": "eval-harbor-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.092538, "router_ms": 0.009582}
{"id": "eval-iris-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.203754, "router_ms": 0.016744}
{"id": "eval-iris-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.128338, "router_ms": 0.010217}
{"id": "eval-iris-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.145481, "router_ms": 0.010245}
{"id": "eval-iris-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.177389, "router_ms": 0.006675}
{"id": "eval-iris-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.322504043579102, 7.934057235717773, -11.388988494873047, -11.355183601379395, -11.403718948364258, -11.435961723327637, -11.38909912109375, -11.349196434020996, -11.307306289672852, -11.441774368286133, -11.316682815551758, -11.352814674377441], "ranking": ["iris", "iris-network", "junction", "garden", "mirror", "lantern", "beacon", "frost", "lantern-charge", "drone", "quartz-batch", "keel", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 102.901501, "router_ms": 0.005995}
{"id": "eval-iris-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.168835, "router_ms": 0.016615}
{"id": "eval-iris-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.126605, "router_ms": 0.010111}
{"id": "eval-iris-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.116546, "router_ms": 0.009112}
{"id": "eval-iris-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113719, "router_ms": 0.009326}
{"id": "eval-iris-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108032, "router_ms": 0.00868}
{"id": "eval-iris-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103961, "router_ms": 0.00577}
{"id": "eval-iris-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [9.597367286682129, -7.179789066314697, -11.409870147705078, -10.546661376953125, -11.468208312988281, -11.472970962524414, -11.443428993225098, -11.378433227539062, -11.446844100952148, -11.410990715026855, -11.459843635559082, -11.413043022155762], "ranking": ["iris", "iris-network", "prism", "frost", "garden", "lantern", "lantern-charge", "harbor", "cargo", "orbit", "archive", "export", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 103.604816, "router_ms": 0.005366}
{"id": "eval-iris-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.165845, "router_ms": 0.017472}
{"id": "eval-iris-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119248, "router_ms": 0.010498}
{"id": "eval-iris-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121251, "router_ms": 0.013979}
{"id": "eval-iris-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113236, "router_ms": 0.008935}
{"id": "eval-junction-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118149, "router_ms": 0.009385}
{"id": "eval-junction-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112435, "router_ms": 0.005873}
{"id": "eval-junction-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [6.33122444152832, -10.568521499633789, -11.287206649780273, -11.336028099060059, -11.451650619506836, -11.436971664428711, -10.63071346282959, -10.962664604187012, -10.813403129577637, -11.352925300598145, -10.614968299865723, -11.220610618591309], "ranking": ["junction", "lantern-charge", "mirror", "cargo", "orbit", "lantern", "keel", "frost", "garden", "soil", "iris", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 106.422704, "router_ms": 0.005407}
{"id": "eval-junction-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.187404, "router_ms": 0.019577}
{"id": "eval-junction-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.157843, "router_ms": 0.009567}
{"id": "eval-junction-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.146895, "router_ms": 0.015205}
{"id": "eval-junction-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.110995, "router_ms": 0.009793}
{"id": "eval-junction-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.156158, "router_ms": 0.026222}
{"id": "eval-junction-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.143271, "router_ms": 0.009498}
{"id": "eval-junction-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.817267417907715, -11.398323059082031, -11.459901809692383, -11.405369758605957, -11.39989185333252, -11.405868530273438, -11.431496620178223, -11.44167423248291, -11.441703796386719, -11.483541488647461, -11.41031551361084, -11.399264335632324], "ranking": ["junction", "beacon", "lantern-charge", "soil", "orbit", "engine", "lantern", "frost", "drone", "harbor", "export", "iris-network", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 106.361323, "router_ms": 0.006532}
{"id": "eval-junction-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.171207, "router_ms": 0.017092}
{"id": "eval-junction-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129641, "router_ms": 0.010812}
{"id": "eval-junction-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119487, "router_ms": 0.010446}
{"id": "eval-junction-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.116216, "router_ms": 0.009385}
{"id": "eval-junction-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113914, "router_ms": 0.009023}
{"id": "eval-junction-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113658, "router_ms": 0.008541}
{"id": "eval-keel-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.503390312194824, -7.309561252593994, -10.551677703857422, -11.386517524719238, -10.565544128417969, -10.63076400756836, -11.392157554626465, -11.005291938781738, -11.023717880249023, -10.225462913513184, -11.327591896057129, -10.9865083694458], "ranking": ["keel", "rivet-calibration", "frost", "engine", "archive", "beacon", "harbor", "drone", "export", "garden", "prism", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 103.401462, "router_ms": 0.005694}
{"id": "eval-keel-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.147658, "router_ms": 0.01668}
{"id": "eval-keel-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10215, "router_ms": 0.009754}
{"id": "eval-keel-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108842, "router_ms": 0.008551}
{"id": "eval-keel-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091686, "router_ms": 0.007964}
{"id": "eval-keel-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.090521, "router_ms": 0.008175}
{"id": "eval-keel-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.214985, "router_ms": 0.045817}
{"id": "eval-keel-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.152241, "router_ms": 0.015112}
{"id": "eval-keel-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.23753, "router_ms": 0.021545}
{"id": "eval-keel-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.173956, "router_ms": 0.016789}
{"id": "eval-keel-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.165044, "router_ms": 0.024205}
{"id": "eval-keel-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126181, "router_ms": 0.010162}
{"id": "eval-keel-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117129, "router_ms": 0.013693}
{"id": "eval-keel-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094767, "router_ms": 0.009355}
{"id": "eval-keel-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.173306, "router_ms": 0.015934}
{"id": "eval-keel-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.379083633422852, -11.161659240722656, -11.338848114013672, -11.390254974365234, -10.53793716430664, -11.310693740844727, -11.470420837402344, -11.427215576171875, -11.357369422912598, -11.317784309387207, -11.438980102539062, -11.386247634887695], "ranking": ["keel", "archive", "rivet-calibration", "beacon", "frost", "engine", "export", "harbor", "prism", "drone", "garden", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 115.735412, "router_ms": 0.049879}
{"id": "eval-lantern-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.184445, "router_ms": 0.020017}
{"id": "eval-lantern-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.138421, "router_ms": 0.010238}
{"id": "eval-lantern-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107916, "router_ms": 0.009211}
{"id": "eval-lantern-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10504, "router_ms": 0.00903}
{"id": "eval-lantern-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10616, "router_ms": 0.008509}
{"id": "eval-lantern-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.100953, "router_ms": 0.00552}
{"id": "eval-lantern-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.077771186828613, -10.966012954711914, -11.201990127563477, -11.393867492675781, -11.288263320922852, -11.190375328063965, -11.290112495422363, -11.279410362243652, -10.673552513122559, -11.278423309326172, -11.345672607421875, -11.221471786499023], "ranking": ["lantern", "lantern-charge", "iris-network", "iris", "frost", "beacon", "soil", "orbit", "garden", "mirror", "drone", "cargo", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 112.433902, "router_ms": 0.00557}
{"id": "eval-lantern-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.151868, "router_ms": 0.016619}
{"id": "eval-lantern-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.110221, "router_ms": 0.010463}
{"id": "eval-lantern-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.101659, "router_ms": 0.009563}
{"id": "eval-lantern-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.10061, "router_ms": 0.009148}
{"id": "eval-lantern-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.099485, "router_ms": 0.00862}
{"id": "eval-lantern-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.096694, "router_ms": 0.00566}
{"id": "eval-lantern-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.703226089477539, 7.691359043121338, -11.438968658447266, -10.154081344604492, -11.016915321350098, -10.71131420135498, -11.458416938781738, -11.044757843017578, -11.049427032470703, -11.024188995361328, -10.528362274169922, -10.827560424804688], "ranking": ["lantern-charge", "lantern", "junction", "beacon", "garden", "nova", "keel", "rivet-calibration", "rivet", "archive", "iris", "iris-network", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 139.369145, "router_ms": 0.005295}
{"id": "eval-lantern-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.266695, "router_ms": 0.023152}
{"id": "eval-lantern-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.139388, "router_ms": 0.029011}
{"id": "eval-mirror-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.212021, "router_ms": 0.01759}
{"id": "eval-mirror-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.136856, "router_ms": 0.01087}
{"id": "eval-mirror-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126966, "router_ms": 0.009287}
{"id": "eval-mirror-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121434, "router_ms": 0.006246}
{"id": "eval-mirror-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.31332540512085, -11.464130401611328, -11.357179641723633, -11.377193450927734, -11.460406303405762, -11.458881378173828, -11.47458267211914, -11.455571174621582, -11.463949203491211, -11.457032203674316, -11.473447799682617, -11.469704627990723], "ranking": ["mirror", "prism", "rivet", "lantern-charge", "frost", "harbor", "cargo", "junction", "nova", "beacon", "drone", "lantern", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 126.577752, "router_ms": 0.023163}
{"id": "eval-mirror-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.173881, "router_ms": 0.017542}
{"id": "eval-mirror-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126656, "router_ms": 0.010004}
{"id": "eval-mirror-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.124427, "router_ms": 0.009783}
{"id": "eval-mirror-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.156106, "router_ms": 0.017737}
{"id": "eval-mirror-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10431, "router_ms": 0.009748}
{"id": "eval-mirror-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.099657, "router_ms": 0.00624}
{"id": "eval-mirror-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.968134880065918, -9.22532844543457, -11.435161590576172, -11.465716361999512, -11.459169387817383, -11.448808670043945, -11.410764694213867, -11.483901977539062, -11.453641891479492, -11.447628021240234, -11.434931755065918, -11.456716537475586], "ranking": ["mirror", "quartz", "harbor", "archive", "frost", "lantern-charge", "export", "lantern", "orbit", "beacon", "drone", "iris-network", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 107.78837, "router_ms": 0.005608}
{"id": "eval-mirror-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.140148, "router_ms": 0.014903}
{"id": "eval-mirror-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.09819, "router_ms": 0.009252}
{"id": "eval-mirror-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094856, "router_ms": 0.008377}
{"id": "eval-mirror-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.102314, "router_ms": 0.008775}
{"id": "eval-nova-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.079532, "router_ms": 0.00836}
{"id": "eval-nova-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.073309, "router_ms": 0.005005}
{"id": "eval-nova-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [5.650143623352051, -11.470599174499512, -11.343707084655762, -11.446039199829102, -11.457576751708984, -11.356437683105469, -11.46668815612793, -11.40383529663086, -11.37934398651123, -11.419157981872559, -11.388242721557617, -11.457596778869629], "ranking": ["nova", "beacon", "engine", "garden", "iris", "frost", "harbor", "cargo", "drone", "iris-network", "export", "archive", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 90.773729, "router_ms": 0.004511}
{"id": "eval-nova-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.128107, "router_ms": 0.025658}
{"id": "eval-nova-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.077274, "router_ms": 0.008599}
{"id": "eval-nova-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.074887, "router_ms": 0.008368}
{"id": "eval-nova-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.073291, "router_ms": 0.007503}
{"id": "eval-nova-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.072828, "router_ms": 0.007531}
{"id": "eval-nova-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117327, "router_ms": 0.006967}
{"id": "eval-nova-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.619958877563477, -11.372095108032227, -10.647622108459473, -10.998038291931152, -10.915349960327148, -11.330671310424805, -10.63299560546875, -11.234281539916992, -11.126276969909668, -11.4449462890625, -11.010820388793945, -11.30985164642334], "ranking": ["nova", "beacon", "rivet", "frost", "cargo", "lantern", "harbor", "export", "lantern-charge", "drone", "prism", "iris-network", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 111.807942, "router_ms": 0.005777}
{"id": "eval-nova-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.159681, "router_ms": 0.016743}
{"id": "eval-nova-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111661, "router_ms": 0.010272}
{"id": "eval-nova-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109034, "router_ms": 0.009027}
{"id": "eval-nova-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107374, "router_ms": 0.008782}
{"id": "eval-nova-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106809, "router_ms": 0.008645}
{"id": "eval-nova-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10653, "router_ms": 0.008306}
{"id": "eval-orbit-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.4112772941589355, -11.332706451416016, -10.062492370605469, -10.203400611877441, -9.598102569580078, -10.79186725616455, -11.121228218078613, -10.526987075805664, -6.598150730133057, -7.125789642333984, -10.53573989868164, -11.27670669555664], "ranking": ["orbit", "engine", "export", "harbor", "quartz", "mirror", "drone", "frost", "archive", "beacon", "garden", "cargo", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 107.218276, "router_ms": 0.006438}
{"id": "eval-orbit-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.166953, "router_ms": 0.016341}
{"id": "eval-orbit-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.123339, "router_ms": 0.00985}
{"id": "eval-orbit-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119032, "router_ms": 0.009061}
{"id": "eval-orbit-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118852, "router_ms": 0.008798}
{"id": "eval-orbit-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118235, "router_ms": 0.009062}
{"id": "eval-orbit-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118325, "router_ms": 0.008955}
{"id": "eval-orbit-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115043, "router_ms": 0.005855}
{"id": "eval-orbit-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129128, "router_ms": 0.010294}
{"id": "eval-orbit-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120934, "router_ms": 0.009213}
{"id": "eval-orbit-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119322, "router_ms": 0.008748}
{"id": "eval-orbit-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118595, "router_ms": 0.008691}
{"id": "eval-orbit-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119106, "router_ms": 0.008711}
{"id": "eval-orbit-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119055, "router_ms": 0.008725}
{"id": "eval-orbit-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115974, "router_ms": 0.005812}
{"id": "eval-orbit-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [6.3033576011657715, -11.41659164428711, -11.214947700500488, -11.41208267211914, -11.312108039855957, -11.461721420288086, -11.430337905883789, -11.417841911315918, -11.446528434753418, -11.4389066696167, -11.460297584533691, -11.443344116210938], "ranking": ["orbit", "iris", "lantern", "rivet-calibration", "lantern-charge", "frost", "nova", "iris-network", "prism", "harbor", "cargo", "quartz-batch", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 103.134062, "router_ms": 0.005793}
{"id": "eval-prism-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.159802, "router_ms": 0.017553}
{"id": "eval-prism-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113861, "router_ms": 0.010936}
{"id": "eval-prism-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113782, "router_ms": 0.00915}
{"id": "eval-prism-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103224, "router_ms": 0.009011}
{"id": "eval-prism-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.100865, "router_ms": 0.008534}
{"id": "eval-prism-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.098744, "router_ms": 0.005694}
{"id": "eval-prism-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [10.058465003967285, -11.35808277130127, -11.395358085632324, -11.298918724060059, -11.376766204833984, -11.360922813415527, -11.384462356567383, -11.3346529006958, -11.421213150024414, -11.38614273071289, -11.231120109558105, -11.291454315185547], "ranking": ["prism", "frost", "garden", "iris", "beacon", "rivet-calibration", "harbor", "drone", "archive", "export", "engine", "cargo", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 92.653664, "router_ms": 0.00511}
{"id": "eval-prism-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.153405, "router_ms": 0.016517}
{"id": "eval-prism-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.137851, "router_ms": 0.011826}
{"id": "eval-prism-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125643, "router_ms": 0.009914}
{"id": "eval-prism-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114291, "router_ms": 0.010054}
{"id": "eval-prism-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112412, "router_ms": 0.009081}
{"id": "eval-prism-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108869, "router_ms": 0.00598}
{"id": "eval-prism-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [7.783456325531006, -11.350871086120605, -11.440305709838867, -11.468302726745605, -11.444717407226562, -11.468291282653809, -11.438386917114258, -11.472097396850586, -11.480454444885254, -11.463541984558105, -11.490352630615234, -11.461370468139648], "ranking": ["prism", "rivet", "cargo", "quartz", "frost", "lantern", "harbor", "drone", "nova", "beacon", "export", "iris-network", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 107.01585, "router_ms": 0.005569}
{"id": "eval-prism-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.162056, "router_ms": 0.017332}
{"id": "eval-prism-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120015, "router_ms": 0.010535}
{"id": "eval-quartz-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12637, "router_ms": 0.010263}
{"id": "eval-quartz-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117067, "router_ms": 0.009277}
{"id": "eval-quartz-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119213, "router_ms": 0.009173}
{"id": "eval-quartz-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114058, "router_ms": 0.005796}
{"id": "eval-quartz-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [9.253325462341309, -0.6516895294189453, -7.4880571365356445, -11.421065330505371, -11.176493644714355, -11.252103805541992, -11.449766159057617, -11.421207427978516, -11.464003562927246, -11.286712646484375, -11.034472465515137, -11.463125228881836], "ranking": ["quartz", "quartz-batch", "mirror", "lantern-charge", "drone", "beacon", "lantern", "frost", "harbor", "export", "archive", "iris-network", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 102.276981, "router_ms": 0.005584}
{"id": "eval-quartz-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.176998, "router_ms": 0.017267}
{"id": "eval-quartz-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119342, "router_ms": 0.010265}
{"id": "eval-quartz-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117454, "router_ms": 0.009676}
{"id": "eval-quartz-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.130972, "router_ms": 0.010266}
{"id": "eval-quartz-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.123165, "router_ms": 0.009531}
{"id": "eval-quartz-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119367, "router_ms": 0.006116}
{"id": "eval-quartz-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [6.443126201629639, -4.9894328117370605, -11.329298973083496, -10.995671272277832, -10.901998519897461, -8.954572677612305, -10.973638534545898, -9.095609664916992, -10.860000610351562, -10.853055000305176, -10.665283203125, -10.9835786819458], "ranking": ["quartz", "quartz-batch", "orbit", "junction", "drone", "frost", "engine", "export", "soil", "harbor", "beacon", "cargo", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 99.709668, "router_ms": 0.005912}
{"id": "eval-quartz-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.198517, "router_ms": 0.019472}
{"id": "eval-quartz-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.136844, "router_ms": 0.012241}
{"id": "eval-quartz-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.124745, "router_ms": 0.010038}
{"id": "eval-quartz-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122231, "router_ms": 0.009294}
{"id": "eval-rivet-1", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112371, "router_ms": 0.009292}
{"id": "eval-rivet-1", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103153, "router_ms": 0.006078}
{"id": "eval-rivet-1", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.032523155212402, 5.0930986404418945, -11.045774459838867, -11.111373901367188, -11.299398422241211, -11.02957820892334, -10.90796947479248, -11.426010131835938, -10.650565147399902, -10.10616683959961, -10.731083869934082, -10.548370361328125], "ranking": ["rivet", "rivet-calibration", "lantern-charge", "mirror", "lantern", "archive", "harbor", "export", "frost", "drone", "beacon", "iris-network", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 105.520435, "router_ms": 0.005421}
{"id": "eval-rivet-1", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.165246, "router_ms": 0.016252}
{"id": "eval-rivet-1", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121058, "router_ms": 0.009938}
{"id": "eval-rivet-1", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108127, "router_ms": 0.009523}
{"id": "eval-rivet-1", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107073, "router_ms": 0.008827}
{"id": "eval-rivet-1", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106561, "router_ms": 0.008571}
{"id": "eval-rivet-2", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118288, "router_ms": 0.007239}
{"id": "eval-rivet-2", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [8.027447700500488, -11.266072273254395, -11.40830135345459, 0.2866683900356293, -11.423832893371582, -11.390559196472168, -11.425186157226562, -11.415412902832031, -11.410115242004395, -11.452159881591797, -11.419374465942383, -11.453349113464355], "ranking": ["rivet", "rivet-calibration", "prism", "frost", "quartz", "beacon", "cargo", "harbor", "nova", "drone", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 102.474023, "router_ms": 0.006311}
{"id": "eval-rivet-2", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.170122, "router_ms": 0.017285}
{"id": "eval-rivet-2", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.141641, "router_ms": 0.011285}
{"id": "eval-rivet-2", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114943, "router_ms": 0.00945}
{"id": "eval-rivet-2", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.116663, "router_ms": 0.010344}
{"id": "eval-rivet-2", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114304, "router_ms": 0.009604}
{"id": "eval-rivet-2", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.112127, "router_ms": 0.008976}
{"id": "eval-unknown-weather", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [-11.279678344726562, -11.37423038482666, -11.425868034362793, -10.790202140808105, -11.403650283813477, -11.357444763183594, -11.363469123840332, -11.213229179382324, -11.39967155456543, -11.318207740783691, -11.420628547668457, -11.412744522094727], "ranking": ["prism", "garden", "keel", "lantern", "junction", "lantern-charge", "nova", "iris-network", "iris", "rivet-calibration", "archive", "rivet", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 98.412938, "router_ms": 0.005656}
{"id": "eval-unknown-weather", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.158276, "router_ms": 0.015107}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.087564, "router_ms": 0.008595}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.084519, "router_ms": 0.007814}
{"id": "eval-unknown-weather", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.083398, "router_ms": 0.007384}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.082609, "router_ms": 0.007559}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.083315, "router_ms": 0.007297}
{"id": "eval-unknown-weather", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.079763, "router_ms": 0.004538}
{"id": "eval-unknown-captain", "policy": "router-b4-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.106485, "router_ms": 0.00817}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.099104, "router_ms": 0.007547}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.098355, "router_ms": 0.007387}
{"id": "eval-unknown-captain", "policy": "router-b12-p0", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.097831, "router_ms": 0.007332}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.01", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.096991, "router_ms": 0.007214}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.04", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.096827, "router_ms": 0.007239}
{"id": "eval-unknown-captain", "policy": "never", "repetition": 1, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.094728, "router_ms": 0.004676}
{"id": "eval-unknown-captain", "policy": "always-12", "repetition": 1, "depth": 12, "logits": [-11.368772506713867, -11.419684410095215, -11.30733871459961, -11.103347778320312, -11.180445671081543, -11.462408065795898, -11.390913009643555, -11.450128555297852, -11.37920093536377, -11.34627914428711, -11.41447925567627, -11.448532104492188], "ranking": ["iris-network", "iris", "garden", "soil", "keel", "lantern-charge", "lantern", "mirror", "frost", "junction", "orbit", "cargo", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 97.516898, "router_ms": 0.004924}
{"id": "eval-garden-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.158293, "router_ms": 0.0179}
{"id": "eval-garden-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117398, "router_ms": 0.01216}
{"id": "eval-garden-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103615, "router_ms": 0.009365}
{"id": "eval-garden-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.175756, "router_ms": 0.017578}
{"id": "eval-garden-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.153531, "router_ms": 0.016569}
{"id": "eval-garden-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108977, "router_ms": 0.010263}
{"id": "eval-garden-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "soil", "quartz", "iris", "junction", "lantern-charge", "keel", "iris-network", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.154818, "router_ms": 0.011056}
{"id": "eval-garden-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [9.829537391662598, 8.905885696411133, -11.418170928955078, -11.429336547851562, -11.416326522827148, -11.383506774902344, -11.41354751586914, -11.452909469604492, -11.40498161315918, -11.423681259155273, -11.392634391784668, -11.419623374938965], "ranking": ["garden", "soil", "lantern-charge", "archive", "lantern", "keel", "junction", "quartz", "rivet-calibration", "rivet", "iris", "iris-network", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.551338, "router_ms": 0.007105}
{"id": "eval-garden-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.193642, "router_ms": 0.016593}
{"id": "eval-garden-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.136161, "router_ms": 0.010198}
{"id": "eval-garden-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.132996, "router_ms": 0.009522}
{"id": "eval-garden-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.132677, "router_ms": 0.009378}
{"id": "eval-garden-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.132328, "router_ms": 0.008801}
{"id": "eval-garden-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12873, "router_ms": 0.006065}
{"id": "eval-garden-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.952169895172119, -10.063556671142578, -5.017200469970703, -11.34083080291748, -11.358210563659668, -11.33510971069336, -10.223990440368652, -11.156427383422852, -11.10240650177002, -9.888599395751953, -10.790337562561035, -10.740549087524414], "ranking": ["garden", "soil", "lantern", "quartz", "lantern-charge", "archive", "rivet", "keel", "junction", "iris", "iris-network", "quartz-batch", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 106.069587, "router_ms": 0.006266}
{"id": "eval-garden-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["garden", "quartz", "soil", "iris-network", "quartz-batch", "iris", "lantern-charge", "junction", "keel", "lantern", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "harbor", "frost", "drone", "orbit", "export", "cargo", "mirror", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.193894, "router_ms": 0.01701}
{"id": "eval-harbor-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103712, "router_ms": 0.009523}
{"id": "eval-harbor-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.09322, "router_ms": 0.008042}
{"id": "eval-harbor-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.093322, "router_ms": 0.007799}
{"id": "eval-harbor-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.093278, "router_ms": 0.00786}
{"id": "eval-harbor-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.087604, "router_ms": 0.004854}
{"id": "eval-harbor-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [9.691278457641602, 5.049304962158203, -11.02155590057373, -11.323006629943848, -10.970941543579102, -11.106721878051758, -11.357669830322266, -9.4385404586792, -8.870769500732422, -10.29703426361084, -10.645711898803711, -11.19121265411377], "ranking": ["harbor", "engine", "orbit", "mirror", "lantern", "lantern-charge", "cargo", "frost", "garden", "soil", "iris-network", "iris", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 91.435619, "router_ms": 0.004941}
{"id": "eval-harbor-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.162682, "router_ms": 0.016686}
{"id": "eval-harbor-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "frost", "iris-network", "cargo", "garden", "iris", "mirror", "orbit", "lantern", "lantern-charge", "soil", "drone", "beacon", "export", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115158, "router_ms": 0.009596}
{"id": "eval-harbor-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.095179, "router_ms": 0.009333}
{"id": "eval-harbor-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.090128, "router_ms": 0.008767}
{"id": "eval-harbor-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.088724, "router_ms": 0.008081}
{"id": "eval-harbor-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.085591, "router_ms": 0.005304}
{"id": "eval-harbor-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.506385803222656, 5.286466598510742, -11.294589042663574, -11.288810729980469, -11.326292037963867, -11.278151512145996, -11.393287658691406, -11.377354621887207, -11.325961112976074, -11.262049674987793, -11.386184692382812, -11.274985313415527], "ranking": ["harbor", "engine", "lantern-charge", "mirror", "beacon", "frost", "quartz", "lantern", "drone", "iris-network", "archive", "export", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 93.521762, "router_ms": 0.004772}
{"id": "eval-harbor-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.137405, "router_ms": 0.015801}
{"id": "eval-harbor-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.094589, "router_ms": 0.009389}
{"id": "eval-harbor-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["harbor", "engine", "quartz", "frost", "drone", "beacon", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119128, "router_ms": 0.00895}
{"id": "eval-iris-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.13278, "router_ms": 0.010195}
{"id": "eval-iris-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.123248, "router_ms": 0.009684}
{"id": "eval-iris-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.118911, "router_ms": 0.006472}
{"id": "eval-iris-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.322504043579102, 7.934057235717773, -11.388988494873047, -11.355183601379395, -11.403718948364258, -11.435961723327637, -11.38909912109375, -11.349196434020996, -11.307306289672852, -11.441774368286133, -11.316682815551758, -11.352814674377441], "ranking": ["iris", "iris-network", "junction", "garden", "mirror", "lantern", "beacon", "frost", "lantern-charge", "drone", "quartz-batch", "keel", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.705392, "router_ms": 0.006103}
{"id": "eval-iris-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.167895, "router_ms": 0.016417}
{"id": "eval-iris-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.126358, "router_ms": 0.010494}
{"id": "eval-iris-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.123987, "router_ms": 0.009828}
{"id": "eval-iris-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris-network", "iris", "frost", "beacon", "drone", "quartz-batch", "lantern-charge", "mirror", "junction", "keel", "garden", "lantern", "rivet", "archive", "rivet-calibration", "nova", "engine", "harbor", "orbit", "export", "cargo", "soil", "quartz", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.121506, "router_ms": 0.009298}
{"id": "eval-iris-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119927, "router_ms": 0.01}
{"id": "eval-iris-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.116722, "router_ms": 0.006122}
{"id": "eval-iris-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [9.597367286682129, -7.179789066314697, -11.409870147705078, -10.546661376953125, -11.468208312988281, -11.472970962524414, -11.443428993225098, -11.378433227539062, -11.446844100952148, -11.410990715026855, -11.459843635559082, -11.413043022155762], "ranking": ["iris", "iris-network", "prism", "frost", "garden", "lantern", "lantern-charge", "harbor", "cargo", "orbit", "archive", "export", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 91.489594, "router_ms": 0.005811}
{"id": "eval-iris-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.1733, "router_ms": 0.016002}
{"id": "eval-iris-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11906, "router_ms": 0.010195}
{"id": "eval-iris-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114565, "router_ms": 0.009285}
{"id": "eval-iris-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.113538, "router_ms": 0.009014}
{"id": "eval-iris-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["iris", "iris-network", "garden", "prism", "archive", "export", "harbor", "frost", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "keel", "rivet", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "drone", "quartz"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118577, "router_ms": 0.008783}
{"id": "eval-junction-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106159, "router_ms": 0.006137}
{"id": "eval-junction-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [6.33122444152832, -10.568521499633789, -11.287206649780273, -11.336028099060059, -11.451650619506836, -11.436971664428711, -10.63071346282959, -10.962664604187012, -10.813403129577637, -11.352925300598145, -10.614968299865723, -11.220610618591309], "ranking": ["junction", "lantern-charge", "mirror", "cargo", "orbit", "lantern", "keel", "frost", "garden", "soil", "iris", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 94.440549, "router_ms": 0.005394}
{"id": "eval-junction-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.15556, "router_ms": 0.015234}
{"id": "eval-junction-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129096, "router_ms": 0.009552}
{"id": "eval-junction-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.128942, "router_ms": 0.011877}
{"id": "eval-junction-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108481, "router_ms": 0.008587}
{"id": "eval-junction-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106508, "router_ms": 0.008742}
{"id": "eval-junction-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "lantern-charge", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "soil", "mirror", "keel", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106384, "router_ms": 0.008675}
{"id": "eval-junction-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.817267417907715, -11.398323059082031, -11.459901809692383, -11.405369758605957, -11.39989185333252, -11.405868530273438, -11.431496620178223, -11.44167423248291, -11.441703796386719, -11.483541488647461, -11.41031551361084, -11.399264335632324], "ranking": ["junction", "beacon", "lantern-charge", "soil", "orbit", "engine", "lantern", "frost", "drone", "harbor", "export", "iris-network", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 95.601427, "router_ms": 0.006803}
{"id": "eval-junction-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.163766, "router_ms": 0.016479}
{"id": "eval-junction-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118674, "router_ms": 0.010193}
{"id": "eval-junction-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.116669, "router_ms": 0.009371}
{"id": "eval-junction-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11532, "router_ms": 0.008985}
{"id": "eval-junction-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11464, "router_ms": 0.008839}
{"id": "eval-junction-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114557, "router_ms": 0.008499}
{"id": "eval-junction-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["junction", "beacon", "export", "orbit", "soil", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "cargo", "quartz-batch", "garden", "prism", "quartz", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111376, "router_ms": 0.005606}
{"id": "eval-keel-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.096896, "router_ms": 0.008498}
{"id": "eval-keel-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.091064, "router_ms": 0.007682}
{"id": "eval-keel-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.090188, "router_ms": 0.007357}
{"id": "eval-keel-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.089831, "router_ms": 0.007567}
{"id": "eval-keel-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.090512, "router_ms": 0.007722}
{"id": "eval-keel-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.089537, "router_ms": 0.007245}
{"id": "eval-keel-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.086402, "router_ms": 0.004614}
{"id": "eval-keel-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.503390312194824, -7.309561252593994, -10.551677703857422, -11.386517524719238, -10.565544128417969, -10.63076400756836, -11.392157554626465, -11.005291938781738, -11.023717880249023, -10.225462913513184, -11.327591896057129, -10.9865083694458], "ranking": ["keel", "rivet-calibration", "frost", "engine", "archive", "beacon", "harbor", "drone", "export", "garden", "prism", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 95.457291, "router_ms": 0.004663}
{"id": "eval-keel-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.144891, "router_ms": 0.01652}
{"id": "eval-keel-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103024, "router_ms": 0.00973}
{"id": "eval-keel-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.13079, "router_ms": 0.008342}
{"id": "eval-keel-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.092863, "router_ms": 0.008346}
{"id": "eval-keel-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.093264, "router_ms": 0.008392}
{"id": "eval-keel-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.088873, "router_ms": 0.005037}
{"id": "eval-keel-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.379083633422852, -11.161659240722656, -11.338848114013672, -11.390254974365234, -10.53793716430664, -11.310693740844727, -11.470420837402344, -11.427215576171875, -11.357369422912598, -11.317784309387207, -11.438980102539062, -11.386247634887695], "ranking": ["keel", "archive", "rivet-calibration", "beacon", "frost", "engine", "export", "harbor", "prism", "drone", "garden", "cargo", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 92.232317, "router_ms": 0.004802}
{"id": "eval-keel-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "rivet-calibration", "engine", "prism", "archive", "beacon", "cargo", "drone", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.162721, "router_ms": 0.018104}
{"id": "eval-lantern-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.131189, "router_ms": 0.010632}
{"id": "eval-lantern-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10744, "router_ms": 0.00927}
{"id": "eval-lantern-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106783, "router_ms": 0.009116}
{"id": "eval-lantern-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.104429, "router_ms": 0.008751}
{"id": "eval-lantern-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.10158, "router_ms": 0.005545}
{"id": "eval-lantern-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.077771186828613, -10.966012954711914, -11.201990127563477, -11.393867492675781, -11.288263320922852, -11.190375328063965, -11.290112495422363, -11.279410362243652, -10.673552513122559, -11.278423309326172, -11.345672607421875, -11.221471786499023], "ranking": ["lantern", "lantern-charge", "iris-network", "iris", "frost", "beacon", "soil", "orbit", "garden", "mirror", "drone", "cargo", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 95.543533, "router_ms": 0.005541}
{"id": "eval-lantern-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.163015, "router_ms": 0.016825}
{"id": "eval-lantern-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern", "iris-network", "frost", "cargo", "garden", "iris", "mirror", "orbit", "lantern-charge", "soil", "drone", "beacon", "export", "harbor", "archive", "keel", "nova", "quartz-batch", "prism", "quartz", "junction", "rivet", "engine", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120836, "router_ms": 0.010843}
{"id": "eval-lantern-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.107247, "router_ms": 0.009659}
{"id": "eval-lantern-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.100507, "router_ms": 0.008969}
{"id": "eval-lantern-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.099006, "router_ms": 0.008645}
{"id": "eval-lantern-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.095312, "router_ms": 0.005646}
{"id": "eval-lantern-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.703226089477539, 7.691359043121338, -11.438968658447266, -10.154081344604492, -11.016915321350098, -10.71131420135498, -11.458416938781738, -11.044757843017578, -11.049427032470703, -11.024188995361328, -10.528362274169922, -10.827560424804688], "ranking": ["lantern-charge", "lantern", "junction", "beacon", "garden", "nova", "keel", "rivet-calibration", "rivet", "archive", "iris", "iris-network", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 99.910545, "router_ms": 0.005251}
{"id": "eval-lantern-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.153402, "router_ms": 0.017201}
{"id": "eval-lantern-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.128975, "router_ms": 0.010645}
{"id": "eval-lantern-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["lantern-charge", "lantern", "iris", "junction", "keel", "garden", "iris-network", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror", "prism"], "quality": {"ndcg3": 0.6309297535714575, "mrr": 0.5}, "answerable": true, "elapsed_ms": 0.101985, "router_ms": 0.009239}
{"id": "eval-mirror-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.140807, "router_ms": 0.010755}
{"id": "eval-mirror-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126436, "router_ms": 0.00937}
{"id": "eval-mirror-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12151, "router_ms": 0.006097}
{"id": "eval-mirror-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.31332540512085, -11.464130401611328, -11.357179641723633, -11.377193450927734, -11.460406303405762, -11.458881378173828, -11.47458267211914, -11.455571174621582, -11.463949203491211, -11.457032203674316, -11.473447799682617, -11.469704627990723], "ranking": ["mirror", "prism", "rivet", "lantern-charge", "frost", "harbor", "cargo", "junction", "nova", "beacon", "drone", "lantern", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 99.940742, "router_ms": 0.005931}
{"id": "eval-mirror-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.184462, "router_ms": 0.017705}
{"id": "eval-mirror-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.150644, "router_ms": 0.020532}
{"id": "eval-mirror-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125288, "router_ms": 0.009498}
{"id": "eval-mirror-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "nova", "prism", "rivet", "cargo", "harbor", "lantern", "lantern-charge", "junction", "frost", "drone", "beacon", "export", "iris-network", "archive", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.126254, "router_ms": 0.009585}
{"id": "eval-mirror-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105563, "router_ms": 0.009926}
{"id": "eval-mirror-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.097489, "router_ms": 0.005916}
{"id": "eval-mirror-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.968134880065918, -9.22532844543457, -11.435161590576172, -11.465716361999512, -11.459169387817383, -11.448808670043945, -11.410764694213867, -11.483901977539062, -11.453641891479492, -11.447628021240234, -11.434931755065918, -11.456716537475586], "ranking": ["mirror", "quartz", "harbor", "archive", "frost", "lantern-charge", "export", "lantern", "orbit", "beacon", "drone", "iris-network", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 90.612692, "router_ms": 0.005367}
{"id": "eval-mirror-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.156433, "router_ms": 0.017269}
{"id": "eval-mirror-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115653, "router_ms": 0.010894}
{"id": "eval-mirror-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111672, "router_ms": 0.00913}
{"id": "eval-mirror-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.099748, "router_ms": 0.009073}
{"id": "eval-mirror-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["mirror", "quartz", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103389, "router_ms": 0.008802}
{"id": "eval-nova-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.076261, "router_ms": 0.00563}
{"id": "eval-nova-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [5.650143623352051, -11.470599174499512, -11.343707084655762, -11.446039199829102, -11.457576751708984, -11.356437683105469, -11.46668815612793, -11.40383529663086, -11.37934398651123, -11.419157981872559, -11.388242721557617, -11.457596778869629], "ranking": ["nova", "beacon", "engine", "garden", "iris", "frost", "harbor", "cargo", "drone", "iris-network", "export", "archive", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 88.881667, "router_ms": 0.0045}
{"id": "eval-nova-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.124408, "router_ms": 0.016328}
{"id": "eval-nova-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.084273, "router_ms": 0.00927}
{"id": "eval-nova-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.080564, "router_ms": 0.008527}
{"id": "eval-nova-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.073517, "router_ms": 0.007522}
{"id": "eval-nova-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.074203, "router_ms": 0.007397}
{"id": "eval-nova-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "archive", "beacon", "cargo", "drone", "engine", "export", "frost", "garden", "harbor", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "orbit", "prism", "quartz", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.073439, "router_ms": 0.007375}
{"id": "eval-nova-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.619958877563477, -11.372095108032227, -10.647622108459473, -10.998038291931152, -10.915349960327148, -11.330671310424805, -10.63299560546875, -11.234281539916992, -11.126276969909668, -11.4449462890625, -11.010820388793945, -11.30985164642334], "ranking": ["nova", "beacon", "rivet", "frost", "cargo", "lantern", "harbor", "export", "lantern-charge", "drone", "prism", "iris-network", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 99.271805, "router_ms": 0.006724}
{"id": "eval-nova-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.163162, "router_ms": 0.017633}
{"id": "eval-nova-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122747, "router_ms": 0.010996}
{"id": "eval-nova-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109157, "router_ms": 0.009422}
{"id": "eval-nova-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108549, "router_ms": 0.009067}
{"id": "eval-nova-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107728, "router_ms": 0.00899}
{"id": "eval-nova-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105633, "router_ms": 0.008587}
{"id": "eval-nova-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["nova", "prism", "rivet", "cargo", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "quartz", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.104427, "router_ms": 0.005727}
{"id": "eval-orbit-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129431, "router_ms": 0.009793}
{"id": "eval-orbit-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.12007, "router_ms": 0.008588}
{"id": "eval-orbit-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119026, "router_ms": 0.008408}
{"id": "eval-orbit-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118359, "router_ms": 0.008318}
{"id": "eval-orbit-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120176, "router_ms": 0.008335}
{"id": "eval-orbit-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.118179, "router_ms": 0.007948}
{"id": "eval-orbit-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "cargo", "quartz", "mirror", "harbor", "archive", "beacon", "drone", "engine", "export", "frost", "garden", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11582, "router_ms": 0.005301}
{"id": "eval-orbit-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.4112772941589355, -11.332706451416016, -10.062492370605469, -10.203400611877441, -9.598102569580078, -10.79186725616455, -11.121228218078613, -10.526987075805664, -6.598150730133057, -7.125789642333984, -10.53573989868164, -11.27670669555664], "ranking": ["orbit", "engine", "export", "harbor", "quartz", "mirror", "drone", "frost", "archive", "beacon", "garden", "cargo", "iris", "iris-network", "junction", "keel", "lantern", "lantern-charge", "nova", "prism", "quartz-batch", "rivet", "rivet-calibration", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 96.760671, "router_ms": 0.005376}
{"id": "eval-orbit-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.180765, "router_ms": 0.017967}
{"id": "eval-orbit-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.146129, "router_ms": 0.01152}
{"id": "eval-orbit-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.124283, "router_ms": 0.009803}
{"id": "eval-orbit-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122523, "router_ms": 0.009776}
{"id": "eval-orbit-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121638, "router_ms": 0.009475}
{"id": "eval-orbit-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117145, "router_ms": 0.00626}
{"id": "eval-orbit-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [6.3033576011657715, -11.41659164428711, -11.214947700500488, -11.41208267211914, -11.312108039855957, -11.461721420288086, -11.430337905883789, -11.417841911315918, -11.446528434753418, -11.4389066696167, -11.460297584533691, -11.443344116210938], "ranking": ["orbit", "iris", "lantern", "rivet-calibration", "lantern-charge", "frost", "nova", "iris-network", "prism", "harbor", "cargo", "quartz-batch", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 93.577424, "router_ms": 0.005861}
{"id": "eval-orbit-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["orbit", "lantern-charge", "iris", "rivet-calibration", "lantern", "quartz-batch", "nova", "frost", "harbor", "iris-network", "cargo", "prism", "garden", "junction", "rivet", "mirror", "soil", "drone", "beacon", "export", "archive", "keel", "quartz", "engine"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.170343, "router_ms": 0.016425}
{"id": "eval-prism-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111424, "router_ms": 0.010174}
{"id": "eval-prism-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103261, "router_ms": 0.009159}
{"id": "eval-prism-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.101669, "router_ms": 0.009047}
{"id": "eval-prism-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.101131, "router_ms": 0.008522}
{"id": "eval-prism-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.09783, "router_ms": 0.005427}
{"id": "eval-prism-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [10.058465003967285, -11.35808277130127, -11.395358085632324, -11.298918724060059, -11.376766204833984, -11.360922813415527, -11.384462356567383, -11.3346529006958, -11.421213150024414, -11.38614273071289, -11.231120109558105, -11.291454315185547], "ranking": ["prism", "frost", "garden", "iris", "beacon", "rivet-calibration", "harbor", "drone", "archive", "export", "engine", "cargo", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 93.574027, "router_ms": 0.007284}
{"id": "eval-prism-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.175582, "router_ms": 0.016608}
{"id": "eval-prism-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet-calibration", "engine", "iris", "drone", "harbor", "archive", "beacon", "cargo", "export", "frost", "garden", "iris-network", "junction", "keel", "lantern", "lantern-charge", "mirror", "nova", "orbit", "quartz", "quartz-batch", "rivet", "soil"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.104634, "router_ms": 0.009792}
{"id": "eval-prism-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.131022, "router_ms": 0.009845}
{"id": "eval-prism-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108191, "router_ms": 0.009441}
{"id": "eval-prism-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.107262, "router_ms": 0.00867}
{"id": "eval-prism-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.103793, "router_ms": 0.005888}
{"id": "eval-prism-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [7.783456325531006, -11.350871086120605, -11.440305709838867, -11.468302726745605, -11.444717407226562, -11.468291282653809, -11.438386917114258, -11.472097396850586, -11.480454444885254, -11.463541984558105, -11.490352630615234, -11.461370468139648], "ranking": ["prism", "rivet", "cargo", "quartz", "frost", "lantern", "harbor", "drone", "nova", "beacon", "export", "iris-network", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 104.450345, "router_ms": 0.005615}
{"id": "eval-prism-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.168297, "router_ms": 0.015749}
{"id": "eval-prism-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111554, "router_ms": 0.010297}
{"id": "eval-prism-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["prism", "rivet", "quartz", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.109022, "router_ms": 0.009412}
{"id": "eval-quartz-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.117497, "router_ms": 0.009727}
{"id": "eval-quartz-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.115455, "router_ms": 0.008893}
{"id": "eval-quartz-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111134, "router_ms": 0.005836}
{"id": "eval-quartz-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [9.253325462341309, -0.6516895294189453, -7.4880571365356445, -11.421065330505371, -11.176493644714355, -11.252103805541992, -11.449766159057617, -11.421207427978516, -11.464003562927246, -11.286712646484375, -11.034472465515137, -11.463125228881836], "ranking": ["quartz", "quartz-batch", "mirror", "lantern-charge", "drone", "beacon", "lantern", "frost", "harbor", "export", "archive", "iris-network", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 93.588181, "router_ms": 0.005403}
{"id": "eval-quartz-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.167236, "router_ms": 0.017232}
{"id": "eval-quartz-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.129649, "router_ms": 0.01036}
{"id": "eval-quartz-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.11588, "router_ms": 0.009549}
{"id": "eval-quartz-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "mirror", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "orbit", "keel", "nova", "cargo", "soil", "garden", "prism", "junction", "rivet", "engine", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.114837, "router_ms": 0.008926}
{"id": "eval-quartz-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.131242, "router_ms": 0.010509}
{"id": "eval-quartz-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120482, "router_ms": 0.00663}
{"id": "eval-quartz-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [6.443126201629639, -4.9894328117370605, -11.329298973083496, -10.995671272277832, -10.901998519897461, -8.954572677612305, -10.973638534545898, -9.095609664916992, -10.860000610351562, -10.853055000305176, -10.665283203125, -10.9835786819458], "ranking": ["quartz", "quartz-batch", "orbit", "junction", "drone", "frost", "engine", "export", "soil", "harbor", "beacon", "cargo", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 104.712559, "router_ms": 0.006189}
{"id": "eval-quartz-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.162043, "router_ms": 0.016174}
{"id": "eval-quartz-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.140579, "router_ms": 0.010282}
{"id": "eval-quartz-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.123612, "router_ms": 0.009613}
{"id": "eval-quartz-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.122232, "router_ms": 0.009019}
{"id": "eval-quartz-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["quartz", "quartz-batch", "cargo", "beacon", "export", "orbit", "soil", "junction", "engine", "frost", "drone", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "keel", "nova", "garden", "prism", "rivet", "iris", "rivet-calibration"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121759, "router_ms": 0.008973}
{"id": "eval-rivet-1", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.111118, "router_ms": 0.006541}
{"id": "eval-rivet-1", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.032523155212402, 5.0930986404418945, -11.045774459838867, -11.111373901367188, -11.299398422241211, -11.02957820892334, -10.90796947479248, -11.426010131835938, -10.650565147399902, -10.10616683959961, -10.731083869934082, -10.548370361328125], "ranking": ["rivet", "rivet-calibration", "lantern-charge", "mirror", "lantern", "archive", "harbor", "export", "frost", "drone", "beacon", "iris-network", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 100.921072, "router_ms": 0.005543}
{"id": "eval-rivet-1", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.164367, "router_ms": 0.016557}
{"id": "eval-rivet-1", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.121318, "router_ms": 0.010859}
{"id": "eval-rivet-1", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.119074, "router_ms": 0.009263}
{"id": "eval-rivet-1", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106029, "router_ms": 0.008686}
{"id": "eval-rivet-1", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106244, "router_ms": 0.008572}
{"id": "eval-rivet-1", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "rivet-calibration", "frost", "drone", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "nova", "cargo", "quartz-batch", "soil", "garden", "prism", "quartz", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.105203, "router_ms": 0.008285}
{"id": "eval-rivet-2", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [8.027447700500488, -11.266072273254395, -11.40830135345459, 0.2866683900356293, -11.423832893371582, -11.390559196472168, -11.425186157226562, -11.415412902832031, -11.410115242004395, -11.452159881591797, -11.419374465942383, -11.453349113464355], "ranking": ["rivet", "rivet-calibration", "prism", "frost", "quartz", "beacon", "cargo", "harbor", "nova", "drone", "export", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 100.158489, "router_ms": 0.007402}
{"id": "eval-rivet-2", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.168381, "router_ms": 0.017661}
{"id": "eval-rivet-2", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.125042, "router_ms": 0.010914}
{"id": "eval-rivet-2", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.120528, "router_ms": 0.009209}
{"id": "eval-rivet-2", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.108441, "router_ms": 0.008707}
{"id": "eval-rivet-2", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106615, "router_ms": 0.008451}
{"id": "eval-rivet-2", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.106465, "router_ms": 0.008131}
{"id": "eval-rivet-2", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["rivet", "prism", "quartz", "rivet-calibration", "nova", "frost", "drone", "cargo", "beacon", "export", "harbor", "iris-network", "lantern", "lantern-charge", "archive", "mirror", "orbit", "keel", "quartz-batch", "soil", "garden", "junction", "engine", "iris"], "quality": {"ndcg3": 1.0, "mrr": 1.0}, "answerable": true, "elapsed_ms": 0.102619, "router_ms": 0.005479}
{"id": "eval-unknown-weather", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.090708, "router_ms": 0.008281}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.084206, "router_ms": 0.007398}
{"id": "eval-unknown-weather", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.082933, "router_ms": 0.007182}
{"id": "eval-unknown-weather", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.10021, "router_ms": 0.007419}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.087259, "router_ms": 0.007627}
{"id": "eval-unknown-weather", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.086707, "router_ms": 0.007341}
{"id": "eval-unknown-weather", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "nova", "rivet", "prism", "iris", "junction", "lantern-charge", "garden", "iris-network", "lantern", "archive", "rivet-calibration", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.083169, "router_ms": 0.004475}
{"id": "eval-unknown-weather", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [-11.279678344726562, -11.37423038482666, -11.425868034362793, -10.790202140808105, -11.403650283813477, -11.357444763183594, -11.363469123840332, -11.213229179382324, -11.39967155456543, -11.318207740783691, -11.420628547668457, -11.412744522094727], "ranking": ["prism", "garden", "keel", "lantern", "junction", "lantern-charge", "nova", "iris-network", "iris", "rivet-calibration", "archive", "rivet", "beacon", "engine", "quartz-batch", "harbor", "frost", "drone", "orbit", "export", "cargo", "soil", "quartz", "mirror"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 96.579878, "router_ms": 0.004583}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.154798, "router_ms": 0.016021}
{"id": "eval-unknown-captain", "policy": "router-b4-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.118481, "router_ms": 0.009324}
{"id": "eval-unknown-captain", "policy": "router-b12-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.102547, "router_ms": 0.008447}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.01", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.101632, "router_ms": 0.008412}
{"id": "eval-unknown-captain", "policy": "router-b12-p0.04", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.100242, "router_ms": 0.007785}
{"id": "eval-unknown-captain", "policy": "never", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.097965, "router_ms": 0.005239}
{"id": "eval-unknown-captain", "policy": "always-12", "repetition": 2, "depth": 12, "logits": [-11.368772506713867, -11.419684410095215, -11.30733871459961, -11.103347778320312, -11.180445671081543, -11.462408065795898, -11.390913009643555, -11.450128555297852, -11.37920093536377, -11.34627914428711, -11.41447925567627, -11.448532104492188], "ranking": ["iris-network", "iris", "garden", "soil", "keel", "lantern-charge", "lantern", "mirror", "frost", "junction", "orbit", "cargo", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 90.304056, "router_ms": 0.004896}
{"id": "eval-unknown-captain", "policy": "router-b4-p0", "repetition": 2, "depth": 0, "logits": [], "ranking": ["keel", "frost", "garden", "iris-network", "iris", "cargo", "lantern", "orbit", "lantern-charge", "soil", "mirror", "junction", "rivet", "archive", "rivet-calibration", "beacon", "nova", "engine", "quartz-batch", "harbor", "drone", "export", "quartz", "prism"], "quality": {"ndcg3": 0.0, "mrr": 0.0}, "answerable": false, "elapsed_ms": 0.160984, "router_ms": 0.016115}