{ "problems": [ { "name": "Vision", "attributes": [ "agi", "vision", "world-modelling" ], "subproblems": [ "Image classification", "Recognise events in videos" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Image comprehension", "attributes": [ "agi", "vision", "language", "world-modelling" ], "subproblems": [], "superproblems": [ "Image classification" ], "metrics": [ { "measures": [ { "date": "2015-05-03", "value": 58.2, "name": "LSTM Q+I", "url": "https://arxiv.org/abs/1505.00468v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 58.2, "maxval": 58.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM Q+I", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2015-12-15", "value": 55.89, "name": "iBOWIMG baseline", "url": "https://arxiv.org/abs/1512.02167", "min_date": "2015-12-07", "max_date": "2015-12-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 55.89, "maxval": 55.89, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Simple Baseline for Visual Question Answering", "label": "iBOWIMG baseline", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-01-26", "value": 58.9, "name": "SAN", "url": "https://arxiv.org/abs/1511.02274v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 58.9, "maxval": 58.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stacked Attention Networks for Image Question Answering", "label": "SAN", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-03-09", "value": 59.5, "name": "CNN-RNN", "url": "https://arxiv.org/abs/1603.02814v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 59.5, "maxval": 59.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Image Captioning and Visual Question Answering Based on Attributes and Their Related External Knowledge", "label": "CNN-RNN", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-03-19", "value": 58.24, "name": "SMem-VQA", "url": "https://arxiv.org/abs/1511.05234v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 58.24, "maxval": 58.24, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Ask, Attend and Answer: Exploring Question-Guided Spatial Attention for Visual Question Answering", "label": "SMem-VQA", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-04-06", "value": 59.5, "name": "FDA", "url": "https://arxiv.org/abs/1604.01485v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 59.5, "maxval": 59.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Focused Dynamic Attention Model for Visual Question Answering", "label": "FDA", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-05-31", "value": 62.1, "name": "HQI+ResNet", "url": "https://arxiv.org/abs/1606.00061v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62.1, "maxval": 62.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Hierarchical Co-Attention for Visual Question Answering", "label": "HQI+ResNet", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-06-05", "value": 61.84, "name": "MRN + global features", "url": "https://arxiv.org/abs/1606.01455v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61.84, "maxval": 61.84, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multimodal Residual Learning for Visual QA", "label": "MRN + global features", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-06-06", "value": 66.5, "name": "MCB 7 att.", "url": "https://arxiv.org/abs/1606.01847v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.5, "maxval": 66.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "https://github.com/akirafukui/vqa-mcb", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding", "label": "MCB 7 att.", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2016-08-06", "value": 63.2, "name": "joint-loss", "url": "https://arxiv.org/abs/1606.03647", "min_date": "2016-06-12", "max_date": "2016-09-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.2, "maxval": 63.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Training Recurrent Answering Units with Joint Loss Minimization for VQA", "label": "joint-loss", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" }, { "date": "2017-08-06", "value": 64.2, "name": "N2NMN", "url": "https://arxiv.org/abs/1704.05526v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 64.2, "maxval": 64.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "http://ronghanghu.com/n2nmn/", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning to Reason: End-to-End Module Networks for Visual Question Answering", "label": "N2NMN", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 open ended)not solved" } ], "name": "COCO Visual Question Answering (VQA) real images 1.0 open ended", "solved": false, "url": "http://visualqa.org/", "notes": "", "scale": "Percentage correct", "target": 83.3, "target_source": "https://arxiv.org/abs/1505.00468", "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L113" }, { "measures": [ { "date": "2015-05-03", "value": 63.1, "name": "LSTM Q+I", "url": "https://arxiv.org/abs/1505.00468v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.1, "maxval": 63.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM Q+I", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2015-12-15", "value": 61.97, "name": "iBOWIMG baseline", "url": "https://arxiv.org/abs/1512.02167", "min_date": "2015-12-07", "max_date": "2015-12-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61.97, "maxval": 61.97, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Simple Baseline for Visual Question Answering", "label": "iBOWIMG baseline", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2016-04-06", "value": 64.2, "name": "FDA", "url": "https://arxiv.org/abs/1604.01485v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 64.2, "maxval": 64.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Focused Dynamic Attention Model for Visual Question Answering", "label": "FDA", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2016-05-31", "value": 66.1, "name": "HQI+ResNet", "url": "https://arxiv.org/abs/1606.00061v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.1, "maxval": 66.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Hierarchical Co-Attention for Visual Question Answering", "label": "HQI+ResNet", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2016-06-05", "value": 66.33, "name": "MRN", "url": "https://arxiv.org/abs/1606.01455v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.33, "maxval": 66.33, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multimodal Residual Learning for Visual QA", "label": "MRN", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2016-06-06", "value": 70.1, "name": "MCB 7 att.", "url": "https://arxiv.org/abs/1606.01847v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.1, "maxval": 70.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "https://github.com/akirafukui/vqa-mcb", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding", "label": "MCB 7 att.", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" }, { "date": "2016-08-06", "value": 67.3, "name": "joint-loss", "url": "https://arxiv.org/abs/1606.03647", "min_date": "2016-06-12", "max_date": "2016-09-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.3, "maxval": 67.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Training Recurrent Answering Units with Joint Loss Minimization for VQA", "label": "joint-loss", "metric": "Metric(COCO Visual Question Answering (VQA) real images 1.0 multiple choice)?" } ], "name": "COCO Visual Question Answering (VQA) real images 1.0 multiple choice", "solved": false, "url": "http://visualqa.org/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L108" }, { "measures": [ { "date": "2016-07-01", "value": 69.73, "name": "Dualnet ensemble", "url": "http://visualqa.org/aoe.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1606.06108", "src_name": "DualNet: Domain-Invariant Network for Visual Question Answering", "uncertainty": 0, "minval": 69.73, "maxval": 69.73, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "Dualnet ensemble", "metric": "Metric(COCO Visual Question Answering (VQA) abstract images 1.0 open ended)not solved" }, { "date": "2016-07-01", "value": 65.02, "name": "LSTM + global features", "url": "http://visualqa.org/aoe.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1511.05099", "src_name": "Yin and Yang: Balancing and Answering Binary Visual Questions", "uncertainty": 0, "minval": 65.02, "maxval": 65.02, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM + global features", "metric": "Metric(COCO Visual Question Answering (VQA) abstract images 1.0 open ended)not solved" }, { "date": "2016-07-01", "value": 57.19, "name": "LSTM blind", "url": "http://visualqa.org/aoe.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1511.05099", "src_name": "Yin and Yang: Balancing and Answering Binary Visual Questions", "uncertainty": 0, "minval": 57.19, "maxval": 57.19, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM blind", "metric": "Metric(COCO Visual Question Answering (VQA) abstract images 1.0 open ended)not solved" }, { "date": "2016-09-19", "value": 70.42, "name": "Graph VQA", "url": "https://arxiv.org/abs/1609.05600v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.42, "maxval": 70.42, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Graph-Structured Representations for Visual Question Answering", "label": "Graph VQA", "metric": "Metric(COCO Visual Question Answering (VQA) abstract images 1.0 open ended)not solved" } ], "name": "COCO Visual Question Answering (VQA) abstract images 1.0 open ended", "solved": false, "url": "http://visualqa.org/", "notes": "", "scale": "Percentage correct", "target": 87.49, "target_source": "https://arxiv.org/abs/1505.00468", "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L111" }, { "measures": [ { "date": "2016-07-01", "value": 71.18, "name": "Dualnet ensemble", "url": "http://visualqa.org/amc.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1606.06108", "src_name": "DualNet: Domain-Invariant Network for Visual Question Answering", "uncertainty": 0, "minval": 71.18, "maxval": 71.18, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "Dualnet ensemble", "metric": "Metric(COCO Visual Question Answering (VQA) abstract 1.0 multiple choice)?" }, { "date": "2016-07-01", "value": 69.21, "name": "LSTM + global features", "url": "http://visualqa.org/amc.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1511.05099", "src_name": "Yin and Yang: Balancing and Answering Binary Visual Questions", "uncertainty": 0, "minval": 69.21, "maxval": 69.21, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM + global features", "metric": "Metric(COCO Visual Question Answering (VQA) abstract 1.0 multiple choice)?" }, { "date": "2016-07-01", "value": 61.41, "name": "LSTM blind", "url": "http://visualqa.org/amc.html", "min_date": "2016-01-01", "max_date": "2016-12-31", "algorithm_src_url": "https://arxiv.org/abs/1511.05099", "src_name": "Yin and Yang: Balancing and Answering Binary Visual Questions", "uncertainty": 0, "minval": 61.41, "maxval": 61.41, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "LSTM blind", "metric": "Metric(COCO Visual Question Answering (VQA) abstract 1.0 multiple choice)?" }, { "date": "2016-09-19", "value": 74.37, "name": "Graph VQA", "url": "https://arxiv.org/abs/1609.05600v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.37, "maxval": 74.37, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Graph-Structured Representations for Visual Question Answering", "label": "Graph VQA", "metric": "Metric(COCO Visual Question Answering (VQA) abstract 1.0 multiple choice)?" } ], "name": "COCO Visual Question Answering (VQA) abstract 1.0 multiple choice", "solved": false, "url": "http://visualqa.org/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L110" }, { "measures": [], "name": "Toronto COCO-QA", "solved": false, "url": "http://www.cs.toronto.edu/~mren/imageqa/data/cocoqa/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L58" }, { "measures": [], "name": "DAQUAR", "solved": false, "url": "https://www.mpi-inf.mpg.de/departments/computer-vision-and-multimodal-computing/research/vision-and-language/visual-turing-challenge/", "notes": "", "scale": "Percentage correct", "target": 60.27, "target_source": "https://arxiv.org/abs/1505.02074", "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L59" }, { "measures": [ { "date": "2016-11-30", "value": 28.52, "name": "CMN", "url": "https://arxiv.org/abs/1611.09978v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.52, "maxval": 28.52, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Modeling Relationships in Referential Expressions with Compositional Modular Networks", "label": "CMN", "metric": "Metric(Visual Genome (pairs)) ?" } ], "name": "Visual Genome (pairs)", "solved": false, "url": "http://visualgenome.org", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Top-1 precision", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L124" }, { "measures": [ { "date": "2016-11-30", "value": 44.24, "name": "CMN", "url": "https://arxiv.org/abs/1611.09978v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 44.24, "maxval": 44.24, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Modeling Relationships in Referential Expressions with Compositional Modular Networks", "label": "CMN", "metric": "Metric(Visual Genome (subjects)) ?" } ], "name": "Visual Genome (subjects)", "solved": false, "url": "http://visualgenome.org", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Top-1 precision", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L125" }, { "measures": [ { "date": "2016-06-06", "value": 62.2, "name": "MCB+Att.", "url": "https://arxiv.org/abs/1606.01847v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62.2, "maxval": 62.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding", "label": "MCB+Att.", "metric": "Metric(Visual7W) ?" }, { "date": "2016-11-30", "value": 72.53, "name": "CMN", "url": "https://arxiv.org/abs/1611.09978v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.53, "maxval": 72.53, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Modeling Relationships in Referential Expressions with Compositional Modular Networks", "label": "CMN", "metric": "Metric(Visual7W) ?" } ], "name": "Visual7W", "solved": false, "url": "https://arxiv.org/abs/1511.03416", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L123" }, { "measures": [], "name": "FM-IQA", "solved": false, "url": "http://idl.baidu.com/FM-IQA.html", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L64" }, { "measures": [], "name": "Visual Madlibs", "solved": false, "url": "http://tamaraberg.com/visualmadlibs/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L65" }, { "measures": [ { "date": "2016-12-02", "value": 62.27, "name": "MCB", "url": "https://arxiv.org/abs/1612.00837v1", "min_date": "2016-06-06", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1606.01847v1", "src_name": "Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding", "uncertainty": 0, "minval": 62.27, "maxval": 62.27, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making the V in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering", "label": "MCB", "metric": "Metric(COCO Visual Question Answering (VQA) real images 2.0 open ended)?" }, { "date": "2016-12-02", "value": 54.22, "name": "d-LSTM+nI", "url": "https://arxiv.org/abs/1612.00837v1", "min_date": "2015-12-14", "max_date": null, "algorithm_src_url": "https://github.com/VT-vision-lab/VQA_LSTM_CNN", "src_name": "GitHub - VT-vision-lab/VQA_LSTM_CNN: Train a deeper LSTM and normalized CNN Visual Question Answering model. This current code can get 58.16 on OpenEnded and 63.09 on Multiple-Choice on test-standard.", "uncertainty": 0, "minval": 54.22, "maxval": 54.22, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making the V in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering", "label": "d-LSTM+nI", "metric": "Metric(COCO Visual Question Answering (VQA) real images 2.0 open ended)?" }, { "date": "2017-07-25", "value": 70.34, "name": "Up-Down", "url": "https://arxiv.org/abs/1707.07998v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.34, "maxval": 70.34, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Bottom-Up and Top-Down Attention for Image Captioning and VQA", "label": "Up-Down", "metric": "Metric(COCO Visual Question Answering (VQA) real images 2.0 open ended)?" }, { "date": "2017-07-26", "value": 68.16, "name": "HDU-USYD-UNCC", "url": "http://www.visualqa.org/roe_2017.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.16, "maxval": 68.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "HDU-USYD-UNCC", "metric": "Metric(COCO Visual Question Answering (VQA) real images 2.0 open ended)?" }, { "date": "2017-07-26", "value": 68.07, "name": "DLAIT", "url": "http://www.visualqa.org/roe_2017.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.07, "maxval": 68.07, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "VQA: Visual Question Answering", "label": "DLAIT", "metric": "Metric(COCO Visual Question Answering (VQA) real images 2.0 open ended)?" } ], "name": "COCO Visual Question Answering (VQA) real images 2.0 open ended", "solved": false, "url": "http://visualqa.org/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image comprehension)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L121" } ], "solved": false, "url": null }, { "name": "Image classification", "attributes": [ "vision", "agi" ], "subproblems": [ "Image comprehension", "Pedestrian, bicycle & obstacle detection" ], "superproblems": [ "Vision" ], "metrics": [ { "measures": [ { "date": "2010-08-31", "value": 0.28191, "name": "NEC UIUC", "url": "http://image-net.org/challenges/LSVRC/2010/results", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.28191, "maxval": 0.28191, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ImageNet Large Scale Visual Recognition Competition 2010 (ILSVRC2010)", "label": "NEC UIUC", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2011-10-26", "value": 0.2577, "name": "XRCE", "url": "http://image-net.org/challenges/LSVRC/2011/results", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.2577, "maxval": 0.2577, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ImageNet Large Scale Visual Recognition Competition 2011 (ILSVRC2011)", "label": "XRCE", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2012-10-13", "value": 0.16422, "name": "AlexNet / SuperVision", "url": "http://image-net.org/challenges/LSVRC/2012/results.html", "min_date": null, "max_date": null, "algorithm_src_url": "https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks", "src_name": "ImageNet Classification with Deep Convolutional Neural Networks", "uncertainty": 0, "minval": 0.16422, "maxval": 0.16422, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ImageNet Large Scale Visual Recognition Competition 2012 (ILSVRC2012)", "label": "AlexNet / SuperVision", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2013-11-14", "value": 0.11743, "name": "Clarifai", "url": "http://www.image-net.org/challenges/LSVRC/2013/results.php", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.11743, "maxval": 0.11743, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ImageNet Large Scale Visual Recognition Competition 2013 (ILSVRC2013)", "label": "Clarifai", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2014-08-18", "value": 0.07405, "name": "VGG", "url": "http://image-net.org/challenges/LSVRC/2014/index", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.07405, "maxval": 0.07405, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ImageNet Large Scale Visual Recognition Competition 2014 (ILSVRC2014)", "label": "VGG", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2015-04-10", "value": 0.0458, "name": "withdrawn", "url": "https://arxiv.org/abs/1501.02876", "min_date": "2015-01-13", "max_date": "2015-07-06", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0458, "maxval": 0.0458, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": true, "papername": "Deep Image: Scaling up Image Recognition", "label": "withdrawn", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2015-12-10", "value": 0.03567, "name": "MSRA", "url": "http://image-net.org/challenges/LSVRC/2015/results", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.03567, "maxval": 0.03567, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [ "residual-networks" ], "offset": null, "notes": "", "withdrawn": false, "papername": "ILSVRC2015 Results", "label": "MSRA", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2016-09-26", "value": 0.02991, "name": "Trimps-Soushen", "url": "http://image-net.org/challenges/LSVRC/2016/results", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.02991, "maxval": 0.02991, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ILSVRC2016 Results", "label": "Trimps-Soushen", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2017-07-21", "value": 0.02251, "name": "SE-ResNet152 / WMW", "url": "http://image-net.org/challenges/LSVRC/2017/results", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.02251, "maxval": 0.02251, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ILSVRC2017 Results", "label": "SE-ResNet152 / WMW", "metric": "Metric(Imagenet Image Recognition) SOLVED" }, { "date": "2018-11-29", "value": 0.03, "name": "AmoebaNet-B", "url": "https://arxiv.org/abs/1811.06965", "min_date": "2018-11-16", "max_date": "2018-12-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.03, "maxval": 0.03, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism", "label": "AmoebaNet-B", "metric": "Metric(Imagenet Image Recognition) SOLVED" } ], "name": "Imagenet Image Recognition", "solved": true, "url": "http://image-net.org", "notes": "\nCorrectly label images from the Imagenet dataset. As of 2016, this includes:\n - Object localization for 1000 categories.\n - Object detection for 200 fully labeled categories.\n - Object detection from video for 30 fully labeled categories.\n - Scene classification for 365 scene categories (Joint with MIT Places team) on Places2 Database http://places2.csail.mit.edu.\n - Scene parsing for 150 stuff and discrete object categories (Joint with MIT Places team).\n", "scale": "Error rate", "target": 0.051, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Top-5 error rate", "data_path": "/home/pde/aip/data/vision.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L39" }, { "measures": [ { "date": "2008-07-01", "value": 67.0, "name": "STF", "url": "http://mi.eng.cam.ac.uk/~cipolla/publications/inproceedings/2008-CVPR-semantic-texton-forests.pdf", "min_date": "2008-01-01", "max_date": "2008-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.0, "maxval": 67.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Semantic Texton Forests for Image Categorization and Segmentation", "label": "STF", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2009-07-01", "value": 57.0, "name": "TextonBoost", "url": "http://research.microsoft.com/pubs/117885/ijcv07a.pdf", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 57.0, "maxval": 57.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "?? / 69.6 % (per-class / per-pixel) the unaries alone (no CRF on top)", "withdrawn": false, "papername": "TextonBoost for Image Understanding", "label": "TextonBoost", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2010-07-01", "value": 77.0, "name": "HCRF+CO", "url": "http://research.microsoft.com/en-us/um/people/pkohli/papers/lrkt_eccv2010.pdf", "min_date": "2010-01-01", "max_date": "2010-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 77.0, "maxval": 77.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Graph Cut based Inference with Co-occurrence Statistics", "label": "HCRF+CO", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2010-07-01", "value": 69.0, "name": "Auto-Context", "url": "http://pages.ucsd.edu/~ztu/publication/pami_autocontext.pdf", "min_date": "2010-01-01", "max_date": "2010-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 69.0, "maxval": 69.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Auto-Context and Its Application to High-Level Vision Tasks and 3D Brain Image Segmentation", "label": "Auto-Context", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2011-07-01", "value": 77.0, "name": "Are Spatial and Global Constraints Really Necessary for Segmentation?", "url": "http://infoscience.epfl.ch/record/169178/files/lucchi_ICCV11.pdf", "min_date": "2011-01-01", "max_date": "2011-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 77.0, "maxval": 77.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Several variants are examined, no single method attains the overall best results, i.e. both best per-class and per-pixel averages simultaneously. Indicated result corresponds to the method that we best on the average (per-class + per-pixel / 2). Experiment data available.", "withdrawn": false, "papername": "Are Spatial and Global Constraints Really Necessary for Segmentation?", "label": "Are Spatial and Global Co...", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2011-12-17", "value": 78.0, "name": "FC CRF", "url": "http://graphics.stanford.edu/projects/densecrf/densecrf.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 78.0, "maxval": 78.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Strong unary used provides 76.6% / 84.0%", "withdrawn": false, "papername": "Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials", "label": "FC CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2012-06-16", "value": 79.0, "name": "Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation", "url": "http://ttic.uchicago.edu/~rurtasun/publications/yao_et_al_cvpr12.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 79.0, "maxval": 79.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation", "label": "Describing the Scene as a...", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2012-07-01", "value": 80.0, "name": "Harmony Potentials", "url": "http://link.springer.com/article/10.1007%2Fs11263-011-0449-8", "min_date": "2012-01-01", "max_date": "2012-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 80.0, "maxval": 80.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "per-class % / per-pixel %", "withdrawn": false, "papername": "Harmony Potentials - Fusing Local and Global Scale for Semantic Image Segmentation", "label": "Harmony Potentials", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2012-10-07", "value": 76.0, "name": "Kernelized SSVM/CRF", "url": "https://infoscience.epfl.ch/record/180188/files/top.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 76.0, "maxval": 76.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "70 % / 73 % when using only local features (not considering global features)", "withdrawn": false, "papername": "Structured Image Segmentation using Kernelized Features", "label": "Kernelized SSVM/CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2012-10-07", "value": 72.8, "name": "PMG", "url": "http://users.cecs.anu.edu.au/~sgould/papers/eccv12-patchGraph.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.8, "maxval": 72.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "8% / 63.3% raw PatchMatchGraph accuracy, 72.8% / 79.0% when using Boosted CRF. Code available.", "withdrawn": false, "papername": "PatchMatchGraph: Building a Graph of Dense Patch Correspondences for Label Transfer", "label": "PMG", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2013-10-29", "value": 78.2, "name": "MPP", "url": "http://mediatum.ub.tum.de/doc/1175516/1175516.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 78.2, "maxval": 78.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Morphological Proximity Priors: Spatial Relationships for Semantic Segmentation", "label": "MPP", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" }, { "date": "2014-07-01", "value": 80.9, "name": "Large FC CRF", "url": "http://ai2-s2-pdfs.s3.amazonaws.com/daba/eb9185990f65f807c95ff4d09057c2bf1cf0.pdf", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 80.9, "maxval": 80.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Large-Scale Semantic Co-Labeling of Image Sets", "label": "Large FC CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-class)) ?" } ], "name": "MSRC-21 image semantic labelling (per-class)", "solved": false, "url": "http://jamie.shotton.org/work/data.html", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L254" }, { "measures": [ { "date": "2008-07-01", "value": 72.0, "name": "STF", "url": "http://mi.eng.cam.ac.uk/~cipolla/publications/inproceedings/2008-CVPR-semantic-texton-forests.pdf", "min_date": "2008-01-01", "max_date": "2008-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.0, "maxval": 72.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Semantic Texton Forests for Image Categorization and Segmentation", "label": "STF", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2009-07-01", "value": 72.0, "name": "TextonBoost", "url": "http://research.microsoft.com/pubs/117885/ijcv07a.pdf", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.0, "maxval": 72.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "?? / 69.6 % (per-class / per-pixel) the unaries alone (no CRF on top)", "withdrawn": false, "papername": "TextonBoost for Image Understanding", "label": "TextonBoost", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2010-07-01", "value": 87.0, "name": "HCRF+CO", "url": "http://research.microsoft.com/en-us/um/people/pkohli/papers/lrkt_eccv2010.pdf", "min_date": "2010-01-01", "max_date": "2010-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 87.0, "maxval": 87.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Graph Cut based Inference with Co-occurrence Statistics", "label": "HCRF+CO", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2010-07-01", "value": 78.0, "name": "Auto-Context", "url": "http://pages.ucsd.edu/~ztu/publication/pami_autocontext.pdf", "min_date": "2010-01-01", "max_date": "2010-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 78.0, "maxval": 78.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Auto-Context and Its Application to High-Level Vision Tasks and 3D Brain Image Segmentation", "label": "Auto-Context", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2011-07-01", "value": 85.0, "name": "Are Spatial and Global Constraints Really Necessary for Segmentation?", "url": "http://infoscience.epfl.ch/record/169178/files/lucchi_ICCV11.pdf", "min_date": "2011-01-01", "max_date": "2011-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 85.0, "maxval": 85.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Several variants are examined, no single method attains the overall best results, i.e. both best per-class and per-pixel averages simultaneously. Indicated result corresponds to the method that we best on the average (per-class + per-pixel / 2). Experiment data available.", "withdrawn": false, "papername": "Are Spatial and Global Constraints Really Necessary for Segmentation?", "label": "Are Spatial and Global Co...", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2011-12-17", "value": 86.0, "name": "FC CRF", "url": "http://graphics.stanford.edu/projects/densecrf/densecrf.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 86.0, "maxval": 86.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Strong unary used provides 76.6% / 84.0%", "withdrawn": false, "papername": "Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials", "label": "FC CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2012-06-16", "value": 86.0, "name": "Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation", "url": "http://ttic.uchicago.edu/~rurtasun/publications/yao_et_al_cvpr12.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 86.0, "maxval": 86.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation", "label": "Describing the Scene as a...", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2012-07-01", "value": 83.0, "name": "Harmony Potentials", "url": "http://link.springer.com/article/10.1007%2Fs11263-011-0449-8", "min_date": "2012-01-01", "max_date": "2012-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 83.0, "maxval": 83.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "per-class % / per-pixel %", "withdrawn": false, "papername": "Harmony Potentials - Fusing Local and Global Scale for Semantic Image Segmentation", "label": "Harmony Potentials", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2012-10-07", "value": 82.0, "name": "Kernelized SSVM/CRF", "url": "https://infoscience.epfl.ch/record/180188/files/top.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 82.0, "maxval": 82.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "70 % / 73 % when using only local features (not considering global features)", "withdrawn": false, "papername": "Structured Image Segmentation using Kernelized Features", "label": "Kernelized SSVM/CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2012-10-07", "value": 79.0, "name": "PatchMatchGraph", "url": "http://users.cecs.anu.edu.au/~sgould/papers/eccv12-patchGraph.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 79.0, "maxval": 79.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "8% / 63.3% raw PatchMatchGraph accuracy, 72.8% / 79.0% when using Boosted CRF. Code available.", "withdrawn": false, "papername": "PatchMatchGraph: Building a Graph of Dense Patch Correspondences for Label Transfer", "label": "PatchMatchGraph", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2013-10-29", "value": 85.0, "name": "MPP", "url": "http://mediatum.ub.tum.de/doc/1175516/1175516.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 85.0, "maxval": 85.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Morphological Proximity Priors: Spatial Relationships for Semantic Segmentation", "label": "MPP", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" }, { "date": "2014-07-01", "value": 86.8, "name": "Large FC CRF", "url": "http://ai2-s2-pdfs.s3.amazonaws.com/daba/eb9185990f65f807c95ff4d09057c2bf1cf0.pdf", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 86.8, "maxval": 86.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Large-Scale Semantic Co-Labeling of Image Sets", "label": "Large FC CRF", "metric": "Metric(MSRC-21 image semantic labelling (per-pixel)) ?" } ], "name": "MSRC-21 image semantic labelling (per-pixel)", "solved": false, "url": "http://jamie.shotton.org/work/data.html", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L267" }, { "measures": [ { "date": "2012-06-16", "value": 54.23, "name": "Receptive Field Learning", "url": "http://www.eecs.berkeley.edu/~jiayq/assets/pdf/cvpr12_pooling.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 54.23, "maxval": 54.23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Beyond Spatial Pyramids: Receptive Field Learning for Pooled Image Features", "label": "Receptive Field Learning", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2013-01-16", "value": 57.49, "name": "Stochastic Pooling", "url": "https://arxiv.org/abs/1301.3557", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 57.49, "maxval": 57.49, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stochastic Pooling for Regularization of Deep Convolutional Neural Networks", "label": "Stochastic Pooling", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2013-06-16", "value": 61.43, "name": "Maxout Networks", "url": "http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 61.43, "maxval": 61.43, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses convolution. Does not use dataset agumentation.", "withdrawn": false, "papername": "Maxout Networks", "label": "Maxout Networks", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2013-07-01", "value": 63.15, "name": "Tree Priors", "url": "http://www.cs.toronto.edu/~nitish/treebasedpriors.pdf", "min_date": "2013-01-01", "max_date": "2013-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 63.15, "maxval": 63.15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "The baseline Convnet + max pooling + dropout reaches 62.80% (without any tree prior).", "withdrawn": false, "papername": "Discriminative Transfer Learning with Tree-based Priors", "label": "Tree Priors", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2013-07-01", "value": 56.29, "name": "Smooth Pooling Regions", "url": "http://www.d2.mpi-inf.mpg.de/content/learning-smooth-pooling-regions-visual-recognition", "min_date": "2013-01-01", "max_date": "2013-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 56.29, "maxval": 56.29, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation.", "withdrawn": false, "papername": "Smooth Pooling Regions", "label": "Smooth Pooling Regions", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-04-14", "value": 64.32, "name": "NiN", "url": "http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 64.32, "maxval": 64.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet", "withdrawn": false, "papername": "Network in Network", "label": "NiN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-04-14", "value": 61.86, "name": "DNN+Probabilistic Maxout", "url": "http://openreview.net/document/28d9c3ab-fe88-4836-b898-403d207a037c#28d9c3ab-fe88-4836-b898-403d207a037c", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 61.86, "maxval": 61.86, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Improving Deep Neural Networks with Probabilistic Maxout Units", "label": "DNN+Probabilistic Maxout", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-06-21", "value": 60.8, "name": "Stable and Efficient Representation Learning with Nonnegativity Constraints ", "url": "http://jmlr.org/proceedings/papers/v32/line14.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 60.8, "maxval": 60.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "3-layers + multi-dict. 7 with 3-layers only. 3 with 1-layers only.", "withdrawn": false, "papername": "Stable and Efficient Representation Learning with Nonnegativity Constraints ", "label": "Stable and Efficient Repr...", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-07-01", "value": 65.43, "name": "DSN", "url": "http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 65.43, "maxval": 65.43, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model, without data augmentation.", "withdrawn": false, "papername": "Deeply-Supervised Nets", "label": "DSN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-09-22", "value": 75.7, "name": "SSCNN", "url": "https://arxiv.org/abs/1409.6070", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 75.7, "maxval": 75.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Spatially-sparse convolutional neural networks", "label": "SSCNN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2014-12-08", "value": 66.22, "name": "Deep Networks with Internal Selective Attention through Feedback Connections", "url": "http://papers.nips.cc/paper/5276-deep-networks-with-internal-selective-attention-through-feedback-connections.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 66.22, "maxval": 66.22, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Networks with Internal Selective Attention through Feedback Connections", "label": "Deep Networks with Intern...", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-02-15", "value": 66.29, "name": "ACN", "url": "https://arxiv.org/abs/1412.6806", "min_date": "2014-12-21", "max_date": "2015-04-13", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 66.29, "maxval": 66.29, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Striving for Simplicity: The All Convolutional Net", "label": "ACN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-02-19", "value": 69.17, "name": "NiN+APL", "url": "https://arxiv.org/abs/1412.6830", "min_date": "2014-12-21", "max_date": "2015-04-21", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 69.17, "maxval": 69.17, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses a piecewise linear activation function. 69.17% accuracy with data augmentation and 65.6% accuracy without data augmentation.", "withdrawn": false, "papername": "Learning Activation Functions to Improve Deep Neural Networks", "label": "NiN+APL", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-02-28", "value": 73.61, "name": "Fractional MP", "url": "https://arxiv.org/abs/1412.6071", "min_date": "2014-12-18", "max_date": "2015-05-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 73.61, "maxval": 73.61, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses 12 passes at test time. Reaches 68.55% when using a single pass at test time. Uses data augmentation during training.", "withdrawn": false, "papername": "Fractional Max-Pooling", "label": "Fractional MP", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-05-02", "value": 72.6, "name": "Tuned CNN", "url": "https://arxiv.org/abs/1502.05700", "min_date": "2015-02-19", "max_date": "2015-07-13", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.6, "maxval": 72.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Scalable Bayesian Optimization Using Deep Neural Networks", "label": "Tuned CNN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-06-08", "value": 68.25, "name": "RCNN-96", "url": "http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 68.25, "maxval": 68.25, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Convolutional Neural Network for Object Recognition", "label": "RCNN-96", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-07-01", "value": 68.53, "name": "MLR DNN", "url": "http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 68.53, "maxval": 68.53, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "With data augmentation, 65.82% without. Based on NiN architecture.", "withdrawn": false, "papername": "Multi-Loss Regularized Deep Neural Network", "label": "MLR DNN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-07-01", "value": 67.38, "name": "HD-CNN", "url": "https://sites.google.com/site/homepagezhichengyan/home/hdcnn", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.38, "maxval": 67.38, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "HD-CNN: Hierarchical Deep Convolutional Neural Network for Large Scale Visual Recognition", "label": "HD-CNN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-07-01", "value": 64.77, "name": "Deep Representation Learning with Target Coding", "url": "http://personal.ie.cuhk.edu.hk/~ccloy/files/aaai_2015_target_coding.pdf", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 64.77, "maxval": 64.77, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Representation Learning with Target Coding", "label": "Deep Representation Learn...", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-07-12", "value": 67.68, "name": "DCNN+GFE", "url": "http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.68, "maxval": 67.68, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-100", "withdrawn": false, "papername": "Deep Convolutional Neural Networks as Generic Feature Extractors", "label": "DCNN+GFE", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-08-16", "value": 59.75, "name": "RReLU", "url": "https://arxiv.org/abs/1505.00853", "min_date": "2015-05-05", "max_date": "2015-11-27", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 59.75, "maxval": 59.75, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Using Randomized Leaky ReLU", "withdrawn": false, "papername": "Empirical Evaluation of Rectified Activations in Convolution Network", "label": "RReLU", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-09-17", "value": 70.8, "name": "MIM", "url": "https://arxiv.org/abs/1508.00330", "min_date": "2015-08-03", "max_date": "2015-11-01", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.2, "minval": 70.6, "maxval": 71.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units", "label": "MIM", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-10-05", "value": 67.63, "name": "Tree+Max-Avg pooling", "url": "https://arxiv.org/abs/1509.08985", "min_date": "2015-09-30", "max_date": "2015-10-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.63, "maxval": 67.63, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model without data augmentation", "withdrawn": false, "papername": "Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree", "label": "Tree+Max-Avg pooling", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-10-11", "value": 69.12, "name": "SWWAE", "url": "https://arxiv.org/abs/1506.02351", "min_date": "2015-06-08", "max_date": "2016-02-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 69.12, "maxval": 69.12, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stacked What-Where Auto-encoders", "label": "SWWAE", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-11-09", "value": 71.14, "name": "BNM NiN", "url": "https://arxiv.org/abs/1511.02583", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 71.14, "maxval": 71.14, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "(k=5 maxout pieces in each maxout unit).", "withdrawn": false, "papername": "Batch-normalized Maxout Network in Network", "label": "BNM NiN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-11-18", "value": 72.44, "name": "CMsC", "url": "https://arxiv.org/abs/1511.05635", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.44, "maxval": 72.44, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Competitive Multi-scale Convolution", "label": "CMsC", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-12-07", "value": 68.4, "name": "Spectral Representations for Convolutional Neural Networks", "url": "http://papers.nips.cc/paper/5649-spectral-representations-for-convolutional-neural-networks.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 68.4, "maxval": 68.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Spectral Representations for Convolutional Neural Networks", "label": "Spectral Representations ...", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2015-12-07", "value": 67.76, "name": "VDN", "url": "http://people.idsia.ch/~rupesh/very_deep_learning/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.76, "maxval": 67.76, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Best result selected on test set. 67.61% average over multiple trained models.", "withdrawn": false, "papername": "Training Very Deep Networks", "label": "VDN", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2016-01-04", "value": 72.34, "name": "Fitnet4-LSUV", "url": "https://arxiv.org/abs/1511.06422", "min_date": "2015-11-19", "max_date": "2016-02-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.34, "maxval": 72.34, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Using RMSProp optimizer", "withdrawn": false, "papername": "All you need is a good init", "label": "Fitnet4-LSUV", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2016-01-07", "value": 75.72, "name": "Exponential Linear Units", "url": "https://arxiv.org/abs/1511.07289", "min_date": "2015-11-23", "max_date": "2016-02-22", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 75.72, "maxval": 75.72, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Without data augmentation.", "withdrawn": false, "papername": "Fast and Accurate Deep Network Learning by Exponential Linear Units", "label": "Exponential Linear Units", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2016-05-15", "value": 67.16, "name": "Universum Prescription", "url": "https://arxiv.org/abs/1511.03719", "min_date": "2015-11-11", "max_date": "2016-11-18", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.16, "maxval": 67.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Universum Prescription: Regularization using Unlabeled Data", "label": "Universum Prescription", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2016-05-20", "value": 77.28999999999999, "name": "ResNet-1001", "url": "https://arxiv.org/abs/1603.05027", "min_date": "2016-03-16", "max_date": "2016-07-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.22, "minval": 77.07, "maxval": 77.50999999999999, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Identity Mappings in Deep Residual Networks", "label": "ResNet-1001", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2016-07-10", "value": 73.45, "name": "ResNet+ELU", "url": "https://arxiv.org/abs/1604.04112", "min_date": "2016-04-14", "max_date": "2016-10-05", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.45, "maxval": 73.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Residual Networks with Exponential Linear Unit", "label": "ResNet+ELU", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2017-04-22", "value": 77.0, "name": "Evolution", "url": "https://arxiv.org/abs/1703.01041", "min_date": "2017-03-03", "max_date": "2017-06-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.0, "maxval": 77.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Large-Scale Evolution of Image Classifiers", "label": "Evolution", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2017-05-30", "value": 72.91, "name": "Deep Complex", "url": "https://arxiv.org/abs/1705.09792v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.91, "maxval": 72.91, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Complex Networks", "label": "Deep Complex", "metric": "Metric(CIFAR-100 Image Recognition) ?" }, { "date": "2017-06-06", "value": 69.0, "name": "NiN+Superclass+CDJ", "url": "https://arxiv.org/abs/1706.02003", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.0, "maxval": 69.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Convolutional Decision Jungle for Image Classification", "label": "NiN+Superclass+CDJ", "metric": "Metric(CIFAR-100 Image Recognition) ?" } ], "name": "CIFAR-100 Image Recognition", "solved": false, "url": "http://https://www.cs.toronto.edu/~kriz/cifar.html", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L135" }, { "measures": [ { "date": "2011-07-01", "value": 80.0, "name": "Hierarchical Kernel Descriptors", "url": "http://research.cs.washington.edu/istc/lfb/paper/cvpr11.pdf", "min_date": "2011-01-01", "max_date": "2011-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 80.0, "maxval": 80.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Object Recognition with Hierarchical Kernel Descriptors", "label": "Hierarchical Kernel Descr...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2011-07-01", "value": 79.6, "name": "An Analysis of Single-Layer Networks in Unsupervised Feature Learning ", "url": "http://www.stanford.edu/~acoates/papers/coatesleeng_aistats_2011.pdf", "min_date": "2011-01-01", "max_date": "2011-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 79.6, "maxval": 79.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "6% obtained using K-means over whitened patches, with triangle encoding and 4000 features (clusters).", "withdrawn": false, "papername": "An Analysis of Single-Layer Networks in Unsupervised Feature Learning ", "label": "An Analysis of Single-Lay...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-06-16", "value": 88.79, "name": "MCDNN", "url": "http://www.idsia.ch/~ciresan/data/cvpr2012.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 88.79, "maxval": 88.79, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Supplemental material, Technical Report", "withdrawn": false, "papername": "Multi-Column Deep Neural Networks for Image Classification ", "label": "MCDNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-06-26", "value": 82.2, "name": "Local Transformations", "url": "http://icml.cc/2012/papers/659.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 82.2, "maxval": 82.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "K= 4,000", "withdrawn": false, "papername": "Learning Invariant Representations with Local Transformations", "label": "Local Transformations", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-07-03", "value": 84.4, "name": "Improving neural networks by preventing co-adaptation of feature detectors", "url": "https://arxiv.org/abs/1207.0580", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 84.4, "maxval": 84.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "So called \"dropout\" method.", "withdrawn": false, "papername": "Improving neural networks by preventing co-adaptation of feature detectors", "label": "Improving neural networks...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-12-03", "value": 90.5, "name": "GP EI", "url": "http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 90.5, "maxval": 90.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Reaches 85.02% without data augmentation. With data augmented with horizontal reflections and translations, 90.5% accuracy on test set is achieved.", "withdrawn": false, "papername": "Practical Bayesian Optimization of Machine Learning Algorithms ", "label": "GP EI", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-12-03", "value": 89.0, "name": "DCNN", "url": "http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 89.0, "maxval": 89.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "87% error on the unaugmented data.", "withdrawn": false, "papername": "ImageNet Classification with Deep Convolutional Neural Networks", "label": "DCNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-12-03", "value": 83.96, "name": "Discriminative Learning of Sum-Product Networks", "url": "http://papers.nips.cc/paper/4516-discriminative-learning-of-sum-product-networks", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 83.96, "maxval": 83.96, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Discriminative Learning of Sum-Product Networks", "label": "Discriminative Learning o...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2012-12-03", "value": 79.7, "name": "Learning with Recursive Perceptual Representations", "url": "http://papers.nips.cc/paper/4747-learning-with-recursive-perceptual-representations", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 79.7, "maxval": 79.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Code size 1600.", "withdrawn": false, "papername": "Learning with Recursive Perceptual Representations", "label": "Learning with Recursive P...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2013-01-16", "value": 84.87, "name": "Stochastic Pooling", "url": "https://arxiv.org/abs/1301.3557", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 84.87, "maxval": 84.87, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stochastic Pooling for Regularization of Deep Convolutional Neural Networks", "label": "Stochastic Pooling", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2013-06-16", "value": 90.68, "name": "DropConnect", "url": "http://cs.nyu.edu/~wanli/dropc/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 90.68, "maxval": 90.68, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Regularization of Neural Networks using DropConnect", "label": "DropConnect", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2013-06-16", "value": 90.65, "name": "Maxout Networks", "url": "http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 90.65, "maxval": 90.65, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "This result was obtained using both convolution and synthetic translations / horizontal reflections of the training data. Reaches 88.32% when using convolution, but without any synthetic transformations of the training data.", "withdrawn": false, "papername": "Maxout Networks", "label": "Maxout Networks", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2013-07-01", "value": 80.02, "name": "Smooth Pooling Regions", "url": "http://www.d2.mpi-inf.mpg.de/content/learning-smooth-pooling-regions-visual-recognition", "min_date": "2013-01-01", "max_date": "2013-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 80.02, "maxval": 80.02, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning Smooth Pooling Regions for Visual Recognition", "label": "Smooth Pooling Regions", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-04-14", "value": 91.2, "name": "NiN", "url": "http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.2, "maxval": 91.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "The code for NIN available at https://github.com/mavenlin/cuda-convnet NIN + Dropout 89.6% NIN + Dropout + Data Augmentation 91.2%", "withdrawn": false, "papername": "Network In Network", "label": "NiN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-04-14", "value": 90.61, "name": "DNN+Probabilistic Maxout", "url": "http://openreview.net/document/28d9c3ab-fe88-4836-b898-403d207a037c#28d9c3ab-fe88-4836-b898-403d207a037c", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 90.61, "maxval": 90.61, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "65% without data augmentation. 61% when using data augmentation.", "withdrawn": false, "papername": "Improving Deep Neural Networks with Probabilistic Maxout Units", "label": "DNN+Probabilistic Maxout", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-06-21", "value": 82.9, "name": "Nonnegativity Constraints ", "url": "http://jmlr.org/proceedings/papers/v32/line14.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 82.9, "maxval": 82.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Full data, 3-layers + multi-dict. 4 with 3-layers only. 0 with 1-layers only.", "withdrawn": false, "papername": "Stable and Efficient Representation Learning with Nonnegativity Constraints ", "label": "Nonnegativity Constraints ", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-06-21", "value": 78.67, "name": "PCANet", "url": "https://arxiv.org/abs/1404.3606", "min_date": "2014-04-14", "max_date": "2014-08-28", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 78.67, "maxval": 78.67, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation. Multiple feature scales combined. 77.14% when using only a single scale.", "withdrawn": false, "papername": "PCANet: A Simple Deep Learning Baseline for Image Classification?", "label": "PCANet", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-07-01", "value": 91.78, "name": "DSN", "url": "http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.78, "maxval": 91.78, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model, with data augmentation: 91.78%. Without data augmentation: 90.22%.", "withdrawn": false, "papername": "Deeply-Supervised Nets", "label": "DSN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-08-28", "value": 82.18, "name": "CKN", "url": "https://arxiv.org/abs/1406.3332", "min_date": "2014-06-12", "max_date": "2014-11-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 82.18, "maxval": 82.18, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation.", "withdrawn": false, "papername": "Convolutional Kernel Networks", "label": "CKN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-09-22", "value": 93.72, "name": "SSCNN", "url": "https://arxiv.org/abs/1409.6070", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.72, "maxval": 93.72, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Spatially-sparse convolutional neural networks", "label": "SSCNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-12-08", "value": 90.78, "name": "Deep Networks with Internal Selective Attention through Feedback Connections", "url": "http://papers.nips.cc/paper/5276-deep-networks-with-internal-selective-attention-through-feedback-connections.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 90.78, "maxval": 90.78, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation", "withdrawn": false, "papername": "Deep Networks with Internal Selective Attention through Feedback Connections", "label": "Deep Networks with Intern...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2014-12-08", "value": 82.0, "name": "Discriminative Unsupervised Feature Learning with Convolutional Neural Networks", "url": "http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 82.0, "maxval": 82.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Unsupervised feature learning + linear SVM", "withdrawn": false, "papername": "Discriminative Unsupervised Feature Learning with Convolutional Neural Networks", "label": "Discriminative Unsupervis...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-02-13", "value": 86.7, "name": "An Analysis of Unsupervised Pre-training in Light of Recent Advances", "url": "https://arxiv.org/abs/1412.6597", "min_date": "2014-12-20", "max_date": "2015-04-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 86.7, "maxval": 86.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Unsupervised pre-training, with supervised fine-tuning. Uses dropout and data-augmentation.", "withdrawn": false, "papername": "An Analysis of Unsupervised Pre-training in Light of Recent Advances", "label": "An Analysis of Unsupervis...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-02-15", "value": 95.59, "name": "ACN", "url": "https://arxiv.org/abs/1412.6806", "min_date": "2014-12-21", "max_date": "2015-04-13", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 95.59, "maxval": 95.59, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "92% without data augmentation, 92.75% with small data augmentation, 95.59% when using agressive data augmentation and larger network.", "withdrawn": false, "papername": "Striving for Simplicity: The All Convolutional Net", "label": "ACN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-02-19", "value": 92.49, "name": "NiN+APL", "url": "https://arxiv.org/abs/1412.6830", "min_date": "2014-12-21", "max_date": "2015-04-21", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 92.49, "maxval": 92.49, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses an adaptive piecewise linear activation function. 92.49% accuracy with data augmentation and 90.41% accuracy without data augmentation.", "withdrawn": false, "papername": "Learning Activation Functions to Improve Deep Neural Networks", "label": "NiN+APL", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-02-28", "value": 96.53, "name": "Fractional MP", "url": "https://arxiv.org/abs/1412.6071", "min_date": "2014-12-18", "max_date": "2015-05-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 96.53, "maxval": 96.53, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses 100 passes at test time. Reaches 95.5% when using a single pass at test time, and 96.33% when using 12 passes.. Uses data augmentation during training.", "withdrawn": false, "papername": "Fractional Max-Pooling", "label": "Fractional MP", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-05-02", "value": 93.63, "name": "Tuned CNN", "url": "https://arxiv.org/abs/1502.05700", "min_date": "2015-02-19", "max_date": "2015-07-13", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.63, "maxval": 93.63, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Scalable Bayesian Optimization Using Deep Neural Networks", "label": "Tuned CNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-05-13", "value": 89.67, "name": "APAC", "url": "https://arxiv.org/abs/1505.03229", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 89.67, "maxval": 89.67, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "APAC: Augmented PAttern Classification with Neural Networks", "label": "APAC", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-05-31", "value": 75.86, "name": "FLSCNN", "url": "https://arxiv.org/abs/1503.04596", "min_date": "2015-03-16", "max_date": "2015-08-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 75.86, "maxval": 75.86, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation", "withdrawn": false, "papername": "Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network", "label": "FLSCNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-06-08", "value": 92.91, "name": "RCNN-96", "url": "http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 92.91, "maxval": 92.91, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Reaches 91.31% without data augmentation.", "withdrawn": false, "papername": "Recurrent Convolutional Neural Network for Object Recognition", "label": "RCNN-96", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-06-12", "value": 87.65, "name": "ReNet", "url": "https://arxiv.org/abs/1505.00393", "min_date": "2015-05-03", "max_date": "2015-07-23", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 87.65, "maxval": 87.65, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks", "label": "ReNet", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-07-01", "value": 92.45, "name": "cifar.torch", "url": "http://torch.ch/blog/2015/07/30/cifar.html", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 92.45, "maxval": 92.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Code available at https://github.com/szagoruyko/cifar.torch", "withdrawn": false, "papername": "cifar.torch", "label": "cifar.torch", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-07-01", "value": 91.88, "name": "MLR DNN", "url": "http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.88, "maxval": 91.88, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "With data augmentation, 90.45% without. Based on NiN architecture.", "withdrawn": false, "papername": "Multi-Loss Regularized Deep Neural Network", "label": "MLR DNN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-07-01", "value": 91.19, "name": "ELC", "url": "http://aad.informatik.uni-freiburg.de/papers/15-IJCAI-Extrapolation_of_Learning_Curves.pdf", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.19, "maxval": 91.19, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Based on the \"call convolutional\" architecture. which reaches 90.92% by itself.", "withdrawn": false, "papername": "Speeding up Automatic Hyperparameter Optimization of Deep Neural Networks by Extrapolation of Learning Curves", "label": "ELC", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-07-12", "value": 89.14, "name": "DCNN+GFE", "url": "http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 89.14, "maxval": 89.14, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-10", "withdrawn": false, "papername": "Deep Convolutional Neural Networks as Generic Feature Extractors", "label": "DCNN+GFE", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-08-16", "value": 88.8, "name": "RReLU", "url": "https://arxiv.org/abs/1505.00853", "min_date": "2015-05-05", "max_date": "2015-11-27", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 88.8, "maxval": 88.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Using Randomized Leaky ReLU", "withdrawn": false, "papername": "Empirical Evaluation of Rectified Activations in Convolution Network", "label": "RReLU", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-09-17", "value": 91.48, "name": "MIM", "url": "https://arxiv.org/abs/1508.00330", "min_date": "2015-08-03", "max_date": "2015-11-01", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.2, "minval": 91.28, "maxval": 91.68, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units", "label": "MIM", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-10-05", "value": 93.95, "name": "Tree+Max-Avg pooling", "url": "https://arxiv.org/abs/1509.08985", "min_date": "2015-09-30", "max_date": "2015-10-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.95, "maxval": 93.95, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model with data augmentation, 92.38% without.", "withdrawn": false, "papername": "Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree", "label": "Tree+Max-Avg pooling", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-10-11", "value": 92.23, "name": "SWWAE", "url": "https://arxiv.org/abs/1506.02351", "min_date": "2015-06-08", "max_date": "2016-02-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 92.23, "maxval": 92.23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stacked What-Where Auto-encoders", "label": "SWWAE", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-11-09", "value": 93.25, "name": "BNM NiN", "url": "https://arxiv.org/abs/1511.02583", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.25, "maxval": 93.25, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "(k=5 maxout pieces in each maxout unit). Reaches 92.15% without data augmentation.", "withdrawn": false, "papername": "Batch-normalized Maxout Network in Network", "label": "BNM NiN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-11-18", "value": 93.13, "name": "CMsC", "url": "https://arxiv.org/abs/1511.05635", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.13, "maxval": 93.13, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Competitive Multi-scale Convolution", "label": "CMsC", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-12-07", "value": 92.4, "name": "VDN", "url": "http://people.idsia.ch/~rupesh/very_deep_learning/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 92.4, "maxval": 92.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Best result selected on test set. 92.31% average over multiple trained models.", "withdrawn": false, "papername": "Training Very Deep Networks", "label": "VDN", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-12-07", "value": 91.73, "name": "BinaryConnect", "url": "http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.73, "maxval": 91.73, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "These results were obtained without using any data-augmentation.", "withdrawn": false, "papername": "BinaryConnect: Training Deep Neural Networks with binary weights during propagations", "label": "BinaryConnect", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-12-07", "value": 91.4, "name": "Spectral Representations for Convolutional Neural Networks", "url": "http://papers.nips.cc/paper/5649-spectral-representations-for-convolutional-neural-networks.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 91.4, "maxval": 91.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Spectral Representations for Convolutional Neural Networks", "label": "Spectral Representations ...", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2015-12-10", "value": 93.57, "name": "DRL", "url": "https://arxiv.org/abs/1512.03385", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.57, "maxval": 93.57, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Best performance reached with 110 layers. Using 1202 layers leads to 92.07%, 56 layers lead to 93.03%.", "withdrawn": false, "papername": "Deep Residual Learning for Image Recognition", "label": "DRL", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2016-01-04", "value": 94.16, "name": "Fitnet4-LSUV", "url": "https://arxiv.org/abs/1511.06422", "min_date": "2015-11-19", "max_date": "2016-02-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 94.16, "maxval": 94.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Only mirroring and random shifts, no extreme data augmentation. Uses thin deep residual net with maxout activations.", "withdrawn": false, "papername": "All you need is a good init", "label": "Fitnet4-LSUV", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2016-01-07", "value": 93.45, "name": "Exponential Linear Units", "url": "https://arxiv.org/abs/1511.07289", "min_date": "2015-11-23", "max_date": "2016-02-22", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.45, "maxval": 93.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Without data augmentation.", "withdrawn": false, "papername": "Fast and Accurate Deep Network Learning by Exponential Linear Units", "label": "Exponential Linear Units", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2016-05-15", "value": 93.34, "name": "Universum Prescription", "url": "https://arxiv.org/abs/1511.03719", "min_date": "2015-11-11", "max_date": "2016-11-18", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 93.34, "maxval": 93.34, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Universum Prescription: Regularization using Unlabeled Data", "label": "Universum Prescription", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2016-05-20", "value": 95.38, "name": "ResNet-1001", "url": "https://arxiv.org/abs/1603.05027", "min_date": "2016-03-16", "max_date": "2016-07-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.2, "minval": 95.17999999999999, "maxval": 95.58, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Identity Mappings in Deep Residual Networks", "label": "ResNet-1001", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2016-07-10", "value": 94.38, "name": "ResNet+ELU", "url": "https://arxiv.org/abs/1604.04112", "min_date": "2016-04-14", "max_date": "2016-10-05", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.38, "maxval": 94.38, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Residual Networks with Exponential Linear Unit", "label": "ResNet+ELU", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2017-02-15", "value": 96.35, "name": "Neural Architecture Search", "url": "https://arxiv.org/abs/1611.01578v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 96.35, "maxval": 96.35, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Architecture Search with Reinforcement Learning", "label": "Neural Architecture Search", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2017-04-22", "value": 95.6, "name": "Evolution ensemble", "url": "https://arxiv.org/abs/1703.01041", "min_date": "2017-03-03", "max_date": "2017-06-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 95.6, "maxval": 95.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Large-Scale Evolution of Image Classifiers", "label": "Evolution ensemble", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2017-04-22", "value": 94.6, "name": "Evolution", "url": "https://arxiv.org/abs/1703.01041", "min_date": "2017-03-03", "max_date": "2017-06-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.6, "maxval": 94.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Large-Scale Evolution of Image Classifiers", "label": "Evolution", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2017-05-30", "value": 94.4, "name": "Deep Complex", "url": "https://arxiv.org/abs/1705.09792v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.4, "maxval": 94.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Complex Networks", "label": "Deep Complex", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2017-07-16", "value": 94.6, "name": "RL+NT", "url": "https://arxiv.org/abs/1707.04873", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.6, "maxval": 94.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Reinforcement Learning for Architecture Search by Network Transformation", "label": "RL+NT", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2018-02-10", "value": 97.11, "name": "ENAS", "url": "https://arxiv.org/abs/1802.03268", "min_date": "2018-02-09", "max_date": "2018-02-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 97.11, "maxval": 97.11, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Efficient Neural Architecture Search via Parameter Sharing", "label": "ENAS", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" }, { "date": "2019-01-30", "value": 77.71, "name": "HyperGAN 100", "url": "https://arxiv.org/abs/1901.11058", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.71, "maxval": 77.71, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "HyperGAN: A Generative Model for Diverse, Performant Neural Networks", "label": "HyperGAN 100", "metric": "Metric(CIFAR-10 Image Recognition) SOLVED" } ], "name": "CIFAR-10 Image Recognition", "solved": true, "url": "http://https://www.cs.toronto.edu/~kriz/cifar.html", "notes": "", "scale": "Percentage correct", "target": 94, "target_source": "http://karpathy.github.io/2011/04/27/manually-classifying-cifar10/", "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L186" }, { "measures": [ { "date": "2012-07-01", "value": 4.9, "name": "Convolutional neural networks applied to house numbers digit classification", "url": "http://yann.lecun.com/exdb/publis/pdf/sermanet-icpr-12.pdf", "min_date": "2012-01-01", "max_date": "2012-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 4.9, "maxval": 4.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "ConvNet / MS / L4 / Padded", "withdrawn": false, "papername": "Convolutional neural networks applied to house numbers digit classification", "label": "Convolutional neural netw...", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2013-01-16", "value": 2.8, "name": "Stochastic Pooling", "url": "https://arxiv.org/abs/1301.3557", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.8, "maxval": 2.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "64-64-128 Stochastic Pooling", "withdrawn": false, "papername": "Stochastic Pooling for Regularization of Deep Convolutional Neural Networks", "label": "Stochastic Pooling", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2013-06-16", "value": 2.47, "name": "Maxout", "url": "http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.47, "maxval": 2.47, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "This result was obtained using convolution but not any synthetic transformations of the training data.", "withdrawn": false, "papername": "Maxout Networks", "label": "Maxout", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2013-06-16", "value": 1.94, "name": "Regularization of Neural Networks using DropConnect", "url": "http://cs.nyu.edu/~wanli/dropc/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.94, "maxval": 1.94, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Regularization of Neural Networks using DropConnect", "label": "Regularization of Neural ...", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2014-04-14", "value": 2.35, "name": "NiN", "url": "http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.35, "maxval": 2.35, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet", "withdrawn": false, "papername": "Network in Network", "label": "NiN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2014-04-14", "value": 2.16, "name": "DCNN", "url": "http://openreview.net/document/0c571b22-f4b6-4d58-87e4-99d7de42a893#0c571b22-f4b6-4d58-87e4-99d7de42a893", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.16, "maxval": 2.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "For classification of individual digits with a single network, error rate is 2.16%. For classification of the entire digit sequence (first paper doing this): error rate of 3.97%.", "withdrawn": false, "papername": "Multi-digit Number Recognition from Street View Imagery using Deep Convolutional Neural Networks", "label": "DCNN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2014-07-01", "value": 1.92, "name": "DSN", "url": "http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.92, "maxval": 1.92, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deeply-Supervised Nets", "label": "DSN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-05-31", "value": 3.96, "name": "FLSCNN", "url": "https://arxiv.org/abs/1503.04596", "min_date": "2015-03-16", "max_date": "2015-08-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 3.96, "maxval": 3.96, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation", "withdrawn": false, "papername": "Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network", "label": "FLSCNN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-06-08", "value": 1.77, "name": "RCNN-96", "url": "http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.77, "maxval": 1.77, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Without data augmentation", "withdrawn": false, "papername": "Recurrent Convolutional Neural Network for Object Recognition", "label": "RCNN-96", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-06-12", "value": 2.38, "name": "ReNet", "url": "https://arxiv.org/abs/1505.00393", "min_date": "2015-05-03", "max_date": "2015-07-23", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.38, "maxval": 2.38, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks", "label": "ReNet", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-07-01", "value": 1.92, "name": "MLR DNN", "url": "http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.92, "maxval": 1.92, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Based on NiN architecture.", "withdrawn": false, "papername": "Multi-Loss Regularized Deep Neural Network", "label": "MLR DNN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-09-17", "value": 1.97, "name": "MIM", "url": "https://arxiv.org/abs/1508.00330", "min_date": "2015-08-03", "max_date": "2015-11-01", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.08, "minval": 1.89, "maxval": 2.05, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units", "label": "MIM", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-10-05", "value": 1.69, "name": "Tree+Max-Avg pooling", "url": "https://arxiv.org/abs/1509.08985", "min_date": "2015-09-30", "max_date": "2015-10-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.69, "maxval": 1.69, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model without data augmentation", "withdrawn": false, "papername": "Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree", "label": "Tree+Max-Avg pooling", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-11-09", "value": 1.81, "name": "BNM NiN", "url": "https://arxiv.org/abs/1511.02583", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.81, "maxval": 1.81, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "(k=5 maxout pieces in each maxout unit).", "withdrawn": false, "papername": "Batch-normalized Maxout Network in Network", "label": "BNM NiN", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-11-18", "value": 1.76, "name": "CMsC", "url": "https://arxiv.org/abs/1511.05635", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.76, "maxval": 1.76, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Competitive Multi-scale Convolution", "label": "CMsC", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2015-12-07", "value": 2.15, "name": "BinaryConnect", "url": "http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 2.15, "maxval": 2.15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "BinaryConnect: Training Deep Neural Networks with binary weights during propagations", "label": "BinaryConnect", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" }, { "date": "2017-05-30", "value": 3.3, "name": "Deep Complex", "url": "https://arxiv.org/abs/1705.09792v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.3, "maxval": 3.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Complex Networks", "label": "Deep Complex", "metric": "Metric(Street View House Numbers (SVHN)) SOLVED" } ], "name": "Street View House Numbers (SVHN)", "solved": true, "url": "http://ufldl.stanford.edu/housenumbers/", "notes": "", "scale": "Percentage error", "target": 2.0, "target_source": "http://ufldl.stanford.edu/housenumbers/nips2011_housenumbers.pdf", "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L103" }, { "measures": [ { "date": "2002-07-01", "value": 0.63, "name": "Shape contexts", "url": "http://citeseerx.ist.psu.edu/viewdoc/download;jsessionid=B2AAC2BC3824F19757CAC66986D5F3FF?doi=10.1.1.18.8852&rep=rep1&type=pdf", "min_date": "2002-01-01", "max_date": "2002-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.63, "maxval": 0.63, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "K-NN, shape context matching (preprocessing: shape context feature extraction)", "withdrawn": false, "papername": "Shape matching and object recognition using shape contexts", "label": "Shape contexts", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2002-07-01", "value": 0.56, "name": "ISVM", "url": "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.88.9924&rep=rep1&type=pdf", "min_date": "2002-01-01", "max_date": "2002-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.56, "maxval": 0.56, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Virtual SVM, deg-9 poly, 2-pixel jittered (Preprocessing: deskewing)", "withdrawn": false, "papername": "Training Invariant Support Vector Machines", "label": "ISVM", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2003-07-01", "value": 1.19, "name": "CNN", "url": "", "min_date": "2003-01-01", "max_date": "2003-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.19, "maxval": 1.19, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "The ConvNN is based on the paper \"Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis\".", "withdrawn": null, "papername": "Convolutional Neural Networks", "label": "CNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2003-07-01", "value": 0.68, "name": "CNN+Gabor Filters", "url": "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.133.6559&rep=rep1&type=pdf", "min_date": "2003-01-01", "max_date": "2003-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.68, "maxval": 0.68, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Handwritten Digit Recognition using Convolutional Neural Networks and Gabor Filters", "label": "CNN+Gabor Filters", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2003-07-01", "value": 0.4, "name": "Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis", "url": "http://citeseerx.ist.psu.edu/viewdoc/download;jsessionid=D1C7D701BD39935473808DA5A93426C5?doi=10.1.1.160.8494&rep=rep1&type=pdf", "min_date": "2003-01-01", "max_date": "2003-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.4, "maxval": 0.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis", "label": "Best Practices for Convol...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2006-07-01", "value": 1.2, "name": "Reducing the dimensionality of data with neural networks", "url": "", "min_date": "2006-01-01", "max_date": "2006-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.2, "maxval": 1.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": null, "papername": "Reducing the dimensionality of data with neural networks", "label": "Reducing the dimensionali...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2006-07-01", "value": 0.39, "name": "Energy-Based Sparse Represenation", "url": "http://papers.nips.cc/paper/3112-efficient-learning-of-sparse-representations-with-an-energy-based-model", "min_date": "2006-01-01", "max_date": "2006-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.39, "maxval": 0.39, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Large conv. net, unsup pretraining, uses elastic distortions", "withdrawn": false, "papername": "Efficient Learning of Sparse Representations with an Energy-Based Model", "label": "Energy-Based Sparse Repre...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2007-07-01", "value": 0.62, "name": "invariant feature hierarchies", "url": "http://yann.lecun.com/exdb/publis/pdf/ranzato-cvpr-07.pdf", "min_date": "2007-01-01", "max_date": "2007-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.62, "maxval": 0.62, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Large conv. net, unsup features, no distortions", "withdrawn": false, "papername": "Unsupervised learning of invariant feature hierarchies with applications to object recognition", "label": "invariant feature hierarc...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2007-07-01", "value": 0.54, "name": "Deformation Models", "url": "http://www.keysers.net/daniel/files/Keysers--Deformation-Models--TPAMI2007.pdf", "min_date": "2007-01-01", "max_date": "2007-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.54, "maxval": 0.54, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "K-NN with non-linear deformation (IDM) (Preprocessing: shiftable edges)", "withdrawn": false, "papername": "Deformation Models for Image Recognition", "label": "Deformation Models", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2007-07-01", "value": 0.54, "name": "Trainable feature extractor", "url": "http://hal.inria.fr/docs/00/05/75/61/PDF/LauerSuenBlochPR.pdf", "min_date": "2007-01-01", "max_date": "2007-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.54, "maxval": 0.54, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Trainable feature extractor + SVMs, uses affine distortions", "withdrawn": false, "papername": "A trainable feature extractor for handwritten digit recognition", "label": "Trainable feature extractor", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2008-07-01", "value": 1.5, "name": "Deep learning via semi-supervised embedding", "url": "", "min_date": "2008-01-01", "max_date": "2008-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.5, "maxval": 1.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": null, "papername": "Deep learning via semi-supervised embedding", "label": "Deep learning via semi-su...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2008-07-01", "value": 1.12, "name": "DBN", "url": "", "min_date": "2008-01-01", "max_date": "2008-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.12, "maxval": 1.12, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": null, "papername": "CS81: Learning words with Deep Belief Networks", "label": "DBN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2008-07-01", "value": 0.59, "name": "Sparse Coding", "url": "http://www.inb.uni-luebeck.de/publikationen/pdfs/LaBaMa08c.pdf", "min_date": "2008-01-01", "max_date": "2008-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.59, "maxval": 0.59, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Unsupervised sparse features + SVM, no distortions", "withdrawn": false, "papername": "Simple Methods for High-Performance Digit Recognition Based on Sparse Coding", "label": "Sparse Coding", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2009-07-01", "value": 0.95, "name": "Deep Boltzmann Machines", "url": "http://www.utstat.toronto.edu/~rsalakhu/papers/dbm.pdf", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.95, "maxval": 0.95, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Boltzmann Machines", "label": "Deep Boltzmann Machines", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2009-07-01", "value": 0.94, "name": "Large-Margin kNN", "url": "", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.94, "maxval": 0.94, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": null, "papername": "Large-Margin kNN Classification using a Deep Encoder Network", "label": "Large-Margin kNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2009-07-01", "value": 0.82, "name": "CDBN", "url": "", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.82, "maxval": 0.82, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": null, "papername": "Convolutional Deep Belief Networks for Scalable Unsupervised Learning of Hierarchical Representations", "label": "CDBN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2009-07-01", "value": 0.53, "name": "The Best Multi-Stage Architecture", "url": "http://yann.lecun.com/exdb/publis/pdf/jarrett-iccv-09.pdf", "min_date": "2009-01-01", "max_date": "2009-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.53, "maxval": 0.53, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Large conv. net, unsup pretraining, no distortions", "withdrawn": false, "papername": "What is the Best Multi-Stage Architecture for Object Recognition?", "label": "The Best Multi-Stage Arch...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2010-03-01", "value": 0.35, "name": "DBSNN", "url": "https://arxiv.org/abs/1003.0358", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.35, "maxval": 0.35, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "6-layer NN 784-2500-2000-1500-1000-500-10 (on GPU), uses elastic distortions", "withdrawn": false, "papername": "Deep Big Simple Neural Nets Excel on Handwritten Digit Recognition", "label": "DBSNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2010-07-01", "value": 0.84, "name": "Supervised Translation-Invariant Sparse Coding", "url": "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.206.339&rep=rep1&type=pdf", "min_date": "2010-01-01", "max_date": "2010-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.84, "maxval": 0.84, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses sparse coding + svm.", "withdrawn": false, "papername": "Supervised Translation-Invariant Sparse Coding", "label": "Supervised Translation-In...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2011-07-01", "value": 0.69, "name": "On Optimization Methods for Deep Learning", "url": "http://ai.stanford.edu/~quocle/LeNgiCoaLahProNg11.pdf", "min_date": "2011-01-01", "max_date": "2011-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.69, "maxval": 0.69, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On Optimization Methods for Deep Learning", "label": "On Optimization Methods f...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2012-06-16", "value": 0.64, "name": "Receptive Field Learning", "url": "http://www.icsi.berkeley.edu/pubs/vision/beyondspatial12.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.64, "maxval": 0.64, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Beyond Spatial Pyramids: Receptive Field Learning for Pooled Image Features", "label": "Receptive Field Learning", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2012-06-16", "value": 0.23, "name": "MCDNN", "url": "http://www.idsia.ch/~ciresan/data/cvpr2012.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.23, "maxval": 0.23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multi-column Deep Neural Networks for Image Classification ", "label": "MCDNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2013-02-28", "value": 0.52, "name": "COSFIRE", "url": "http://www.cs.rug.nl/~george/articles/PAMI2013.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.52, "maxval": 0.52, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Trainable COSFIRE Filters for Keypoint Detection and Pattern Recognition", "label": "COSFIRE", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2013-06-16", "value": 0.45, "name": "Maxout Networks", "url": "http://jmlr.org/proceedings/papers/v28/goodfellow13.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.45, "maxval": 0.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses convolution. Does not use dataset augmentation.", "withdrawn": false, "papername": "Maxout Networks", "label": "Maxout Networks", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2013-06-16", "value": 0.21, "name": "DropConnect", "url": "http://cs.nyu.edu/~wanli/dropc/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.21, "maxval": 0.21, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Regularization of Neural Networks using DropConnect", "label": "DropConnect", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2013-07-01", "value": 0.75, "name": "Sparse Activity and Sparse Connectivity in Supervised Learning", "url": "http://jmlr.org/papers/v14/thom13a.html", "min_date": "2013-01-01", "max_date": "2013-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.75, "maxval": 0.75, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Sparse Activity and Sparse Connectivity in Supervised Learning", "label": "Sparse Activity and Spars...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2014-04-14", "value": 0.47, "name": "NiN", "url": "http://openreview.net/document/9b05a3bb-3a5e-49cb-91f7-0f482af65aea#9b05a3bb-3a5e-49cb-91f7-0f482af65aea", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.47, "maxval": 0.47, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "NIN + Dropout The code for NIN available at https://github.com/mavenlin/cuda-convnet", "withdrawn": false, "papername": "Network in Network", "label": "NiN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2014-06-21", "value": 0.62, "name": "PCANet", "url": "https://arxiv.org/abs/1404.3606", "min_date": "2014-04-14", "max_date": "2014-08-28", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.62, "maxval": 0.62, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation.", "withdrawn": false, "papername": "PCANet: A Simple Deep Learning Baseline for Image Classification?", "label": "PCANet", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2014-07-01", "value": 1.1, "name": "StrongNet", "url": "http://www.alglib.net/articles/tr-20140813-strongnet.pdf", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.1, "maxval": 1.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "StrongNet is a neural design which uses two innovations: (a) strong neurons - highly nonlinear neurons with multiple outputs and (b) mostly unsupervised architecture backpropagation-free design with all layers except for the last one being trained in a completely unsupervised setting.", "withdrawn": false, "papername": "StrongNet: mostly unsupervised image recognition with strong neurons", "label": "StrongNet", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2014-07-01", "value": 0.39, "name": "DSN", "url": "http://vcl.ucsd.edu/~sxie/2014/09/12/dsn-project/", "min_date": "2014-01-01", "max_date": "2014-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.39, "maxval": 0.39, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deeply-Supervised Nets", "label": "DSN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2014-08-28", "value": 0.39, "name": "CKN", "url": "https://arxiv.org/abs/1406.3332", "min_date": "2014-06-12", "max_date": "2014-11-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.39, "maxval": 0.39, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation.", "withdrawn": false, "papername": "Convolutional Kernel Networks", "label": "CKN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-02-03", "value": 0.78, "name": "Explaining and Harnessing Adversarial Examples", "url": "https://arxiv.org/abs/1412.6572", "min_date": "2014-12-20", "max_date": "2015-03-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.78, "maxval": 0.78, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "permutation invariant network used", "withdrawn": false, "papername": "Explaining and Harnessing Adversarial Examples", "label": "Explaining and Harnessing...", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-02-28", "value": 0.32, "name": "Fractional MP", "url": "https://arxiv.org/abs/1412.6071", "min_date": "2014-12-18", "max_date": "2015-05-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.32, "maxval": 0.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses 12 passes at test time. Reaches 0.5% when using a single pass at test time.", "withdrawn": false, "papername": "Fractional Max-Pooling", "label": "Fractional MP", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-03-11", "value": 0.35, "name": "C-SVDDNet", "url": "https://arxiv.org/abs/1412.7259", "min_date": "2014-12-23", "max_date": "2015-05-29", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.35, "maxval": 0.35, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "C-SVDDNet: An Effective Single-Layer Network for Unsupervised Feature Learning", "label": "C-SVDDNet", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-04-05", "value": 0.4, "name": "HOPE", "url": "https://arxiv.org/abs/1502.00702", "min_date": "2015-02-03", "max_date": "2015-06-06", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.4, "maxval": 0.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Hybrid Orthogonal Projection and Estimation (HOPE): A New Framework to Probe and Learn Neural Networks", "label": "HOPE", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-05-13", "value": 0.23, "name": "APAC", "url": "https://arxiv.org/abs/1505.03229", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.23, "maxval": 0.23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "APAC: Augmented PAttern Classification with Neural Networks", "label": "APAC", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-05-31", "value": 0.37, "name": "FLSCNN", "url": "https://arxiv.org/abs/1503.04596", "min_date": "2015-03-16", "max_date": "2015-08-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.37, "maxval": 0.37, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation", "withdrawn": false, "papername": "Enhanced Image Classification With a Fast-Learning Shallow Convolutional Neural Network", "label": "FLSCNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-06-08", "value": 0.31, "name": "RCNN-96", "url": "http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_004.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.31, "maxval": 0.31, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Convolutional Neural Network for Object Recognition", "label": "RCNN-96", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-06-12", "value": 0.45, "name": "ReNet", "url": "https://arxiv.org/abs/1505.00393", "min_date": "2015-05-03", "max_date": "2015-07-23", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.45, "maxval": 0.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReNet: A Recurrent Neural Network Based Alternative to Convolutional Networks", "label": "ReNet", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-07-01", "value": 0.71, "name": "Deep Fried Convnets", "url": "http://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Yang_Deep_Fried_Convnets_ICCV_2015_paper.pdf", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.71, "maxval": 0.71, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Uses about 10x fewer parameters than the reference model, which reaches 0.87%.", "withdrawn": false, "papername": "Deep Fried Convnets", "label": "Deep Fried Convnets", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-07-01", "value": 0.42, "name": "MLR DNN", "url": "http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=7258343", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.42, "maxval": 0.42, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Based on NiN architecture.", "withdrawn": false, "papername": "Multi-Loss Regularized Deep Neural Network", "label": "MLR DNN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-07-12", "value": 0.46, "name": "DCNN+GFE", "url": "http://www.isip.uni-luebeck.de/fileadmin/uploads/tx_wapublications/hertel_ijcnn_2015.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.46, "maxval": 0.46, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "feature extraction part of convnet is trained on imagenet (external training data), classification part is trained on cifar-10", "withdrawn": false, "papername": "Deep Convolutional Neural Networks as Generic Feature Extractors", "label": "DCNN+GFE", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-09-17", "value": 0.35, "name": "MIM", "url": "https://arxiv.org/abs/1508.00330", "min_date": "2015-08-03", "max_date": "2015-11-01", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.03, "minval": 0.31999999999999995, "maxval": 0.38, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On the Importance of Normalisation Layers in Deep Learning with Piecewise Linear Activation Units", "label": "MIM", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-10-05", "value": 0.29, "name": "Tree+Max-Avg pooling", "url": "https://arxiv.org/abs/1509.08985", "min_date": "2015-09-30", "max_date": "2015-10-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.29, "maxval": 0.29, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Single model without data augmentation", "withdrawn": false, "papername": "Generalizing Pooling Functions in Convolutional Neural Networks: Mixed, Gated, and Tree", "label": "Tree+Max-Avg pooling", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-11-09", "value": 0.24, "name": "BNM NiN", "url": "https://arxiv.org/abs/1511.02583", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.24, "maxval": 0.24, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "(k=5 maxout pieces in each maxout unit).", "withdrawn": false, "papername": "Batch-normalized Maxout Network in Network", "label": "BNM NiN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-11-18", "value": 0.33, "name": "CMsC", "url": "https://arxiv.org/abs/1511.05635", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.33, "maxval": 0.33, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Competitive Multi-scale Convolution", "label": "CMsC", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-12-07", "value": 1.01, "name": "BinaryConnect", "url": "http://papers.nips.cc/paper/5647-binaryconnect-training-deep-neural-networks-with-binary-weights-during-propagations.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.01, "maxval": 1.01, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Using 50% dropout", "withdrawn": false, "papername": "BinaryConnect: Training Deep Neural Networks with binary weights during propagations", "label": "BinaryConnect", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2015-12-07", "value": 0.45, "name": "VDN", "url": "http://people.idsia.ch/~rupesh/very_deep_learning/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.45, "maxval": 0.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Best result selected on test set. 0.46% average over multiple trained models.", "withdrawn": false, "papername": "Training Very Deep Networks", "label": "VDN", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2016-01-02", "value": 1.4, "name": "Convolutional Clustering", "url": "https://arxiv.org/abs/1511.06241", "min_date": "2015-11-19", "max_date": "2016-02-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 1.4, "maxval": 1.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "2 layers + multi dict.", "withdrawn": false, "papername": "Convolutional Clustering for Unsupervised Learning", "label": "Convolutional Clustering", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2016-01-04", "value": 0.38, "name": "Fitnet-LSUV-SVM", "url": "https://arxiv.org/abs/1511.06422", "min_date": "2015-11-19", "max_date": "2016-02-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.38, "maxval": 0.38, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "All you need is a good init", "label": "Fitnet-LSUV-SVM", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" }, { "date": "2019-01-30", "value": 0.6899999999999977, "name": "HyperGAN 100", "url": "https://arxiv.org/abs/1901.11058", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.6899999999999977, "maxval": 0.6899999999999977, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "HyperGAN: A Generative Model for Diverse, Performant Neural Networks", "label": "HyperGAN 100", "metric": "Metric(MNIST handwritten digit recognition) SOLVED" } ], "name": "MNIST handwritten digit recognition", "solved": true, "url": "http://yann.lecun.com/exdb/mnist/", "notes": "", "scale": "Percentage error", "target": 0.2, "target_source": "http://people.idsia.ch/~juergen/superhumanpatternrecognition.html", "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L238" }, { "measures": [ { "date": "2011-12-17", "value": 60.1, "name": "Receptive Fields", "url": "http://www.stanford.edu/~acoates/papers/coatesng_nips_2011.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 60.1, "maxval": 60.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Selecting Receptive Fields in Deep Networks ", "label": "Receptive Fields", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2012-06-26", "value": 58.7, "name": "Invariant Representations with Local Transformations", "url": "http://web.eecs.umich.edu/~honglak/icml12-invariantFeatureLearning.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 58.7, "maxval": 58.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning Invariant Representations with Local Transformations", "label": "Invariant Representations...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2012-07-01", "value": 64.5, "name": "RGB-D Based Object Recognition", "url": "http://homes.cs.washington.edu/~lfb/paper/iser12.pdf", "min_date": "2012-01-01", "max_date": "2012-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 64.5, "maxval": 64.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Hierarchical sparse coding using Matching Pursuit and K-SVD", "withdrawn": false, "papername": "Unsupervised Feature Learning for RGB-D Based Object Recognition", "label": "RGB-D Based Object Recogn...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2012-07-01", "value": 61.0, "name": "Simulated Fixations", "url": "http://papers.nips.cc/paper/4730-deep-learning-of-invariant-features-via-simulated-fixations-in-video", "min_date": "2012-01-01", "max_date": "2012-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 61.0, "maxval": 61.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Learning of Invariant Features via Simulated Fixations in Video", "label": "Simulated Fixations", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2012-12-03", "value": 62.3, "name": "Discriminative Learning of Sum-Product Networks", "url": "http://homes.cs.washington.edu/~rcg/papers/dspn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 62.3, "maxval": 62.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Discriminative Learning of Sum-Product Networks", "label": "Discriminative Learning o...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2012-12-03", "value": 56.5, "name": "Deep Learning of Invariant Features via Simulated Fixations in Video", "url": "http://ai.stanford.edu/~wzou/nips_ZouZhuNgYu12.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 56.5, "maxval": 56.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Trained also with video (unrelated to STL-10) obtained 61%", "withdrawn": false, "papername": "Deep Learning of Invariant Features via Simulated Fixations in Video", "label": "Deep Learning of Invarian...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2013-01-15", "value": 58.28, "name": "Pooling-Invariant", "url": "https://arxiv.org/abs/1302.5056v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 58.28, "maxval": 58.28, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "1600 codes, learnt using 2x PDL", "withdrawn": false, "papername": "Pooling-Invariant Image Feature Learning ", "label": "Pooling-Invariant", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2013-07-01", "value": 70.1, "name": "Multi-Task Bayesian Optimization", "url": "http://hips.seas.harvard.edu/files/swersky-multi-nips-2013.pdf", "min_date": "2013-01-01", "max_date": "2013-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 70.1, "maxval": 70.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Also uses CIFAR-10 training data", "withdrawn": false, "papername": "Multi-Task Bayesian Optimization", "label": "Multi-Task Bayesian Optim...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2014-02-24", "value": 61.0, "name": "No more meta-parameter tuning in unsupervised sparse feature learning", "url": "https://arxiv.org/abs/1402.5766", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 61.0, "maxval": 61.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "No more meta-parameter tuning in unsupervised sparse feature learning", "label": "No more meta-parameter tu...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2014-06-21", "value": 67.9, "name": "Nonnegativity Constraints ", "url": "http://jmlr.org/proceedings/papers/v32/line14.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 67.9, "maxval": 67.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "3-layers + multi-dict. 5 \u00b1 0.5 with 3-layers only. 6 \u00b1 0.6 with 1-layers only.", "withdrawn": false, "papername": "Stable and Efficient Representation Learning with Nonnegativity Constraints ", "label": "Nonnegativity Constraints ", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2014-06-23", "value": 68.0, "name": "DFF Committees", "url": "https://arxiv.org/abs/1406.5947", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 68.0, "maxval": 68.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Committees of deep feedforward networks trained with few data", "label": "DFF Committees", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2014-08-28", "value": 62.32, "name": "CKN", "url": "https://arxiv.org/abs/1406.3332", "min_date": "2014-06-12", "max_date": "2014-11-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 62.32, "maxval": 62.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "No data augmentation.", "withdrawn": false, "papername": "Convolutional Kernel Networks", "label": "CKN", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2014-12-08", "value": 72.8, "name": "Discriminative Unsupervised Feature Learning with Convolutional Neural Networks", "url": "http://papers.nips.cc/paper/5548-discriminative-unsupervised-feature-learning-with-convolutional-neural-networks.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 72.8, "maxval": 72.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Unsupervised feature learning + linear SVM", "withdrawn": false, "papername": "Discriminative Unsupervised Feature Learning with Convolutional Neural Networks", "label": "Discriminative Unsupervis...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2015-02-13", "value": 70.2, "name": "An Analysis of Unsupervised Pre-training in Light of Recent Advances", "url": "https://arxiv.org/abs/1412.6597", "min_date": "2014-12-20", "max_date": "2015-04-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 70.2, "maxval": 70.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "Unsupervised pre-training, with supervised fine-tuning. Uses dropout and data-augmentation.", "withdrawn": false, "papername": "An Analysis of Unsupervised Pre-training in Light of Recent Advances", "label": "An Analysis of Unsupervis...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2015-03-11", "value": 68.23, "name": "C-SVDDNet", "url": "https://arxiv.org/abs/1412.7259", "min_date": "2014-12-23", "max_date": "2015-05-29", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 68.23, "maxval": 68.23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "C-SVDDNet: An Effective Single-Layer Network for Unsupervised Feature Learning", "label": "C-SVDDNet", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2015-07-01", "value": 73.15, "name": "Deep Representation Learning with Target Coding", "url": "http://personal.ie.cuhk.edu.hk/~ccloy/files/aaai_2015_target_coding.pdf", "min_date": "2015-01-01", "max_date": "2015-12-31", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 73.15, "maxval": 73.15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Representation Learning with Target Coding", "label": "Deep Representation Learn...", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2015-10-11", "value": 74.33, "name": "SWWAE", "url": "https://arxiv.org/abs/1506.02351", "min_date": "2015-06-08", "max_date": "2016-02-14", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 74.33, "maxval": 74.33, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Stacked What-Where Auto-encoders", "label": "SWWAE", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2016-01-02", "value": 74.1, "name": "Convolutional Clustering", "url": "https://arxiv.org/abs/1511.06241", "min_date": "2015-11-19", "max_date": "2016-02-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 74.1, "maxval": 74.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "3 layers + multi dict. With 2 layers, reaches 71.4%", "withdrawn": false, "papername": "Convolutional Clustering for Unsupervised Learning", "label": "Convolutional Clustering", "metric": "Metric(STL-10 Image Recognition) ?" }, { "date": "2016-11-19", "value": 77.79, "name": "CC-GAN\u00b2", "url": "https://arxiv.org/abs/1611.06430v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.8, "minval": 76.99000000000001, "maxval": 78.59, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Semi-Supervised Learning with Context-Conditional Generative Adversarial Networks", "label": "CC-GAN\u00b2", "metric": "Metric(STL-10 Image Recognition) ?" } ], "name": "STL-10 Image Recognition", "solved": false, "url": "https://cs.stanford.edu/~acoates/stl10/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/awty.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L84" }, { "measures": [], "name": "Leeds Sport Poses", "solved": false, "url": null, "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Image classification)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/awty.py#L24" } ], "solved": false, "url": null }, { "name": "Recognise events in videos", "attributes": [], "subproblems": [], "superproblems": [ "Vision" ], "metrics": [ { "measures": [], "name": "YouTube-8M video labelling", "solved": false, "url": "https://research.google.com/youtube8m/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Recognise events in videos)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/vision.py#L43" } ], "solved": false, "url": null }, { "name": "Abstract strategy games", "attributes": [ "agi", "abstract-games" ], "subproblems": [ "Playing abstract games with extensive hints", "Superhuman mastery of arbitrary abstract strategy games", "Learning the rules of complex strategy games from examples", "Play an arbitrary abstract game, first learning the rules" ], "superproblems": [], "metrics": [], "solved": true, "url": null }, { "name": "Playing abstract games with extensive hints", "attributes": [ "abstract-games" ], "subproblems": [], "superproblems": [ "Abstract strategy games" ], "metrics": [ { "measures": [ { "date": "1984-12-31", "value": 1631, "name": "Novag Super Constellation 6502 4 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1631, "maxval": 1631, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Novag Super Constellation...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1985-12-31", "value": 1827, "name": "Mephisto Amsterdam 68000 12 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1827, "maxval": 1827, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Amsterdam 68000 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1986-12-31", "value": 1827, "name": "Mephisto Amsterdam 68000 12 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1827, "maxval": 1827, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Amsterdam 68000 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1987-12-31", "value": 1923, "name": "Mephisto Dallas 68020 14 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1923, "maxval": 1923, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Dallas 68020 14 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1988-12-31", "value": 1993, "name": "Mephisto MM 4 Turbo Kit 6502 16 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1993, "maxval": 1993, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto MM 4 Turbo Kit 6...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1989-12-31", "value": 2027, "name": "Mephisto Portorose 68020 12 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2027, "maxval": 2027, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Portorose 68020 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1990-12-31", "value": 2138, "name": "Mephisto Portorose 68030 36 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2138, "maxval": 2138, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Portorose 68030 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1991-12-31", "value": 2127, "name": "Mephisto Vancouver 68030 36 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2127, "maxval": 2127, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Vancouver 68030 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1992-12-31", "value": 2174, "name": "Chess Machine Schroder 3.0 ARM2 30 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2174, "maxval": 2174, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Chess Machine Schroder 3....", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1993-12-31", "value": 2235, "name": "Mephisto Genius 2.0 486/50-66 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2235, "maxval": 2235, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Mephisto Genius 2.0 486/5...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1995-12-31", "value": 2306, "name": "MChess Pro 5.0 Pentium 90 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2306, "maxval": 2306, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "MChess Pro 5.0 Pentium 90...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1996-12-31", "value": 2337, "name": "Rebel 8.0 Pentium 90 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2337, "maxval": 2337, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Rebel 8.0 Pentium 90 MHz", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1997-05-11", "value": 2725, "name": "Deep Blue", "url": "https://www.quora.com/What-was-Deep-Blues-Elo-rating", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 25, "minval": 2700, "maxval": 2750, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "What was Deep Blue's Elo rating? - Quora", "label": "Deep Blue", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1997-12-31", "value": 2418, "name": "HIARCS 6.0 49MB P200 MMX", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2418, "maxval": 2418, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "HIARCS 6.0 49MB P200 MMX", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1998-12-31", "value": 2460, "name": "Fritz 5.0 PB29% 67MB P200 MMX", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2460, "maxval": 2460, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Fritz 5.0 PB29% 67MB P200...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "1999-12-31", "value": 2594, "name": "Chess Tiger 12.0 DOS 128MB K6-2 450 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2594, "maxval": 2594, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Chess Tiger 12.0 DOS 128M...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2000-12-31", "value": 2607, "name": "Fritz 6.0 128MB K6-2 450 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2607, "maxval": 2607, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Fritz 6.0 128MB K6-2 450 ...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2001-12-31", "value": 2709, "name": "Chess Tiger 14.0 CB 256MB Athlon 1200", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2709, "maxval": 2709, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Chess Tiger 14.0 CB 256MB...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2002-12-31", "value": 2759, "name": "Deep Fritz 7.0 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2759, "maxval": 2759, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Fritz 7.0 256MB Athl...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2003-12-31", "value": 2791, "name": "Shredder 7.04 UCI 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2791, "maxval": 2791, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Shredder 7.04 UCI 256MB A...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2004-12-31", "value": 2800, "name": "Shredder 8.0 CB 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2800, "maxval": 2800, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Shredder 8.0 CB 256MB Ath...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2005-12-31", "value": 2808, "name": "Shredder 9.0 UCI 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2808, "maxval": 2808, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Shredder 9.0 UCI 256MB At...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2006-05-27", "value": 2995, "name": "Rybka 1.1 64bit", "url": "https://web.archive.org/web/20060531091049/http://www.computerchess.org.uk/ccrl/4040/rating_list_all.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 25, "minval": 2970, "maxval": 3020, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "CCRL 40/40 - Complete list", "label": "Rybka 1.1 64bit", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2006-12-31", "value": 2902, "name": "Rybka 1.2 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2902, "maxval": 2902, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Rybka 1.2 256MB Athlon 12...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2007-12-31", "value": 2935, "name": "Rybka 2.3.1 Arena 256MB Athlon 1200 MHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2935, "maxval": 2935, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Rybka 2.3.1 Arena 256MB A...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2008-12-31", "value": 3238, "name": "Deep Rybka 3 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3238, "maxval": 3238, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Rybka 3 2GB Q6600 2....", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2009-12-31", "value": 3232, "name": "Deep Rybka 3 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3232, "maxval": 3232, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Rybka 3 2GB Q6600 2....", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2010-08-07", "value": 3269, "name": "Rybka 4 64bit", "url": "https://web.archive.org/web/20100923131123/http://www.computerchess.org.uk/ccrl/4040/rating_list_all.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 22, "minval": 3247, "maxval": 3291, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "CCRL 40/40 - Complete list", "label": "Rybka 4 64bit", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2010-12-31", "value": 3227, "name": "Deep Rybka 3 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3227, "maxval": 3227, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Rybka 3 2GB Q6600 2....", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2011-12-31", "value": 3216, "name": "Deep Rybka 4 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3216, "maxval": 3216, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Rybka 4 2GB Q6600 2....", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2012-12-31", "value": 3221, "name": "Deep Rybka 4 x64 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3221, "maxval": 3221, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Deep Rybka 4 x64 2GB Q660...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2013-07-20", "value": 3248, "name": "Houdini 3 64bit", "url": "https://web.archive.org/web/20130415000000*/http://www.computerchess.org.uk/ccrl/4040/rating_list_all.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 16, "minval": 3232, "maxval": 3264, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Wayback Machine", "label": "Houdini 3 64bit", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2013-12-31", "value": 3241, "name": "Komodo 5.1 MP x64 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3241, "maxval": 3241, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Komodo 5.1 MP x64 2GB Q66...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2014-12-31", "value": 3295, "name": "Komodo 7.0 MP x64 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3295, "maxval": 3295, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Komodo 7.0 MP x64 2GB Q66...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2015-07-04", "value": 3332, "name": "Komodo 9", "url": "https://web.archive.org/web/20150708104805/http://www.computerchess.org.uk/ccrl/4040/rating_list_all.html", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 24, "minval": 3308, "maxval": 3356, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "CCRL 40/40 - Complete list", "label": "Komodo 9", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2015-12-31", "value": 3334, "name": "Stockfish 6 MP x64 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3334, "maxval": 3334, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Stockfish 6 MP x64 2GB Q6...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2016-12-31", "value": 3366, "name": "Komodo 9.1 MP x64 2GB Q6600 2.4 GHz", "url": "https://en.wikipedia.org/wiki/Swedish_Chess_Computer_Association#Rating_list_year-end_leaders", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3366, "maxval": 3366, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Swedish Chess Computer Association - Wikipedia", "label": "Komodo 9.1 MP x64 2GB Q66...", "metric": "Metric(Computer Chess) SOLVED" }, { "date": "2017-02-27", "value": 3393, "name": "Stockfish", "url": "https://web.archive.org/web/20170227044521/http://www.computerchess.org.uk/ccrl/4040/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 50, "minval": 3343, "maxval": 3443, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "CCRL 40/40 - Index", "label": "Stockfish", "metric": "Metric(Computer Chess) SOLVED" } ], "name": "Computer Chess", "solved": true, "url": null, "notes": "", "scale": "ELO rating", "target": 2882, "target_source": "https://en.wikipedia.org/w/index.php?title=Comparison_of_top_chess_players_throughout_history&oldid=777500496#Elo_system", "changeable": false, "graphed": true, "parent": "Problem(Playing abstract games with extensive hints)", "target_label": "Best human play", "axis_label": "ELO rating", "data_path": "/home/pde/aip/data/strategy_games.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/strategy_games.py#L59" }, { "measures": [ { "date": "2009-02-19", "value": 431, "name": "GnuGo", "url": "https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 431, "maxval": 431, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go with deep neural networks and tree search", "label": "GnuGo", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2011-01-01", "value": 1298, "name": "Pachi", "url": "https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1298, "maxval": 1298, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go with deep neural networks and tree search", "label": "Pachi", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2015-01-01", "value": 1929, "name": "Crazy Stone", "url": "https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1929, "maxval": 1929, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go with deep neural networks and tree search", "label": "Crazy Stone", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2015-10-01", "value": 3144, "name": "AlphaGo Fan", "url": "https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3144, "maxval": 3144, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go without human knowledge", "label": "AlphaGo Fan", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2016-03-09", "value": 3739, "name": "AlphaGo Lee", "url": "https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3739, "maxval": 3739, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go without human knowledge", "label": "AlphaGo Lee", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2016-12-29", "value": 4858, "name": "AlphaGo Master", "url": "https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4858, "maxval": 4858, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go without human knowledge", "label": "AlphaGo Master", "metric": "Metric(Computer Go) SOLVED" }, { "date": "2017-04-07", "value": 5185, "name": "AlphaGo Zero", "url": "https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5185, "maxval": 5185, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mastering the game of Go without human knowledge", "label": "AlphaGo Zero", "metric": "Metric(Computer Go) SOLVED" } ], "name": "Computer Go", "solved": true, "url": null, "notes": "", "scale": "ELO rating", "target": 3650, "target_source": "https://www.goratings.org/en/history/", "changeable": false, "graphed": true, "parent": "Problem(Playing abstract games with extensive hints)", "target_label": "Best human play", "axis_label": "ELO rating", "data_path": "/home/pde/aip/data/strategy_games.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/strategy_games.py#L68" } ], "solved": true, "url": null, "notes": "\n Complex abstract strategy games have been solved to super-human levels\n by computer systems with extensive rule-hinting and heuristics,\n in some cases combined with machine learning techniques.\n" }, { "name": "Superhuman mastery of arbitrary abstract strategy games", "attributes": [ "super", "abstract-games" ], "subproblems": [], "superproblems": [ "Abstract strategy games" ], "metrics": [ { "measures": [], "name": "mastering chess", "solved": false, "url": null, "notes": "\n Beating all humans at chess, given a corpus of past play amongst masters,\n but no human-crafted policy constraints and heuristics. This will probably fall out\n immediately once learning_abstract_game_rules is solved, since playing_with_hints\n has been solved.\n", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Superhuman mastery of arbitrary abstract strategy games)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/strategy_games.py#L74" } ], "solved": false, "url": null }, { "name": "Learning the rules of complex strategy games from examples", "attributes": [ "agi", "abstract-games" ], "subproblems": [], "superproblems": [ "Abstract strategy games" ], "metrics": [ { "measures": [], "name": "learning chess", "solved": false, "url": null, "notes": "\n Chess software contains hard-coded policy constraints for valid play; this metric is whether RL\n or other agents can correctly build those policy constraints from examples or oracles", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Learning the rules of complex strategy games from examples)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/strategy_games.py#L85" }, { "measures": [], "name": "learning go", "solved": false, "url": null, "notes": "\n Go software contains policy constraints for valid play and evaluating the number of\n liberties for groups. This metric is whether RL or other agents can correctly build those \n policy constraints from examples or oracles", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Learning the rules of complex strategy games from examples)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/strategy_games.py#L89" } ], "solved": false, "url": null }, { "name": "Play an arbitrary abstract game, first learning the rules", "attributes": [ "agi", "abstract-games" ], "subproblems": [], "superproblems": [ "Abstract strategy games" ], "metrics": [], "solved": false, "url": null }, { "name": "Play real-time computer & video games", "attributes": [ "world-modelling", "realtime-games", "agi", "language" ], "subproblems": [ "Games that require inventing novel language, forms of speech, or communication", "Simple video games" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Games that require inventing novel language, forms of speech, or communication", "attributes": [], "subproblems": [ "Games that require both understanding and speaking a language" ], "superproblems": [ "Play real-time computer & video games" ], "metrics": [], "solved": false, "url": null }, { "name": "Games that require both understanding and speaking a language", "attributes": [], "subproblems": [ "Games that require language comprehension" ], "superproblems": [ "Games that require inventing novel language, forms of speech, or communication" ], "metrics": [ { "measures": [], "name": "Starcraft", "solved": false, "url": null, "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Games that require both understanding and speaking a language)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/video_games.py#L10" } ], "solved": false, "url": null }, { "name": "Games that require language comprehension", "attributes": [ "agi", "languge" ], "subproblems": [], "superproblems": [ "Games that require both understanding and speaking a language" ], "metrics": [], "solved": false, "url": null }, { "name": "Simple video games", "attributes": [ "world-modelling", "realtime-games", "agi" ], "subproblems": [], "superproblems": [ "Play real-time computer & video games" ], "metrics": [ { "measures": [ { "date": "2012-07-14", "value": 103.2, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 103.2, "maxval": 103.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2012-07-19", "value": 939.2, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 939.2, "maxval": 939.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-02-26", "value": 3069.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1093.0, "minval": 1976.0, "maxval": 4162.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-07-15", "value": 813.5, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 813.5, "maxval": 813.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-09-22", "value": 1620.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1620.0, "maxval": 1620.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-09-22", "value": 634.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 634.0, "maxval": 634.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-11-20", "value": 4461.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4461.4, "maxval": 4461.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-11-20", "value": 3747.7, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 3747.7, "maxval": 3747.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-11-20", "value": 1486.5, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1486.5, "maxval": 1486.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-12-08", "value": 1033.4, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1033.4, "maxval": 1033.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2015-12-08", "value": 823.7, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 823.7, "maxval": 823.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-01-06", "value": 4203.8, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4203.8, "maxval": 4203.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-01-06", "value": 1334.7, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1334.7, "maxval": 1334.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-02-24", "value": 3213.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3213.5, "maxval": 3213.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-04-05", "value": 3941.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 3941.0, "maxval": 3941.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-04-10", "value": 945.3, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 945.3, "maxval": 945.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-04-10", "value": 518.4, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 518.4, "maxval": 518.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2016-04-10", "value": 182.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 182.1, "maxval": 182.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2017-03-10", "value": 994.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 994.0, "maxval": 994.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Alien) not solved" }, { "date": "2017-07-21", "value": 3166.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3166.0, "maxval": 3166.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Alien) not solved" } ], "name": "Atari 2600 Alien", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 6875, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 183.6, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 183.6, "maxval": 183.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2012-07-19", "value": 103.4, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 103.4, "maxval": 103.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-02-26", "value": 739.5, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 3024.0, "minval": -2284.5, "maxval": 3763.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-07-15", "value": 189.2, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 189.2, "maxval": 189.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-09-22", "value": 978.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 978.0, "maxval": 978.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-09-22", "value": 178.4, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 178.4, "maxval": 178.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-11-20", "value": 2354.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2354.5, "maxval": 2354.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-11-20", "value": 1793.3, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1793.3, "maxval": 1793.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-11-20", "value": 172.7, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 172.7, "maxval": 172.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-12-08", "value": 238.4, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 238.4, "maxval": 238.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2015-12-08", "value": 169.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 169.1, "maxval": 169.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-01-06", "value": 1838.9, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1838.9, "maxval": 1838.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-01-06", "value": 129.1, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 129.1, "maxval": 129.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-02-24", "value": 782.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 782.5, "maxval": 782.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-04-05", "value": 2296.8, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 2296.8, "maxval": 2296.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-04-10", "value": 283.9, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 283.9, "maxval": 283.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-04-10", "value": 263.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 263.9, "maxval": 263.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2016-04-10", "value": 173.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 173.0, "maxval": 173.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2017-03-10", "value": 112.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 112.0, "maxval": 112.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" }, { "date": "2017-07-21", "value": 1735.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1735.0, "maxval": 1735.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Amidar) SOLVED" } ], "name": "Atari 2600 Amidar", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 1676, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 537.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 537.0, "maxval": 537.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2012-07-19", "value": 628.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 628.0, "maxval": 628.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-02-26", "value": 3359.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 775.0, "minval": 2584.0, "maxval": 4134.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-07-15", "value": 1195.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1195.8, "maxval": 1195.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-09-22", "value": 4280.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4280.4, "maxval": 4280.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-09-22", "value": 3489.3, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 3489.3, "maxval": 3489.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-11-20", "value": 5393.2, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 5393.2, "maxval": 5393.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-11-20", "value": 4621.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4621.0, "maxval": 4621.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-11-20", "value": 3994.8, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3994.8, "maxval": 3994.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-12-08", "value": 10950.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 10950.6, "maxval": 10950.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2015-12-08", "value": 6060.8, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6060.8, "maxval": 6060.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-01-06", "value": 7672.1, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7672.1, "maxval": 7672.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-01-06", "value": 6548.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6548.9, "maxval": 6548.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-02-24", "value": 9011.6, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9011.6, "maxval": 9011.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-04-05", "value": 11477.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 11477.0, "maxval": 11477.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-04-10", "value": 14497.9, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14497.9, "maxval": 14497.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-04-10", "value": 5474.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5474.9, "maxval": 5474.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2016-04-10", "value": 3746.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3746.1, "maxval": 3746.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2017-03-10", "value": 1673.9, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1673.9, "maxval": 1673.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Assault) SOLVED" }, { "date": "2017-07-21", "value": 7203.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7203.0, "maxval": 7203.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Assault) SOLVED" } ], "name": "Atari 2600 Assault", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 1496, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 1332.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1332.0, "maxval": 1332.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2012-07-19", "value": 987.3, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 987.3, "maxval": 987.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-02-26", "value": 6012.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1744.0, "minval": 4268.0, "maxval": 7756.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-07-15", "value": 3324.7, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3324.7, "maxval": 3324.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-09-22", "value": 4359.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4359.0, "maxval": 4359.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-09-22", "value": 3170.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 3170.5, "maxval": 3170.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-11-20", "value": 28188.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28188.0, "maxval": 28188.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-11-20", "value": 17356.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 17356.5, "maxval": 17356.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-11-20", "value": 15840.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15840.0, "maxval": 15840.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-12-08", "value": 364200.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 364200.0, "maxval": 364200.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2015-12-08", "value": 16837.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16837.0, "maxval": 16837.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-01-06", "value": 31527.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31527.0, "maxval": 31527.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-01-06", "value": 22484.5, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22484.5, "maxval": 22484.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-02-24", "value": 18919.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18919.5, "maxval": 18919.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-04-05", "value": 375080.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 375080.0, "maxval": 375080.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-04-10", "value": 22140.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22140.5, "maxval": 22140.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-04-10", "value": 17244.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17244.5, "maxval": 17244.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2016-04-10", "value": 6723.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6723.0, "maxval": 6723.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2017-03-10", "value": 1440.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1440.0, "maxval": 1440.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" }, { "date": "2017-07-21", "value": 406211.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 406211.0, "maxval": 406211.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Asterix) SOLVED" } ], "name": "Atari 2600 Asterix", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 8503, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 89.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.0, "maxval": 89.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2012-07-19", "value": 907.3, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 907.3, "maxval": 907.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-02-26", "value": 1629.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 542.0, "minval": 1087.0, "maxval": 2171.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-07-15", "value": 933.6, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 933.6, "maxval": 933.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-09-22", "value": 1458.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1458.7, "maxval": 1458.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-09-22", "value": 1364.5, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1364.5, "maxval": 1364.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-11-20", "value": 2837.7, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2837.7, "maxval": 2837.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-11-20", "value": 2035.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2035.4, "maxval": 2035.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-11-20", "value": 734.7, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 734.7, "maxval": 734.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-12-08", "value": 1193.2, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1193.2, "maxval": 1193.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2015-12-08", "value": 1021.9, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1021.9, "maxval": 1021.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-01-06", "value": 2654.3, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2654.3, "maxval": 2654.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-01-06", "value": 1745.1, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1745.1, "maxval": 1745.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-02-24", "value": 2869.3, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2869.3, "maxval": 2869.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-04-05", "value": 1192.7, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1192.7, "maxval": 1192.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-04-10", "value": 5093.1, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5093.1, "maxval": 5093.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-04-10", "value": 4474.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4474.5, "maxval": 4474.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2016-04-10", "value": 3009.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3009.4, "maxval": 3009.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2017-03-10", "value": 1562.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1562.0, "maxval": 1562.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Asteroids) not solved" }, { "date": "2017-07-21", "value": 1516.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1516.0, "maxval": 1516.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Asteroids) not solved" } ], "name": "Atari 2600 Asteroids", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 13157, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 852.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 852.9, "maxval": 852.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2012-07-19", "value": 62687.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62687.0, "maxval": 62687.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-02-26", "value": 85641.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 17600.0, "minval": 68041.0, "maxval": 103241.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-07-15", "value": 629166.5, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 629166.5, "maxval": 629166.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-09-22", "value": 292491.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 292491.0, "maxval": 292491.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-09-22", "value": 279987.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 279987.0, "maxval": 279987.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-11-20", "value": 445360.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 445360.0, "maxval": 445360.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-11-20", "value": 382572.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 382572.0, "maxval": 382572.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-11-20", "value": 106056.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 106056.0, "maxval": 106056.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-12-08", "value": 423252.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 423252.0, "maxval": 423252.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2015-12-08", "value": 319688.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 319688.0, "maxval": 319688.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-01-06", "value": 357324.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 357324.0, "maxval": 357324.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-01-06", "value": 330647.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 330647.0, "maxval": 330647.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-02-24", "value": 340076.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 340076.0, "maxval": 340076.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-04-05", "value": 395762.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 395762.0, "maxval": 395762.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-04-10", "value": 911091.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 911091.0, "maxval": 911091.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-04-10", "value": 875822.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 875822.0, "maxval": 875822.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2016-04-10", "value": 772392.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 772392.0, "maxval": 772392.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2017-03-10", "value": 1267410.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1267410.0, "maxval": 1267410.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" }, { "date": "2017-07-21", "value": 841075.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 841075.0, "maxval": 841075.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Atlantis) SOLVED" } ], "name": "Atari 2600 Atlantis", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 29028, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 67.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.4, "maxval": 67.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2012-07-19", "value": 190.8, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 190.8, "maxval": 190.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-02-26", "value": 429.7, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 650.0, "minval": -220.3, "maxval": 1079.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-07-15", "value": 399.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 399.4, "maxval": 399.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-09-22", "value": 455.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 455.0, "maxval": 455.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-09-22", "value": 312.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 312.7, "maxval": 312.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-11-20", "value": 1611.9, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1611.9, "maxval": 1611.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-11-20", "value": 1129.3, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1129.3, "maxval": 1129.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-11-20", "value": 1030.6, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1030.6, "maxval": 1030.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-12-08", "value": 1004.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1004.6, "maxval": 1004.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2015-12-08", "value": 886.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 886.0, "maxval": 886.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-01-06", "value": 1054.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1054.6, "maxval": 1054.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-01-06", "value": 876.6, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 876.6, "maxval": 876.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-02-24", "value": 1103.3, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1103.3, "maxval": 1103.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-04-05", "value": 1503.1, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1503.1, "maxval": 1503.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-04-10", "value": 970.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 970.1, "maxval": 970.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-04-10", "value": 946.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 946.0, "maxval": 946.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2016-04-10", "value": 932.8, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 932.8, "maxval": 932.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2017-03-10", "value": 225.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 225.0, "maxval": 225.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" }, { "date": "2017-07-21", "value": 976.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 976.0, "maxval": 976.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Bank Heist) SOLVED" } ], "name": "Atari 2600 Bank Heist", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 734.4, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 16.2, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.2, "maxval": 16.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2012-07-19", "value": 15820.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15820.0, "maxval": 15820.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-02-26", "value": 26300.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 7725.0, "minval": 18575.0, "maxval": 34025.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-07-15", "value": 19938.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19938.0, "maxval": 19938.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-09-22", "value": 29900.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 29900.0, "maxval": 29900.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-09-22", "value": 23750.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 23750.0, "maxval": 23750.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-11-20", "value": 37150.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 37150.0, "maxval": 37150.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-11-20", "value": 31700.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 31700.0, "maxval": 31700.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-11-20", "value": 31320.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31320.0, "maxval": 31320.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-12-08", "value": 30650.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 30650.0, "maxval": 30650.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2015-12-08", "value": 24740.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24740.0, "maxval": 24740.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-01-06", "value": 31530.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31530.0, "maxval": 31530.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-01-06", "value": 25520.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 25520.0, "maxval": 25520.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-02-24", "value": 8220.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8220.0, "maxval": 8220.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-04-05", "value": 35520.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 35520.0, "maxval": 35520.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-04-10", "value": 20760.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20760.0, "maxval": 20760.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-04-10", "value": 12950.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12950.0, "maxval": 12950.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2016-04-10", "value": 11340.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11340.0, "maxval": 11340.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2017-03-10", "value": 16600.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16600.0, "maxval": 16600.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" }, { "date": "2017-07-21", "value": 28742.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28742.0, "maxval": 28742.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Battle Zone) not solved" } ], "name": "Atari 2600 Battle Zone", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 37800, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 1743.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1743.0, "maxval": 1743.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2012-07-19", "value": 929.4, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 929.4, "maxval": 929.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2013-12-19", "value": 5184, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5184, "maxval": 5184, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-02-26", "value": 6846.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1619.0, "minval": 5227.0, "maxval": 8465.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-07-15", "value": 3822.1, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3822.1, "maxval": 3822.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-09-22", "value": 9743.2, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 9743.2, "maxval": 9743.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-09-22", "value": 8627.5, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8627.5, "maxval": 8627.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-11-20", "value": 14591.3, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14591.3, "maxval": 14591.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-11-20", "value": 13772.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 13772.8, "maxval": 13772.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-11-20", "value": 12164.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12164.0, "maxval": 12164.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-12-08", "value": 37412.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 37412.2, "maxval": 37412.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2015-12-08", "value": 17417.2, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17417.2, "maxval": 17417.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-01-06", "value": 31181.3, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31181.3, "maxval": 31181.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-01-06", "value": 23384.2, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23384.2, "maxval": 23384.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-02-24", "value": 8299.4, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8299.4, "maxval": 8299.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-04-05", "value": 30276.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 30276.5, "maxval": 30276.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-04-10", "value": 24622.2, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24622.2, "maxval": 24622.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-04-10", "value": 22707.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22707.9, "maxval": 22707.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2016-04-10", "value": 13235.9, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13235.9, "maxval": 13235.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2017-03-10", "value": 744.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 744.0, "maxval": 744.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" }, { "date": "2017-07-21", "value": 14074.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14074.0, "maxval": 14074.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Beam Rider) SOLVED" } ], "name": "Atari 2600 Beam Rider", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 7456, "target_source": "https://arxiv.org/pdf/1312.5602.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2015-09-22", "value": 585.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 585.6, "maxval": 585.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-09-22", "value": 493.4, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 493.4, "maxval": 493.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-11-20", "value": 1472.6, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1472.6, "maxval": 1472.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-11-20", "value": 1225.4, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1225.4, "maxval": 1225.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-11-20", "value": 910.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 910.6, "maxval": 910.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-12-08", "value": 2178.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 2178.6, "maxval": 2178.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2015-12-08", "value": 1011.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1011.1, "maxval": 1011.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-01-06", "value": 1305.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1305.6, "maxval": 1305.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-01-06", "value": 865.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 865.9, "maxval": 865.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-02-24", "value": 1199.6, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1199.6, "maxval": 1199.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-04-05", "value": 3409.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 3409.0, "maxval": 3409.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-04-10", "value": 1433.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1433.4, "maxval": 1433.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-04-10", "value": 862.2, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 862.2, "maxval": 862.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2016-04-10", "value": 817.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 817.9, "maxval": 817.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2017-03-10", "value": 686.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 686.0, "maxval": 686.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" }, { "date": "2017-07-21", "value": 1645.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1645.0, "maxval": 1645.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Berzerk) SOLVED" } ], "name": "Atari 2600 Berzerk", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 2630.4, "target_source": "https://arxiv.org/abs/1511.06581v1", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" }, { "measures": [ { "date": "2012-07-14", "value": 36.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36.4, "maxval": 36.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2012-07-19", "value": 43.9, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 43.9, "maxval": 43.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-02-26", "value": 42.4, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 88.0, "minval": -45.6, "maxval": 130.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-07-15", "value": 54.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 54.0, "maxval": 54.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-09-22", "value": 56.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 56.5, "maxval": 56.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-09-22", "value": 50.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 50.4, "maxval": 50.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-11-20", "value": 68.1, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 68.1, "maxval": 68.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-11-20", "value": 65.7, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65.7, "maxval": 65.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-11-20", "value": 65.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65.5, "maxval": 65.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-12-08", "value": 69.6, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.6, "maxval": 69.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2015-12-08", "value": 50.4, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 50.4, "maxval": 50.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-01-06", "value": 52.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 52.0, "maxval": 52.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-01-06", "value": 47.9, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 47.9, "maxval": 47.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-02-24", "value": 102.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 102.1, "maxval": 102.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-04-05", "value": 46.7, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 46.7, "maxval": 46.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-04-10", "value": 41.8, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 41.8, "maxval": 41.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-04-10", "value": 36.2, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36.2, "maxval": 36.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2016-04-10", "value": 35.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 35.1, "maxval": 35.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2017-03-10", "value": 30.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 30.0, "maxval": 30.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Bowling) not solved" }, { "date": "2017-07-21", "value": 81.8, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.8, "maxval": 81.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Bowling) not solved" } ], "name": "Atari 2600 Bowling", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 154.8, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 9.8, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.8, "maxval": 9.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2012-07-19", "value": 44.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 44.0, "maxval": 44.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-02-26", "value": 71.8, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 8.0, "minval": 63.8, "maxval": 79.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-07-15", "value": 74.2, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.2, "maxval": 74.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-09-22", "value": 88.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 88.0, "maxval": 88.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-09-22", "value": 70.3, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 70.3, "maxval": 70.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-11-20", "value": 99.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 99.4, "maxval": 99.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-11-20", "value": 91.6, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 91.6, "maxval": 91.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-11-20", "value": 77.3, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.3, "maxval": 77.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-12-08", "value": 79.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 79.2, "maxval": 79.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2015-12-08", "value": 73.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.5, "maxval": 73.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-01-06", "value": 95.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 95.6, "maxval": 95.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-01-06", "value": 72.3, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.3, "maxval": 72.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-02-24", "value": 99.3, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 99.3, "maxval": 99.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-04-05", "value": 98.9, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 98.9, "maxval": 98.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-04-10", "value": 59.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 59.8, "maxval": 59.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-04-10", "value": 37.3, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 37.3, "maxval": 37.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2016-04-10", "value": 33.7, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33.7, "maxval": 33.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2017-03-10", "value": 49.8, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 49.8, "maxval": 49.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" }, { "date": "2017-07-21", "value": 97.8, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 97.8, "maxval": 97.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Boxing) SOLVED" } ], "name": "Atari 2600 Boxing", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 4.3, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 6.1, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.1, "maxval": 6.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2012-07-19", "value": 5.2, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.2, "maxval": 5.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2013-12-19", "value": 225, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 225, "maxval": 225, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-02-26", "value": 401.2, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 26.0, "minval": 375.2, "maxval": 427.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-07-15", "value": 313.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 313.0, "maxval": 313.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-09-22", "value": 385.5, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 385.5, "maxval": 385.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-09-22", "value": 354.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 354.5, "maxval": 354.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-11-20", "value": 418.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 418.5, "maxval": 418.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-11-20", "value": 411.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 411.6, "maxval": 411.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-11-20", "value": 345.3, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 345.3, "maxval": 345.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-12-08", "value": 368.9, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 368.9, "maxval": 368.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2015-12-08", "value": 354.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 354.6, "maxval": 354.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-01-06", "value": 373.9, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 373.9, "maxval": 373.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-01-06", "value": 343.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 343.0, "maxval": 343.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-02-24", "value": 344.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 344.1, "maxval": 344.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-04-05", "value": 366.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 366.0, "maxval": 366.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-04-10", "value": 766.8, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 766.8, "maxval": 766.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-04-10", "value": 681.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 681.9, "maxval": 681.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2016-04-10", "value": 551.6, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 551.6, "maxval": 551.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2017-03-10", "value": 9.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.5, "maxval": 9.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" }, { "date": "2017-07-21", "value": 748.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 748.0, "maxval": 748.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Breakout) SOLVED" } ], "name": "Atari 2600 Breakout", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 31.8, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 4647.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4647.0, "maxval": 4647.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2012-07-19", "value": 8803.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8803.0, "maxval": 8803.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-02-26", "value": 8309.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 5237.0, "minval": 3072.0, "maxval": 13546.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-07-15", "value": 6296.9, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6296.9, "maxval": 6296.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-09-22", "value": 4657.7, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4657.7, "maxval": 4657.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-09-22", "value": 3973.9, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 3973.9, "maxval": 3973.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-11-20", "value": 7561.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7561.4, "maxval": 7561.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-11-20", "value": 5409.4, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 5409.4, "maxval": 5409.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-11-20", "value": 4881.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4881.0, "maxval": 4881.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-12-08", "value": 5570.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 5570.2, "maxval": 5570.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2015-12-08", "value": 3853.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3853.5, "maxval": 3853.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-01-06", "value": 4463.2, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4463.2, "maxval": 4463.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-01-06", "value": 3489.1, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3489.1, "maxval": 3489.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-02-24", "value": 49065.8, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 49065.8, "maxval": 49065.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-04-05", "value": 7687.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 7687.5, "maxval": 7687.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-04-10", "value": 3755.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3755.8, "maxval": 3755.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-04-10", "value": 3306.5, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3306.5, "maxval": 3306.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2016-04-10", "value": 1997.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1997.0, "maxval": 1997.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2017-03-10", "value": 7783.9, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7783.9, "maxval": 7783.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" }, { "date": "2017-07-21", "value": 9646.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9646.0, "maxval": 9646.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Centipede) SOLVED" } ], "name": "Atari 2600 Centipede", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 11963, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 16.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.9, "maxval": 16.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2012-07-19", "value": 1582.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1582.0, "maxval": 1582.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-02-26", "value": 6687.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2916.0, "minval": 3771.0, "maxval": 9603.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-07-15", "value": 3191.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3191.8, "maxval": 3191.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-09-22", "value": 6126.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 6126.0, "maxval": 6126.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-09-22", "value": 5017.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 5017.0, "maxval": 5017.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-11-20", "value": 11215.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11215.0, "maxval": 11215.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-11-20", "value": 5809.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 5809.0, "maxval": 5809.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-11-20", "value": 3784.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3784.0, "maxval": 3784.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-12-08", "value": 8058.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 8058.0, "maxval": 8058.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2015-12-08", "value": 3495.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3495.0, "maxval": 3495.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-01-06", "value": 8600.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8600.0, "maxval": 8600.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-01-06", "value": 4635.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4635.0, "maxval": 4635.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-02-24", "value": 775.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 775.0, "maxval": 775.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-04-05", "value": 13185.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 13185.0, "maxval": 13185.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-04-10", "value": 10150.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10150.0, "maxval": 10150.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-04-10", "value": 7021.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7021.0, "maxval": 7021.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2016-04-10", "value": 4669.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4669.0, "maxval": 4669.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2017-03-10", "value": 3710.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3710.0, "maxval": 3710.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" }, { "date": "2017-07-21", "value": 15600.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15600.0, "maxval": 15600.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Chopper Command) SOLVED" } ], "name": "Atari 2600 Chopper Command", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 9882, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 149.8, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 149.8, "maxval": 149.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2012-07-19", "value": 23411.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23411.0, "maxval": 23411.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-02-26", "value": 114103.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 22797.0, "minval": 91306.0, "maxval": 136900.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-07-15", "value": 65451.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65451.0, "maxval": 65451.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-09-22", "value": 110763.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 110763.0, "maxval": 110763.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-09-22", "value": 98128.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 98128.0, "maxval": 98128.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-11-20", "value": 143570.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 143570.0, "maxval": 143570.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-11-20", "value": 124566.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 124566.0, "maxval": 124566.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-11-20", "value": 117282.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 117282.0, "maxval": 117282.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-12-08", "value": 127853.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 127853.0, "maxval": 127853.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2015-12-08", "value": 113782.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 113782.0, "maxval": 113782.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-01-06", "value": 141161.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 141161.0, "maxval": 141161.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-01-06", "value": 127512.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 127512.0, "maxval": 127512.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-02-24", "value": 119679.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 119679.0, "maxval": 119679.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-04-05", "value": 162224.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 162224.0, "maxval": 162224.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-04-10", "value": 138518.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 138518.0, "maxval": 138518.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-04-10", "value": 112646.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 112646.0, "maxval": 112646.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2016-04-10", "value": 101624.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 101624.0, "maxval": 101624.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2017-03-10", "value": 26430.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26430.0, "maxval": 26430.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" }, { "date": "2017-07-21", "value": 179877.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 179877.0, "maxval": 179877.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Crazy Climber) SOLVED" } ], "name": "Atari 2600 Crazy Climber", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 35411, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 0.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2012-07-19", "value": 520.5, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 520.5, "maxval": 520.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-02-26", "value": 9711.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2406.0, "minval": 7305.0, "maxval": 12117.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-07-15", "value": 14880.1, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14880.1, "maxval": 14880.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-09-22", "value": 12550.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 12550.7, "maxval": 12550.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-09-22", "value": 12149.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 12149.4, "maxval": 12149.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-11-20", "value": 60813.3, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 60813.3, "maxval": 60813.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-11-20", "value": 58044.2, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 58044.2, "maxval": 58044.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-11-20", "value": 56322.8, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56322.8, "maxval": 56322.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-12-08", "value": 73371.3, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 73371.3, "maxval": 73371.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2015-12-08", "value": 69803.4, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69803.4, "maxval": 69803.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-01-06", "value": 71846.4, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71846.4, "maxval": 71846.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-01-06", "value": 61277.5, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61277.5, "maxval": 61277.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-02-24", "value": 63644.9, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63644.9, "maxval": 63644.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-04-05", "value": 72878.6, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 72878.6, "maxval": 72878.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-04-10", "value": 115201.9, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 115201.9, "maxval": 115201.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-04-10", "value": 113308.4, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 113308.4, "maxval": 113308.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2016-04-10", "value": 84997.5, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 84997.5, "maxval": 84997.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2017-03-10", "value": 1166.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1166.5, "maxval": 1166.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" }, { "date": "2017-07-21", "value": 130955.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 130955.0, "maxval": 130955.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Demon Attack) SOLVED" } ], "name": "Atari 2600 Demon Attack", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 3401, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": -16.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -16.0, "maxval": -16.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2012-07-19", "value": -13.1, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -13.1, "maxval": -13.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-02-26", "value": -18.1, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2.0, "minval": -20.1, "maxval": -16.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-07-15", "value": -11.3, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -11.3, "maxval": -11.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-09-22", "value": -6.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -6.0, "maxval": -6.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-09-22", "value": -6.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -6.6, "maxval": -6.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-11-20", "value": 0.1, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-11-20", "value": -0.8, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.8, "maxval": -0.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-11-20", "value": -5.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -5.5, "maxval": -5.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-12-08", "value": -0.3, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.3, "maxval": -0.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2015-12-08", "value": -10.7, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -10.7, "maxval": -10.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-01-06", "value": 18.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.5, "maxval": 18.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-01-06", "value": 16.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.0, "maxval": 16.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-02-24", "value": -11.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -11.5, "maxval": -11.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-04-05", "value": -12.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -12.5, "maxval": -12.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-04-10", "value": 0.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-04-10", "value": 0.1, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2016-04-10", "value": -0.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.1, "maxval": -0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2017-03-10", "value": 0.2, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.2, "maxval": 0.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" }, { "date": "2017-07-21", "value": 2.5, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2.5, "maxval": 2.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Double Dunk) SOLVED" } ], "name": "Atari 2600 Double Dunk", "solved": true, "url": null, "notes": "", "scale": "Score", "target": -15.5, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 159.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 159.4, "maxval": 159.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2012-07-19", "value": 129.1, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 129.1, "maxval": 129.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2013-12-19", "value": 661, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 661, "maxval": 661, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-02-26", "value": 301.8, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 24.0, "minval": 277.8, "maxval": 325.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-07-15", "value": 71.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.0, "maxval": 71.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-09-22", "value": 729.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 729.0, "maxval": 729.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-09-22", "value": 626.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 626.7, "maxval": 626.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-11-20", "value": 2258.2, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2258.2, "maxval": 2258.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-11-20", "value": 2077.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2077.4, "maxval": 2077.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-11-20", "value": 1211.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1211.8, "maxval": 1211.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-12-08", "value": 2223.9, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 2223.9, "maxval": 2223.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2015-12-08", "value": 1216.6, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1216.6, "maxval": 1216.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-01-06", "value": 2093.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2093.0, "maxval": 2093.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-01-06", "value": 1831.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1831.0, "maxval": 1831.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-02-24", "value": 2002.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2002.1, "maxval": 2002.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-04-05", "value": 2306.4, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 2306.4, "maxval": 2306.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-04-10", "value": -82.2, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -82.2, "maxval": -82.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-04-10", "value": -82.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -82.5, "maxval": -82.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2016-04-10", "value": -82.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -82.5, "maxval": -82.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2017-03-10", "value": 95.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 95.0, "maxval": 95.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" }, { "date": "2017-07-21", "value": 3454.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3454.0, "maxval": 3454.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Enduro) SOLVED" } ], "name": "Atari 2600 Enduro", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 309.6, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": -85.1, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -85.1, "maxval": -85.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2012-07-19", "value": -89.5, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -89.5, "maxval": -89.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-02-26", "value": -0.8, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 19.0, "minval": -19.8, "maxval": 18.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-07-15", "value": 4.6, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.6, "maxval": 4.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-09-22", "value": -1.6, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -1.6, "maxval": -1.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-09-22", "value": -4.9, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -4.9, "maxval": -4.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-11-20", "value": 46.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 46.4, "maxval": 46.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-11-20", "value": 15.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 15.5, "maxval": 15.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-11-20", "value": -4.1, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -4.1, "maxval": -4.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-12-08", "value": 17.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 17.0, "maxval": 17.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2015-12-08", "value": 3.2, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.2, "maxval": 3.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-01-06", "value": 39.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 39.5, "maxval": 39.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-01-06", "value": 9.8, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.8, "maxval": 9.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-02-24", "value": 45.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 45.1, "maxval": 45.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-04-05", "value": 41.3, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 41.3, "maxval": 41.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-04-10", "value": 22.6, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22.6, "maxval": 22.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-04-10", "value": 18.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.8, "maxval": 18.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2016-04-10", "value": 13.6, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13.6, "maxval": 13.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2017-03-10", "value": -49.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -49.0, "maxval": -49.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" }, { "date": "2017-07-21", "value": 8.9, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.9, "maxval": 8.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Fishing Derby) SOLVED" } ], "name": "Atari 2600 Fishing Derby", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 5.5, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 19.7, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.7, "maxval": 19.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2012-07-19", "value": 19.1, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.1, "maxval": 19.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-02-26", "value": 30.3, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 30.3, "maxval": 30.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-07-15", "value": 10.2, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10.2, "maxval": 10.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-09-10", "value": 27.0, "name": "MP-EB", "url": "https://arxiv.org/abs/1507.00814", "min_date": "2015-07-03", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 27.0, "maxval": 27.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Incentivizing Exploration In Reinforcement Learning With Deep Predictive Models", "label": "MP-EB", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-09-22", "value": 30.8, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 30.8, "maxval": 30.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-09-22", "value": 26.9, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 26.9, "maxval": 26.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-11-20", "value": 33.3, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 33.3, "maxval": 33.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-11-20", "value": 0.2, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.2, "maxval": 0.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-11-20", "value": 0.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-12-08", "value": 28.8, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.8, "maxval": 28.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2015-12-08", "value": 28.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 28.2, "maxval": 28.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-01-06", "value": 33.7, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33.7, "maxval": 33.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-01-06", "value": 28.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.9, "maxval": 28.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-02-24", "value": 33.4, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33.4, "maxval": 33.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-04-05", "value": 33.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 33.0, "maxval": 33.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-04-10", "value": 0.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-04-10", "value": 0.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-04-10", "value": 0.1, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.1, "maxval": 0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-08-22", "value": 30.48, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 30.48, "maxval": 30.48, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2016-12-13", "value": 34.0, "name": "TRPO-hash", "url": "https://arxiv.org/abs/1611.04717", "min_date": "2016-11-15", "max_date": "2017-01-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 34.0, "maxval": 34.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploration: A Study of Count-Based Exploration for Deep Reinforcement Learning", "label": "TRPO-hash", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-03-03", "value": 33.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33.0, "maxval": 33.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-03-03", "value": 31.7, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31.7, "maxval": 31.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-03-10", "value": 31.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31.0, "maxval": 31.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-06-25", "value": 29.9, "name": "Sarsa-\u03b5", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 29.9, "maxval": 29.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03b5", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-06-25", "value": 0.0, "name": "Sarsa-\u03c6-EB", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03c6-EB", "metric": "Metric(Atari 2600 Freeway) SOLVED" }, { "date": "2017-07-21", "value": 33.9, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33.9, "maxval": 33.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Freeway) SOLVED" } ], "name": "Atari 2600 Freeway", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 29.6, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2012-07-14", "value": 180.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 180.9, "maxval": 180.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2012-07-19", "value": 216.9, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 216.9, "maxval": 216.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-02-26", "value": 328.3, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 250.0, "minval": 78.30000000000001, "maxval": 578.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-07-15", "value": 426.6, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 426.6, "maxval": 426.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-09-10", "value": 507.0, "name": "MP-EB", "url": "https://arxiv.org/abs/1507.00814", "min_date": "2015-07-03", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 507.0, "maxval": 507.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Incentivizing Exploration In Reinforcement Learning With Deep Predictive Models", "label": "MP-EB", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-09-22", "value": 797.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 797.4, "maxval": 797.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-09-22", "value": 496.1, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 496.1, "maxval": 496.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-11-20", "value": 4672.8, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4672.8, "maxval": 4672.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-11-20", "value": 2332.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2332.4, "maxval": 2332.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-11-20", "value": 1683.3, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1683.3, "maxval": 1683.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-12-08", "value": 4038.4, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 4038.4, "maxval": 4038.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2015-12-08", "value": 1448.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1448.1, "maxval": 1448.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-01-06", "value": 4380.1, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4380.1, "maxval": 4380.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-01-06", "value": 3510.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3510.0, "maxval": 3510.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-02-24", "value": 3469.6, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3469.6, "maxval": 3469.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-04-05", "value": 7413.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 7413.0, "maxval": 7413.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-04-10", "value": 197.6, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 197.6, "maxval": 197.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-04-10", "value": 190.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 190.5, "maxval": 190.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-04-10", "value": 180.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 180.1, "maxval": 180.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2016-12-13", "value": 5214.0, "name": "TRPO-hash", "url": "https://arxiv.org/abs/1611.04717", "min_date": "2016-11-15", "max_date": "2017-01-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5214.0, "maxval": 5214.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploration: A Study of Count-Based Exploration for Deep Reinforcement Learning", "label": "TRPO-hash", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2017-03-10", "value": 370.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 370.0, "maxval": 370.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2017-06-25", "value": 2770.1, "name": "Sarsa-\u03c6-EB", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2770.1, "maxval": 2770.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03c6-EB", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2017-06-25", "value": 1394.3, "name": "Sarsa-\u03b5", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1394.3, "maxval": 1394.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03b5", "metric": "Metric(Atari 2600 Frostbite) SOLVED" }, { "date": "2017-07-21", "value": 3965.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3965.0, "maxval": 3965.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Frostbite) SOLVED" } ], "name": "Atari 2600 Frostbite", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 4355, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 2368.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2368.0, "maxval": 2368.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2012-07-19", "value": 1288.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1288.0, "maxval": 1288.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-02-26", "value": 8520.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 3279.0, "minval": 5241.0, "maxval": 11799.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-07-15", "value": 4373.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4373.0, "maxval": 4373.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-09-22", "value": 8777.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8777.4, "maxval": 8777.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-09-22", "value": 8190.4, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8190.4, "maxval": 8190.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-11-20", "value": 20051.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20051.4, "maxval": 20051.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-11-20", "value": 15718.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15718.4, "maxval": 15718.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-11-20", "value": 14840.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 14840.8, "maxval": 14840.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-12-08", "value": 105148.4, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 105148.4, "maxval": 105148.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2015-12-08", "value": 15253.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15253.0, "maxval": 15253.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-01-06", "value": 34858.8, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 34858.8, "maxval": 34858.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-01-06", "value": 32487.2, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 32487.2, "maxval": 32487.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-02-24", "value": 56218.2, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56218.2, "maxval": 56218.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-04-05", "value": 104368.2, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 104368.2, "maxval": 104368.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-04-10", "value": 17106.8, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17106.8, "maxval": 17106.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-04-10", "value": 10022.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10022.8, "maxval": 10022.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2016-04-10", "value": 8442.8, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8442.8, "maxval": 8442.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2017-03-10", "value": 582.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 582.0, "maxval": 582.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" }, { "date": "2017-07-21", "value": 33641.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33641.0, "maxval": 33641.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Gopher) SOLVED" } ], "name": "Atari 2600 Gopher", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 2321, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 429.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 429.0, "maxval": 429.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2012-07-19", "value": 387.7, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 387.7, "maxval": 387.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-02-26", "value": 306.7, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 223.0, "minval": 83.69999999999999, "maxval": 529.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-07-15", "value": 538.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 538.4, "maxval": 538.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-09-22", "value": 473.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 473.0, "maxval": 473.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-09-22", "value": 298.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 298.0, "maxval": 298.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-11-20", "value": 588.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 588.0, "maxval": 588.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-11-20", "value": 412.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 412.0, "maxval": 412.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-11-20", "value": 297.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 297.0, "maxval": 297.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-12-08", "value": 200.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 200.5, "maxval": 200.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2015-12-08", "value": 167.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 167.0, "maxval": 167.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-01-06", "value": 548.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 548.5, "maxval": 548.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-01-06", "value": 269.5, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 269.5, "maxval": 269.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-02-24", "value": 483.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 483.5, "maxval": 483.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-04-05", "value": 238.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 238.0, "maxval": 238.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-04-10", "value": 320.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 320.0, "maxval": 320.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-04-10", "value": 303.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 303.5, "maxval": 303.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-04-10", "value": 269.5, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 269.5, "maxval": 269.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2016-08-22", "value": 238.68, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 238.68, "maxval": 238.68, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2017-03-03", "value": 498.3, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 498.3, "maxval": 498.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2017-03-03", "value": 238.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 238.0, "maxval": 238.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2017-03-10", "value": 805.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 805.0, "maxval": 805.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Gravitar) not solved" }, { "date": "2017-07-21", "value": 440.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 440.0, "maxval": 440.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Gravitar) not solved" } ], "name": "Atari 2600 Gravitar", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 2672, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2012-07-14", "value": 7295.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7295.0, "maxval": 7295.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2012-07-19", "value": 6459.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6459.0, "maxval": 6459.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-02-26", "value": 19950.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 158.0, "minval": 19792.0, "maxval": 20108.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-07-15", "value": 8963.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8963.4, "maxval": 8963.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-09-22", "value": 20437.8, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 20437.8, "maxval": 20437.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-09-22", "value": 14992.9, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 14992.9, "maxval": 14992.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-11-20", "value": 20818.2, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20818.2, "maxval": 20818.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-11-20", "value": 20130.2, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 20130.2, "maxval": 20130.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-11-20", "value": 15207.9, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15207.9, "maxval": 15207.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-12-08", "value": 15459.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 15459.2, "maxval": 15459.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2015-12-08", "value": 14892.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14892.5, "maxval": 14892.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-01-06", "value": 23037.7, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23037.7, "maxval": 23037.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-01-06", "value": 20889.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20889.9, "maxval": 20889.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-02-24", "value": 14225.2, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14225.2, "maxval": 14225.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-04-05", "value": 21036.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 21036.5, "maxval": 21036.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-04-10", "value": 32464.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 32464.1, "maxval": 32464.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-04-10", "value": 28889.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28889.5, "maxval": 28889.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2016-04-10", "value": 28765.8, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28765.8, "maxval": 28765.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 HERO) SOLVED" }, { "date": "2017-07-21", "value": 38874.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 38874.0, "maxval": 38874.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 HERO) SOLVED" } ], "name": "Atari 2600 HERO", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 25763, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": -3.2, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -3.2, "maxval": -3.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2012-07-19", "value": -9.5, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -9.5, "maxval": -9.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-02-26", "value": -1.6, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2.0, "minval": -3.6, "maxval": 0.3999999999999999, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-07-15", "value": -1.7, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -1.7, "maxval": -1.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-09-22", "value": -1.6, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -1.6, "maxval": -1.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-09-22", "value": -1.9, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -1.9, "maxval": -1.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-11-20", "value": 0.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.5, "maxval": 0.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-11-20", "value": -1.3, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -1.3, "maxval": -1.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-11-20", "value": -2.7, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -2.7, "maxval": -2.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-12-08", "value": 0.5, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 0.5, "maxval": 0.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2015-12-08", "value": -2.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -2.5, "maxval": -2.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-01-06", "value": 1.3, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.3, "maxval": 1.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-01-06", "value": -0.2, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.2, "maxval": -0.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-02-24", "value": -4.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -4.1, "maxval": -4.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-04-05", "value": -0.4, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -0.4, "maxval": -0.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-04-10", "value": -1.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -1.7, "maxval": -1.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-04-10", "value": -2.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -2.8, "maxval": -2.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2016-04-10", "value": -4.7, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -4.7, "maxval": -4.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2017-03-10", "value": -4.1, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -4.1, "maxval": -4.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" }, { "date": "2017-07-21", "value": -3.5, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -3.5, "maxval": -3.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Ice Hockey) SOLVED" } ], "name": "Atari 2600 Ice Hockey", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 0.9, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 354.1, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 354.1, "maxval": 354.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2012-07-19", "value": 202.8, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 202.8, "maxval": 202.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-02-26", "value": 576.7, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 175.0, "minval": 401.70000000000005, "maxval": 751.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-07-15", "value": 444.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 444.0, "maxval": 444.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-09-22", "value": 768.5, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 768.5, "maxval": 768.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-09-22", "value": 697.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 697.5, "maxval": 697.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-11-20", "value": 1358.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 1358.0, "maxval": 1358.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-11-20", "value": 1312.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1312.5, "maxval": 1312.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-11-20", "value": 835.5, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 835.5, "maxval": 835.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-12-08", "value": 585.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 585.0, "maxval": 585.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2015-12-08", "value": 573.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 573.0, "maxval": 573.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-01-06", "value": 5148.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5148.0, "maxval": 5148.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-01-06", "value": 3961.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3961.0, "maxval": 3961.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-02-24", "value": 507.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 507.5, "maxval": 507.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-04-05", "value": 812.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 812.0, "maxval": 812.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-04-10", "value": 613.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 613.0, "maxval": 613.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-04-10", "value": 541.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 541.0, "maxval": 541.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2016-04-10", "value": 351.5, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 351.5, "maxval": 351.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 James Bond) SOLVED" }, { "date": "2017-07-21", "value": 1909.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1909.0, "maxval": 1909.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 James Bond) SOLVED" } ], "name": "Atari 2600 James Bond", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 406.7, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 8.8, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.8, "maxval": 8.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2012-07-19", "value": 1622.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1622.0, "maxval": 1622.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-02-26", "value": 6740.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2959.0, "minval": 3781.0, "maxval": 9699.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-07-15", "value": 1431.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1431.0, "maxval": 1431.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-09-22", "value": 7259.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 7259.0, "maxval": 7259.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-09-22", "value": 4496.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4496.0, "maxval": 4496.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-11-20", "value": 14854.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14854.0, "maxval": 14854.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-11-20", "value": 12992.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 12992.0, "maxval": 12992.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-11-20", "value": 10334.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10334.0, "maxval": 10334.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-12-08", "value": 11204.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11204.0, "maxval": 11204.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2015-12-08", "value": 861.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 861.0, "maxval": 861.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-01-06", "value": 16200.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16200.0, "maxval": 16200.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-01-06", "value": 12185.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12185.0, "maxval": 12185.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-02-24", "value": 13150.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13150.0, "maxval": 13150.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-04-05", "value": 1792.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1792.0, "maxval": 1792.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-04-10", "value": 125.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 125.0, "maxval": 125.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-04-10", "value": 106.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 106.0, "maxval": 106.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2016-04-10", "value": 94.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.0, "maxval": 94.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2017-03-10", "value": 11200.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11200.0, "maxval": 11200.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" }, { "date": "2017-07-21", "value": 12853.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12853.0, "maxval": 12853.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Kangaroo) SOLVED" } ], "name": "Atari 2600 Kangaroo", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 3035, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 3341.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3341.0, "maxval": 3341.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2012-07-19", "value": 3372.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3372.0, "maxval": 3372.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-02-26", "value": 3805.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1033.0, "minval": 2772.0, "maxval": 4838.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-07-15", "value": 6363.1, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6363.1, "maxval": 6363.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-09-22", "value": 8422.3, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8422.3, "maxval": 8422.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-09-22", "value": 6206.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 6206.0, "maxval": 6206.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-11-20", "value": 11451.9, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11451.9, "maxval": 11451.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-11-20", "value": 8051.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8051.6, "maxval": 8051.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-11-20", "value": 7920.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 7920.5, "maxval": 7920.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-12-08", "value": 7658.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 7658.6, "maxval": 7658.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2015-12-08", "value": 6796.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6796.1, "maxval": 6796.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-01-06", "value": 9728.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9728.0, "maxval": 9728.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-01-06", "value": 6872.8, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6872.8, "maxval": 6872.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-02-24", "value": 9745.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9745.1, "maxval": 9745.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-04-05", "value": 10374.4, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 10374.4, "maxval": 10374.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-04-10", "value": 8066.6, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8066.6, "maxval": 8066.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-04-10", "value": 5911.4, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5911.4, "maxval": 5911.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2016-04-10", "value": 5560.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5560.0, "maxval": 5560.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2017-03-10", "value": 8647.2, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8647.2, "maxval": 8647.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Krull) SOLVED" }, { "date": "2017-07-21", "value": 9735.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9735.0, "maxval": 9735.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Krull) SOLVED" } ], "name": "Atari 2600 Krull", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 2395, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 29151.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 29151.0, "maxval": 29151.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2012-07-19", "value": 19544.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19544.0, "maxval": 19544.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-02-26", "value": 23270.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 5955.0, "minval": 17315.0, "maxval": 29225.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-07-15", "value": 20620.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20620.0, "maxval": 20620.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-09-22", "value": 26059.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 26059.0, "maxval": 26059.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-09-22", "value": 20882.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 20882.0, "maxval": 20882.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-11-20", "value": 34294.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 34294.0, "maxval": 34294.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-11-20", "value": 29710.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 29710.0, "maxval": 29710.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-11-20", "value": 24288.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24288.0, "maxval": 24288.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-12-08", "value": 37484.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 37484.0, "maxval": 37484.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2015-12-08", "value": 30207.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 30207.0, "maxval": 30207.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-01-06", "value": 39581.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 39581.0, "maxval": 39581.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-01-06", "value": 31676.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31676.0, "maxval": 31676.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-02-24", "value": 34393.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 34393.0, "maxval": 34393.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-04-05", "value": 48375.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 48375.0, "maxval": 48375.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-04-10", "value": 40835.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 40835.0, "maxval": 40835.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-04-10", "value": 28819.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28819.0, "maxval": 28819.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2016-04-10", "value": 3046.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3046.0, "maxval": 3046.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" }, { "date": "2017-07-21", "value": 48192.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 48192.0, "maxval": 48192.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Kung-Fu Master) SOLVED" } ], "name": "Atari 2600 Kung-Fu Master", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 22736, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 259.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 259.0, "maxval": 259.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2012-07-19", "value": 10.7, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10.7, "maxval": 10.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-02-26", "value": 0.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0.0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-07-15", "value": 84.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 84.0, "maxval": 84.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-09-10", "value": 142.0, "name": "MP-EB", "url": "https://arxiv.org/abs/1507.00814", "min_date": "2015-07-03", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 142.0, "maxval": 142.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Incentivizing Exploration In Reinforcement Learning With Deep Predictive Models", "label": "MP-EB", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-09-22", "value": 47.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 47.0, "maxval": 47.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-09-22", "value": 0.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-11-20", "value": 22.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22.0, "maxval": 22.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-11-20", "value": 0.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-11-20", "value": 0.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-12-08", "value": 42.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 42.0, "maxval": 42.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2015-12-08", "value": 24.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 24.0, "maxval": 24.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-01-06", "value": 51.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 51.0, "maxval": 51.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-01-06", "value": 0.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-02-24", "value": 0.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-04-05", "value": 0.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-04-10", "value": 67.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.0, "maxval": 67.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-04-10", "value": 53.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 53.0, "maxval": 53.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-04-10", "value": 41.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 41.0, "maxval": 41.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-08-22", "value": 3459.0, "name": "DDQN-PC", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3459.0, "maxval": 3459.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "DDQN-PC", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-08-22", "value": 273.7, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 273.7, "maxval": 273.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2016-12-13", "value": 75.0, "name": "TRPO-hash", "url": "https://arxiv.org/abs/1611.04717", "min_date": "2016-11-15", "max_date": "2017-01-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.0, "maxval": 75.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploration: A Study of Count-Based Exploration for Deep Reinforcement Learning", "label": "TRPO-hash", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-03-03", "value": 3705.5, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3705.5, "maxval": 3705.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-03-03", "value": 0.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-03-10", "value": 0.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-06-25", "value": 2745.4, "name": "Sarsa-\u03c6-EB", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2745.4, "maxval": 2745.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03c6-EB", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-06-25", "value": 399.5, "name": "Sarsa-\u03b5", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 399.5, "maxval": 399.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03b5", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" }, { "date": "2017-07-21", "value": 0.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Montezuma's Revenge) not solved" } ], "name": "Atari 2600 Montezuma's Revenge", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 4367, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2012-07-14", "value": 1227.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1227.0, "maxval": 1227.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2012-07-19", "value": 1692.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1692.0, "maxval": 1692.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-02-26", "value": 2311.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 525.0, "minval": 1786.0, "maxval": 2836.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-07-15", "value": 1263.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1263.0, "maxval": 1263.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-09-22", "value": 3085.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 3085.6, "maxval": 3085.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-09-22", "value": 1092.3, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1092.3, "maxval": 1092.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-11-20", "value": 6283.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6283.5, "maxval": 6283.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-11-20", "value": 2711.4, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 2711.4, "maxval": 2711.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-11-20", "value": 2250.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2250.6, "maxval": 2250.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-12-08", "value": 1241.3, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1241.3, "maxval": 1241.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2015-12-08", "value": 1007.8, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1007.8, "maxval": 1007.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-01-06", "value": 6518.7, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6518.7, "maxval": 6518.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-01-06", "value": 1865.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1865.9, "maxval": 1865.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-02-24", "value": 4963.8, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4963.8, "maxval": 4963.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-04-05", "value": 3327.3, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 3327.3, "maxval": 3327.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-04-10", "value": 850.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 850.7, "maxval": 850.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-04-10", "value": 653.7, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 653.7, "maxval": 653.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2016-04-10", "value": 594.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 594.4, "maxval": 594.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" }, { "date": "2017-07-21", "value": 3415.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3415.0, "maxval": 3415.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Ms. Pacman) not solved" } ], "name": "Atari 2600 Ms. Pacman", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 15693, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 2247.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2247.0, "maxval": 2247.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2012-07-19", "value": 2500.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2500.0, "maxval": 2500.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-02-26", "value": 7257.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 547.0, "minval": 6710.0, "maxval": 7804.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-07-15", "value": 9238.5, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9238.5, "maxval": 9238.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-09-22", "value": 8207.8, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8207.8, "maxval": 8207.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-09-22", "value": 6738.8, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 6738.8, "maxval": 6738.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-11-20", "value": 11971.1, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11971.1, "maxval": 11971.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-11-20", "value": 11185.1, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11185.1, "maxval": 11185.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-11-20", "value": 10616.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 10616.0, "maxval": 10616.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-12-08", "value": 13637.9, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 13637.9, "maxval": 13637.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2015-12-08", "value": 8960.3, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8960.3, "maxval": 8960.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-01-06", "value": 12270.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12270.5, "maxval": 12270.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-01-06", "value": 10497.6, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10497.6, "maxval": 10497.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-02-24", "value": 15851.2, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15851.2, "maxval": 15851.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-04-05", "value": 15572.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 15572.5, "maxval": 15572.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-04-10", "value": 12093.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12093.7, "maxval": 12093.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-04-10", "value": 10476.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10476.1, "maxval": 10476.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2016-04-10", "value": 5614.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5614.0, "maxval": 5614.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2017-03-10", "value": 4503.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4503.0, "maxval": 4503.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" }, { "date": "2017-07-21", "value": 12542.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12542.0, "maxval": 12542.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Name This Game) SOLVED" } ], "name": "Atari 2600 Name This Game", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 4076, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": -17.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -17.4, "maxval": -17.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2012-07-19", "value": -19.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -19.0, "maxval": -19.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2013-12-19", "value": 21, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21, "maxval": 21, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-02-26", "value": 18.9, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1.0, "minval": 17.9, "maxval": 19.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-07-15", "value": 16.7, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.7, "maxval": 16.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-09-22", "value": 19.5, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 19.5, "maxval": 19.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-09-22", "value": 18.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 18.0, "maxval": 18.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-11-20", "value": 21.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.0, "maxval": 21.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-11-20", "value": 20.9, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 20.9, "maxval": 20.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-11-20", "value": 18.8, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.8, "maxval": 18.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-12-08", "value": 19.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.1, "maxval": 19.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2015-12-08", "value": 18.4, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 18.4, "maxval": 18.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-01-06", "value": 20.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20.6, "maxval": 20.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-01-06", "value": 18.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.9, "maxval": 18.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-02-24", "value": 20.6, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20.6, "maxval": 20.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-04-05", "value": 20.9, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 20.9, "maxval": 20.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-04-10", "value": 11.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11.4, "maxval": 11.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-04-10", "value": 10.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10.7, "maxval": 10.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2016-04-10", "value": 5.6, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.6, "maxval": 5.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2017-03-10", "value": 21.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.0, "maxval": 21.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Pong) SOLVED" }, { "date": "2017-07-21", "value": 20.9, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20.9, "maxval": 20.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Pong) SOLVED" } ], "name": "Atari 2600 Pong", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 9.3, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 86.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 86.0, "maxval": 86.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2012-07-19", "value": 684.3, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 684.3, "maxval": 684.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-02-26", "value": 1788.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 5473.0, "minval": -3685.0, "maxval": 7261.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-07-15", "value": 2598.6, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2598.6, "maxval": 2598.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-09-22", "value": 207.9, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 207.9, "maxval": 207.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-09-22", "value": 146.7, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 146.7, "maxval": 146.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-11-20", "value": 292.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 292.6, "maxval": 292.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-11-20", "value": 129.7, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 129.7, "maxval": 129.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-11-20", "value": 103.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 103.0, "maxval": 103.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-12-08", "value": 1277.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1277.6, "maxval": 1277.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2015-12-08", "value": -575.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -575.5, "maxval": -575.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-01-06", "value": 670.7, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 670.7, "maxval": 670.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-01-06", "value": 200.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 200.0, "maxval": 200.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-02-24", "value": 286.7, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 286.7, "maxval": 286.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-04-05", "value": 206.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 206.0, "maxval": 206.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-04-10", "value": 421.1, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 421.1, "maxval": 421.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-04-10", "value": 206.9, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 206.9, "maxval": 206.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-04-10", "value": 194.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 194.4, "maxval": 194.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2016-08-22", "value": 99.32, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 99.32, "maxval": 99.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2017-03-03", "value": 8358.7, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8358.7, "maxval": 8358.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2017-03-03", "value": 206.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 206.0, "maxval": 206.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2017-03-10", "value": 100.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 100.0, "maxval": 100.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Private Eye) not solved" }, { "date": "2017-07-21", "value": 15095.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15095.0, "maxval": 15095.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Private Eye) not solved" } ], "name": "Atari 2600 Private Eye", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 69571, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2012-07-14", "value": 960.3, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 960.3, "maxval": 960.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2012-07-19", "value": 613.5, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 613.5, "maxval": 613.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2013-12-19", "value": 4500, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4500, "maxval": 4500, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-02-26", "value": 10596.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 3294.0, "minval": 7302.0, "maxval": 13890.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-07-15", "value": 7089.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7089.8, "maxval": 7089.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-09-10", "value": 15805.0, "name": "MP-EB", "url": "https://arxiv.org/abs/1507.00814", "min_date": "2015-07-03", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15805.0, "maxval": 15805.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Incentivizing Exploration In Reinforcement Learning With Deep Predictive Models", "label": "MP-EB", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-09-22", "value": 13117.3, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 13117.3, "maxval": 13117.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-09-22", "value": 9271.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 9271.5, "maxval": 9271.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-11-20", "value": 19220.3, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19220.3, "maxval": 19220.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-11-20", "value": 15088.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 15088.5, "maxval": 15088.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-11-20", "value": 14175.8, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14175.8, "maxval": 14175.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-12-08", "value": 14063.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 14063.0, "maxval": 14063.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2015-12-08", "value": 11020.8, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11020.8, "maxval": 11020.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-01-06", "value": 16256.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16256.5, "maxval": 16256.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-01-06", "value": 9944.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9944.0, "maxval": 9944.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-02-24", "value": 5236.8, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5236.8, "maxval": 5236.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-04-05", "value": 18760.3, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 18760.3, "maxval": 18760.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-04-10", "value": 21307.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21307.5, "maxval": 21307.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-04-10", "value": 15148.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15148.8, "maxval": 15148.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2016-04-10", "value": 13752.3, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13752.3, "maxval": 13752.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2017-03-10", "value": 147.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 147.5, "maxval": 147.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2017-06-25", "value": 4111.8, "name": "Sarsa-\u03c6-EB", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4111.8, "maxval": 4111.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03c6-EB", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2017-06-25", "value": 3895.3, "name": "Sarsa-\u03b5", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3895.3, "maxval": 3895.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03b5", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" }, { "date": "2017-07-21", "value": 23784.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23784.0, "maxval": 23784.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Q*Bert) SOLVED" } ], "name": "Atari 2600 Q*Bert", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 13455, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 2650.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2650.0, "maxval": 2650.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2012-07-19", "value": 1904.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1904.0, "maxval": 1904.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-02-26", "value": 8316.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1049.0, "minval": 7267.0, "maxval": 9365.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-07-15", "value": 5310.3, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5310.3, "maxval": 5310.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-09-22", "value": 7377.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 7377.6, "maxval": 7377.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-09-22", "value": 4748.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4748.5, "maxval": 4748.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-11-20", "value": 21162.6, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21162.6, "maxval": 21162.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-11-20", "value": 16569.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16569.4, "maxval": 16569.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-11-20", "value": 14884.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 14884.5, "maxval": 14884.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-12-08", "value": 16496.8, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 16496.8, "maxval": 16496.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2015-12-08", "value": 10838.4, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10838.4, "maxval": 10838.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-01-06", "value": 14522.3, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14522.3, "maxval": 14522.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-01-06", "value": 11807.2, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11807.2, "maxval": 11807.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-02-24", "value": 12530.8, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12530.8, "maxval": 12530.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-04-05", "value": 20607.6, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 20607.6, "maxval": 20607.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-04-10", "value": 12201.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12201.8, "maxval": 12201.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-04-10", "value": 10001.2, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10001.2, "maxval": 10001.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2016-04-10", "value": 6591.9, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6591.9, "maxval": 6591.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2017-03-10", "value": 5009.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5009.0, "maxval": 5009.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" }, { "date": "2017-07-21", "value": 17322.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17322.0, "maxval": 17322.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 River Raid) SOLVED" } ], "name": "Atari 2600 River Raid", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 13513, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 89.1, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.1, "maxval": 89.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2012-07-19", "value": 67.7, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.7, "maxval": 67.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-02-26", "value": 18257.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 4268.0, "minval": 13989.0, "maxval": 22525.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-07-15", "value": 43079.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 43079.8, "maxval": 43079.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-09-22", "value": 39544.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 39544.0, "maxval": 39544.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-09-22", "value": 35215.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 35215.0, "maxval": 35215.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-11-20", "value": 69524.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69524.0, "maxval": 69524.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-11-20", "value": 58549.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 58549.0, "maxval": 58549.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-11-20", "value": 44127.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 44127.0, "maxval": 44127.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-12-08", "value": 54630.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 54630.0, "maxval": 54630.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2015-12-08", "value": 43156.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 43156.0, "maxval": 43156.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-01-06", "value": 57608.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 57608.0, "maxval": 57608.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-01-06", "value": 52264.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 52264.0, "maxval": 52264.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-02-24", "value": 47770.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 47770.0, "maxval": 47770.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-04-05", "value": 62151.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 62151.0, "maxval": 62151.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-04-10", "value": 73949.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73949.0, "maxval": 73949.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-04-10", "value": 34216.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 34216.0, "maxval": 34216.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2016-04-10", "value": 31769.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31769.0, "maxval": 31769.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2017-03-10", "value": 16590.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16590.0, "maxval": 16590.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" }, { "date": "2017-07-21", "value": 55839.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 55839.0, "maxval": 55839.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Road Runner) SOLVED" } ], "name": "Atari 2600 Road Runner", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 7845, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 12.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.4, "maxval": 12.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2012-07-19", "value": 28.7, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.7, "maxval": 28.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-02-26", "value": 51.6, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 4.0, "minval": 47.6, "maxval": 55.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-07-15", "value": 61.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61.8, "maxval": 61.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-09-22", "value": 63.9, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 63.9, "maxval": 63.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-09-22", "value": 58.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 58.7, "maxval": 58.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-11-20", "value": 65.3, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65.3, "maxval": 65.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-11-20", "value": 65.1, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 65.1, "maxval": 65.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-11-20", "value": 62.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62.0, "maxval": 62.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-12-08", "value": 59.1, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 59.1, "maxval": 59.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2015-12-08", "value": 24.7, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 24.7, "maxval": 24.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-01-06", "value": 62.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62.6, "maxval": 62.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-01-06", "value": 56.2, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56.2, "maxval": 56.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-02-24", "value": 64.3, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 64.3, "maxval": 64.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-04-05", "value": 27.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 27.5, "maxval": 27.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-04-10", "value": 32.8, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 32.8, "maxval": 32.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-04-10", "value": 2.6, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2.6, "maxval": 2.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2016-04-10", "value": 2.3, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2.3, "maxval": 2.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2017-03-10", "value": 11.9, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11.9, "maxval": 11.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" }, { "date": "2017-07-21", "value": 52.3, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 52.3, "maxval": 52.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Robotank) SOLVED" } ], "name": "Atari 2600 Robotank", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 11.9, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 675.5, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 675.5, "maxval": 675.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2012-07-19", "value": 664.8, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 664.8, "maxval": 664.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2013-12-19", "value": 1740, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1740, "maxval": 1740, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-02-26", "value": 5286.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1310.0, "minval": 3976.0, "maxval": 6596.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-07-15", "value": 10145.9, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10145.9, "maxval": 10145.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-09-22", "value": 5860.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 5860.6, "maxval": 5860.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-09-22", "value": 4216.7, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4216.7, "maxval": 4216.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-11-20", "value": 50254.2, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 50254.2, "maxval": 50254.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-11-20", "value": 37361.6, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 37361.6, "maxval": 37361.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-11-20", "value": 16452.7, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 16452.7, "maxval": 16452.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-12-08", "value": 14498.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14498.0, "maxval": 14498.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2015-12-08", "value": 1431.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1431.2, "maxval": 1431.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-01-06", "value": 26357.8, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26357.8, "maxval": 26357.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-01-06", "value": 25463.7, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 25463.7, "maxval": 25463.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-02-24", "value": 10932.3, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10932.3, "maxval": 10932.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-04-05", "value": 931.6, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 931.6, "maxval": 931.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-04-10", "value": 2355.4, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2355.4, "maxval": 2355.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-04-10", "value": 2300.2, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2300.2, "maxval": 2300.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2016-04-10", "value": 1326.1, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1326.1, "maxval": 1326.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2017-03-10", "value": 1390.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1390.0, "maxval": 1390.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" }, { "date": "2017-07-21", "value": 266434.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 266434.0, "maxval": 266434.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Seaquest) SOLVED" } ], "name": "Atari 2600 Seaquest", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 20182, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 267.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 267.9, "maxval": 267.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2012-07-19", "value": 250.1, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 250.1, "maxval": 250.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2013-12-19", "value": 1075, "name": "DQN best", "url": "https://arxiv.org/abs/1312.5602", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1075, "maxval": 1075, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Playing Atari with Deep Reinforcement Learning", "label": "DQN best", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-02-26", "value": 1976.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 893.0, "minval": 1083.0, "maxval": 2869.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-07-15", "value": 1183.3, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1183.3, "maxval": 1183.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-09-22", "value": 1692.3, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1692.3, "maxval": 1692.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-09-22", "value": 1293.8, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1293.8, "maxval": 1293.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-11-20", "value": 6427.3, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6427.3, "maxval": 6427.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-11-20", "value": 5993.1, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5993.1, "maxval": 5993.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-11-20", "value": 2525.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 2525.5, "maxval": 2525.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-12-08", "value": 8978.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 8978.0, "maxval": 8978.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2015-12-08", "value": 2628.7, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2628.7, "maxval": 2628.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-01-06", "value": 3912.1, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3912.1, "maxval": 3912.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-01-06", "value": 2865.8, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2865.8, "maxval": 2865.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-02-24", "value": 2589.7, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2589.7, "maxval": 2589.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-04-05", "value": 15311.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 15311.5, "maxval": 15311.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-04-10", "value": 23846.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23846.0, "maxval": 23846.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-04-10", "value": 15730.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15730.5, "maxval": 15730.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2016-04-10", "value": 2214.7, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2214.7, "maxval": 2214.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2017-03-10", "value": 678.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 678.5, "maxval": 678.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" }, { "date": "2017-07-21", "value": 5747.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5747.0, "maxval": 5747.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Space Invaders) SOLVED" } ], "name": "Atari 2600 Space Invaders", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 1652, "target_source": "https://pdfs.semanticscholar.org/340f/48901f72278f6bf78a04ee5b01df208cc508.pdf", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 9.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.4, "maxval": 9.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2012-07-19", "value": 1070.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1070.0, "maxval": 1070.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-02-26", "value": 57997.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 3152.0, "minval": 54845.0, "maxval": 61149.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-07-15", "value": 14919.2, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14919.2, "maxval": 14919.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-09-22", "value": 54282.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 54282.0, "maxval": 54282.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-09-22", "value": 52970.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 52970.0, "maxval": 52970.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-11-20", "value": 90804.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 90804.0, "maxval": 90804.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-11-20", "value": 89238.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89238.0, "maxval": 89238.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-11-20", "value": 60142.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 60142.0, "maxval": 60142.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-12-08", "value": 127073.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 127073.0, "maxval": 127073.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2015-12-08", "value": 58365.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 58365.0, "maxval": 58365.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-01-06", "value": 63302.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63302.0, "maxval": 63302.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-01-06", "value": 61582.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61582.0, "maxval": 61582.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-02-24", "value": 589.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 589.0, "maxval": 589.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-04-05", "value": 125117.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 125117.0, "maxval": 125117.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-04-10", "value": 164766.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 164766.0, "maxval": 164766.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-04-10", "value": 138218.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 138218.0, "maxval": 138218.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2016-04-10", "value": 64393.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 64393.0, "maxval": 64393.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2017-03-10", "value": 1470.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1470.0, "maxval": 1470.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" }, { "date": "2017-07-21", "value": 49095.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 49095.0, "maxval": 49095.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Star Gunner) SOLVED" } ], "name": "Atari 2600 Star Gunner", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 10250, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 0.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2012-07-19", "value": -0.1, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.1, "maxval": -0.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-02-26", "value": -2.5, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1.0, "minval": -3.5, "maxval": -1.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-07-15", "value": -0.7, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -0.7, "maxval": -0.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-09-22", "value": 12.2, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 12.2, "maxval": 12.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-09-22", "value": 11.1, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 11.1, "maxval": 11.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-11-20", "value": 5.1, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.1, "maxval": 5.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-11-20", "value": 4.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.4, "maxval": 4.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-11-20", "value": -22.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -22.8, "maxval": -22.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-12-08", "value": -7.8, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -7.8, "maxval": -7.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2015-12-08", "value": -13.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -13.2, "maxval": -13.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-01-06", "value": 0.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-01-06", "value": -5.3, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -5.3, "maxval": -5.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-02-24", "value": 12.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.1, "maxval": 12.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-04-05", "value": 0.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-04-10", "value": -6.3, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -6.3, "maxval": -6.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-04-10", "value": -6.4, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -6.4, "maxval": -6.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2016-04-10", "value": -10.2, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -10.2, "maxval": -10.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2017-03-10", "value": -4.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -4.5, "maxval": -4.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" }, { "date": "2017-07-21", "value": 23.1, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23.1, "maxval": 23.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Tennis) SOLVED" } ], "name": "Atari 2600 Tennis", "solved": true, "url": null, "notes": "", "scale": "Score", "target": -8.9, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 24.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24.9, "maxval": 24.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2012-07-19", "value": 3741.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3741.0, "maxval": 3741.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-02-26", "value": 5947.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1600.0, "minval": 4347.0, "maxval": 7547.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-07-15", "value": 8267.8, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8267.8, "maxval": 8267.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-09-22", "value": 4870.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4870.0, "maxval": 4870.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-09-22", "value": 4786.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4786.0, "maxval": 4786.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-11-20", "value": 11666.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11666.0, "maxval": 11666.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-11-20", "value": 8339.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 8339.0, "maxval": 8339.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-11-20", "value": 6601.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6601.0, "maxval": 6601.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-12-08", "value": 6608.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6608.0, "maxval": 6608.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2015-12-08", "value": 4871.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 4871.0, "maxval": 4871.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-01-06", "value": 9197.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9197.0, "maxval": 9197.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-01-06", "value": 5963.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5963.0, "maxval": 5963.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-02-24", "value": 4870.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4870.0, "maxval": 4870.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-04-05", "value": 7553.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 7553.0, "maxval": 7553.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-04-10", "value": 27202.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 27202.0, "maxval": 27202.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-04-10", "value": 12679.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12679.0, "maxval": 12679.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2016-04-10", "value": 5825.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5825.0, "maxval": 5825.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2017-03-10", "value": 4970.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4970.0, "maxval": 4970.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" }, { "date": "2017-07-21", "value": 8329.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8329.0, "maxval": 8329.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Time Pilot) SOLVED" } ], "name": "Atari 2600 Time Pilot", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 5925, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 98.2, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 98.2, "maxval": 98.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2012-07-19", "value": 114.3, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 114.3, "maxval": 114.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-02-26", "value": 186.7, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 41.0, "minval": 145.7, "maxval": 227.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-07-15", "value": 118.5, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 118.5, "maxval": 118.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-09-22", "value": 68.1, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 68.1, "maxval": 68.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-09-22", "value": 45.6, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 45.6, "maxval": 45.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-11-20", "value": 218.4, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 218.4, "maxval": 218.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-11-20", "value": 211.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 211.4, "maxval": 211.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-11-20", "value": 48.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 48.0, "maxval": 48.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-12-08", "value": 108.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 108.6, "maxval": 108.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2015-12-08", "value": 92.2, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 92.2, "maxval": 92.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-01-06", "value": 204.6, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 204.6, "maxval": 204.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-01-06", "value": 56.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56.9, "maxval": 56.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-02-24", "value": 183.9, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 183.9, "maxval": 183.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-04-05", "value": 245.9, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 245.9, "maxval": 245.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-04-10", "value": 156.3, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 156.3, "maxval": 156.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-04-10", "value": 144.2, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 144.2, "maxval": 144.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2016-04-10", "value": 26.1, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.1, "maxval": 26.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2017-03-10", "value": 130.3, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 130.3, "maxval": 130.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" }, { "date": "2017-07-21", "value": 280.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 280.0, "maxval": 280.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Tutankham) SOLVED" } ], "name": "Atari 2600 Tutankham", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 167.6, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 2449.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2449.0, "maxval": 2449.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2012-07-19", "value": 3533.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3533.0, "maxval": 3533.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-02-26", "value": 8456.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 3162.0, "minval": 5294.0, "maxval": 11618.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-07-15", "value": 8747.7, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8747.7, "maxval": 8747.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-09-22", "value": 9989.9, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 9989.9, "maxval": 9989.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-09-22", "value": 8038.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8038.5, "maxval": 8038.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-11-20", "value": 44939.6, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 44939.6, "maxval": 44939.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-11-20", "value": 24759.2, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24759.2, "maxval": 24759.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-11-20", "value": 22972.2, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 22972.2, "maxval": 22972.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-12-08", "value": 22681.3, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 22681.3, "maxval": 22681.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2015-12-08", "value": 19086.9, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19086.9, "maxval": 19086.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-01-06", "value": 16154.1, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16154.1, "maxval": 16154.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-01-06", "value": 12157.4, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12157.4, "maxval": 12157.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-02-24", "value": 22474.4, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22474.4, "maxval": 22474.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-04-05", "value": 33879.1, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 33879.1, "maxval": 33879.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-04-10", "value": 105728.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 105728.7, "maxval": 105728.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-04-10", "value": 74705.7, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74705.7, "maxval": 74705.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2016-04-10", "value": 54525.4, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 54525.4, "maxval": 54525.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2017-03-10", "value": 67974.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67974.0, "maxval": 67974.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" }, { "date": "2017-07-21", "value": 15612.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15612.0, "maxval": 15612.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Up and Down) SOLVED" } ], "name": "Atari 2600 Up and Down", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 9082, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 0.6, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.6, "maxval": 0.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2012-07-19", "value": 66.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.0, "maxval": 66.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-02-26", "value": 380.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 238.0, "minval": 142.0, "maxval": 618.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-07-15", "value": 523.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 523.4, "maxval": 523.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-09-10", "value": 0.0, "name": "MP-EB", "url": "https://arxiv.org/abs/1507.00814", "min_date": "2015-07-03", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Incentivizing Exploration In Reinforcement Learning With Deep Predictive Models", "label": "MP-EB", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-09-22", "value": 163.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 163.0, "maxval": 163.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-09-22", "value": 136.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 136.0, "maxval": 136.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-11-20", "value": 497.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 497.0, "maxval": 497.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-11-20", "value": 200.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 200.0, "maxval": 200.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-11-20", "value": 98.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 98.0, "maxval": 98.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-12-08", "value": 29.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 29.0, "maxval": 29.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2015-12-08", "value": 21.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.0, "maxval": 21.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-01-06", "value": 94.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 94.0, "maxval": 94.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-01-06", "value": 54.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 54.0, "maxval": 54.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-02-24", "value": 1172.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1172.0, "maxval": 1172.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-04-05", "value": 48.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 48.0, "maxval": 48.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-04-10", "value": 25.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 25.0, "maxval": 25.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-04-10", "value": 23.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23.0, "maxval": 23.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-04-10", "value": 19.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.0, "maxval": 19.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-08-22", "value": 0.0, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2016-12-13", "value": 445.0, "name": "TRPO-hash", "url": "https://arxiv.org/abs/1611.04717", "min_date": "2016-11-15", "max_date": "2017-01-11", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 445.0, "maxval": 445.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploration: A Study of Count-Based Exploration for Deep Reinforcement Learning", "label": "TRPO-hash", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-03-03", "value": 82.2, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 82.2, "maxval": 82.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-03-03", "value": 48.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 48.0, "maxval": 48.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-03-10", "value": 760.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 760.0, "maxval": 760.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-06-25", "value": 1169.2, "name": "Sarsa-\u03c6-EB", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1169.2, "maxval": 1169.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03c6-EB", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-06-25", "value": 0.0, "name": "Sarsa-\u03b5", "url": "https://arxiv.org/abs/1706.08090", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration in Feature Space for Reinforcement Learning", "label": "Sarsa-\u03b5", "metric": "Metric(Atari 2600 Venture) SOLVED" }, { "date": "2017-07-21", "value": 1520.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1520.0, "maxval": 1520.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Venture) SOLVED" } ], "name": "Atari 2600 Venture", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 1188, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2012-07-14", "value": 19761.0, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19761.0, "maxval": 19761.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2012-07-19", "value": 16871.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16871.0, "maxval": 16871.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-02-26", "value": 42684.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 16287.0, "minval": 26397.0, "maxval": 58971.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-07-15", "value": 112093.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 112093.4, "maxval": 112093.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-09-22", "value": 196760.4, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 196760.4, "maxval": 196760.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-09-22", "value": 154414.1, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 154414.1, "maxval": 154414.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-11-20", "value": 309941.9, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 309941.9, "maxval": 309941.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-11-20", "value": 110976.2, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 110976.2, "maxval": 110976.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-11-20", "value": 98209.5, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 98209.5, "maxval": 98209.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-12-08", "value": 447408.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 447408.6, "maxval": 447408.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2015-12-08", "value": 367823.7, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 367823.7, "maxval": 367823.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-01-06", "value": 295972.8, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 295972.8, "maxval": 295972.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-01-06", "value": 282007.3, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 282007.3, "maxval": 282007.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-02-24", "value": 56287.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56287.0, "maxval": 56287.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-04-05", "value": 479197.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 479197.0, "maxval": 479197.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-04-10", "value": 470310.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 470310.5, "maxval": 470310.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-04-10", "value": 331628.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 331628.1, "maxval": 331628.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2016-04-10", "value": 185852.6, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 185852.6, "maxval": 185852.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2017-03-10", "value": 22834.8, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22834.8, "maxval": 22834.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" }, { "date": "2017-07-21", "value": 949604.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 949604.0, "maxval": 949604.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Video Pinball) SOLVED" } ], "name": "Atari 2600 Video Pinball", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 17298, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 36.9, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36.9, "maxval": 36.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2012-07-19", "value": 1981.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1981.0, "maxval": 1981.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-02-26", "value": 3393.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 2019.0, "minval": 1374.0, "maxval": 5412.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-07-15", "value": 10431.0, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10431.0, "maxval": 10431.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-09-22", "value": 2704.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 2704.0, "maxval": 2704.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-09-22", "value": 1609.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1609.0, "maxval": 1609.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-11-20", "value": 7855.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7855.0, "maxval": 7855.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-11-20", "value": 7492.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 7492.0, "maxval": 7492.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-11-20", "value": 7054.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7054.0, "maxval": 7054.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-12-08", "value": 10471.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 10471.0, "maxval": 10471.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2015-12-08", "value": 6201.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6201.0, "maxval": 6201.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-01-06", "value": 5727.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5727.0, "maxval": 5727.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-01-06", "value": 4802.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4802.0, "maxval": 4802.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-02-24", "value": 483.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 483.0, "maxval": 483.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-04-05", "value": 12352.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 12352.0, "maxval": 12352.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-04-10", "value": 18082.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18082.0, "maxval": 18082.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-04-10", "value": 17244.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17244.0, "maxval": 17244.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2016-04-10", "value": 5278.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5278.0, "maxval": 5278.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2017-03-10", "value": 3480.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3480.0, "maxval": 3480.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" }, { "date": "2017-07-21", "value": 9300.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9300.0, "maxval": 9300.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Wizard of Wor) SOLVED" } ], "name": "Atari 2600 Wizard of Wor", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 4757, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2012-07-14", "value": 21.4, "name": "SARSA", "url": "https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5162", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.4, "maxval": 21.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Investigating Contingency Awareness Using Atari 2600 Games", "label": "SARSA", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2012-07-19", "value": 3365.0, "name": "Best linear", "url": "https://arxiv.org/abs/1207.4708v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3365.0, "maxval": 3365.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Arcade Learning Environment: An Evaluation Platform for General Agents", "label": "Best linear", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-02-26", "value": 4977.0, "name": "Nature DQN", "url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 1235.0, "minval": 3742.0, "maxval": 6212.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Human-level control through deep reinforcement learning", "label": "Nature DQN", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-07-15", "value": 6159.4, "name": "Gorila", "url": "https://arxiv.org/abs/1507.04296", "min_date": "2015-07-15", "max_date": "2015-07-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6159.4, "maxval": 6159.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Massively Parallel Methods for Deep Reinforcement Learning", "label": "Gorila", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-09-22", "value": 5363.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 5363.0, "maxval": 5363.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-09-22", "value": 4412.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4412.0, "maxval": 4412.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-11-20", "value": 12944.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12944.0, "maxval": 12944.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-11-20", "value": 10164.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10164.0, "maxval": 10164.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-11-20", "value": 10163.0, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 10163.0, "maxval": 10163.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-12-08", "value": 11320.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 11320.0, "maxval": 11320.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2015-12-08", "value": 8593.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8593.0, "maxval": 8593.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-01-06", "value": 10469.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10469.0, "maxval": 10469.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-01-06", "value": 9474.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9474.0, "maxval": 9474.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-02-24", "value": 14402.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 14402.0, "maxval": 14402.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-04-05", "value": 13886.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 13886.0, "maxval": 13886.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-04-10", "value": 24622.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24622.0, "maxval": 24622.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-04-10", "value": 23519.0, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23519.0, "maxval": 23519.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2016-04-10", "value": 2659.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2659.0, "maxval": 2659.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2017-03-10", "value": 6380.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6380.0, "maxval": 6380.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" }, { "date": "2017-07-21", "value": 10513.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10513.0, "maxval": 10513.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Zaxxon) SOLVED" } ], "name": "Atari 2600 Zaxxon", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 9173, "target_source": "https://www.semanticscholar.org/paper/Human-level-control-through-deep-reinforcement-Mnih-Kavukcuoglu/340f48901f72278f6bf78a04ee5b01df208cc508", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2436" }, { "measures": [ { "date": "2015-09-22", "value": 8485.2, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 8485.2, "maxval": 8485.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-09-22", "value": 7484.8, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 7484.8, "maxval": 7484.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-11-20", "value": 23092.2, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23092.2, "maxval": 23092.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-11-20", "value": 20410.5, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20410.5, "maxval": 20410.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-11-20", "value": 12252.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 12252.5, "maxval": 12252.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-12-08", "value": 63597.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 63597.0, "maxval": 63597.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2015-12-08", "value": 12366.5, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12366.5, "maxval": 12366.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-01-06", "value": 18992.7, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18992.7, "maxval": 18992.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-01-06", "value": 16903.6, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16903.6, "maxval": 16903.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-02-24", "value": 6202.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6202.5, "maxval": 6202.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-04-05", "value": 70324.3, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 70324.3, "maxval": 70324.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-04-10", "value": 74786.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74786.7, "maxval": 74786.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-04-10", "value": 52894.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 52894.1, "maxval": 52894.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2016-04-10", "value": 28181.8, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28181.8, "maxval": 28181.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2017-03-10", "value": 4041.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4041.0, "maxval": 4041.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" }, { "date": "2017-07-21", "value": 17490.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17490.0, "maxval": 17490.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Phoenix) SOLVED" } ], "name": "Atari 2600 Phoenix", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 2665.5, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" }, { "measures": [ { "date": "2015-09-22", "value": -113.2, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -113.2, "maxval": -113.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-09-22", "value": -286.1, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -286.1, "maxval": -286.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-11-20", "value": 0.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-11-20", "value": -29.9, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -29.9, "maxval": -29.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-11-20", "value": -46.9, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -46.9, "maxval": -46.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-12-08", "value": -186.7, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -186.7, "maxval": -186.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2015-12-08", "value": -243.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -243.6, "maxval": -243.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-01-06", "value": -356.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -356.5, "maxval": -356.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-01-06", "value": -427.0, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -427.0, "maxval": -427.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-02-24", "value": -2.6, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -2.6, "maxval": -2.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-04-05", "value": 0.0, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-04-10", "value": -78.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -78.5, "maxval": -78.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-04-10", "value": -123.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -123.0, "maxval": -123.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-04-10", "value": -135.7, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -135.7, "maxval": -135.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2016-08-22", "value": -259.09, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -259.09, "maxval": -259.09, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2017-03-03", "value": 0.0, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2017-03-03", "value": 0.0, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2017-03-10", "value": 0.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" }, { "date": "2017-07-21", "value": 0.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 0.0, "maxval": 0.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Pitfall!) not solved" } ], "name": "Atari 2600 Pitfall!", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 22736.3, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2015-09-22", "value": -12142.1, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -12142.1, "maxval": -12142.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-09-22", "value": -13062.3, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -13062.3, "maxval": -13062.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-11-20", "value": -8857.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -8857.4, "maxval": -8857.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-11-20", "value": -9021.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -9021.8, "maxval": -9021.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-11-20", "value": -11928.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -11928.0, "maxval": -11928.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-12-08", "value": -11490.4, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -11490.4, "maxval": -11490.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2015-12-08", "value": -18955.8, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -18955.8, "maxval": -18955.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-01-06", "value": -9996.9, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -9996.9, "maxval": -9996.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-01-06", "value": -10169.1, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -10169.1, "maxval": -10169.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-02-24", "value": -13585.1, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -13585.1, "maxval": -13585.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-04-05", "value": -19949.9, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -19949.9, "maxval": -19949.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-04-10", "value": -10911.1, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -10911.1, "maxval": -10911.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-04-10", "value": -13700.0, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -13700.0, "maxval": -13700.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2016-04-10", "value": -14863.8, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -14863.8, "maxval": -14863.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2017-03-10", "value": -15442.5, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -15442.5, "maxval": -15442.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Skiing) not solved" }, { "date": "2017-07-21", "value": -13901.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -13901.0, "maxval": -13901.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Skiing) not solved" } ], "name": "Atari 2600 Skiing", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 13455.0, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" }, { "measures": [ { "date": "2015-09-22", "value": 3482.8, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 3482.8, "maxval": 3482.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-09-22", "value": 1295.4, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 1295.4, "maxval": 1295.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-11-20", "value": 3067.8, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 3067.8, "maxval": 3067.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-11-20", "value": 2250.8, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2250.8, "maxval": 2250.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-11-20", "value": 1768.4, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1768.4, "maxval": 1768.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-12-08", "value": 810.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 810.0, "maxval": 810.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2015-12-08", "value": 280.6, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 280.6, "maxval": 280.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-01-06", "value": 4309.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4309.0, "maxval": 4309.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-01-06", "value": 2272.8, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2272.8, "maxval": 2272.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-02-24", "value": 4544.8, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4544.8, "maxval": 4544.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-04-05", "value": 133.4, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 133.4, "maxval": 133.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-04-10", "value": 1956.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1956.0, "maxval": 1956.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-04-10", "value": 1936.4, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1936.4, "maxval": 1936.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-04-10", "value": 1884.8, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1884.8, "maxval": 1884.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2016-08-22", "value": 2270.15, "name": "A3C-CTS", "url": "https://arxiv.org/abs/1606.01868", "min_date": "2016-06-06", "max_date": "2016-11-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2270.15, "maxval": 2270.15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Unifying Count-Based Exploration and Intrinsic Motivation", "label": "A3C-CTS", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2017-03-03", "value": 2863.6, "name": "DQN-PixelCNN", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2863.6, "maxval": 2863.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-PixelCNN", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2017-03-03", "value": 133.4, "name": "DQN-CTS", "url": "https://arxiv.org/abs/1703.01310v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 133.4, "maxval": 133.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Count-Based Exploration with Neural Density Models", "label": "DQN-CTS", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2017-03-10", "value": 2090.0, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2090.0, "maxval": 2090.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Solaris) not solved" }, { "date": "2017-07-21", "value": 8342.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8342.0, "maxval": 8342.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Solaris) not solved" } ], "name": "Atari 2600 Solaris", "solved": false, "url": null, "notes": "", "scale": "Score", "target": 17118.0, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2455" }, { "measures": [ { "date": "2015-09-22", "value": 18098.9, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 18098.9, "maxval": 18098.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-09-22", "value": 4577.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 4577.5, "maxval": 4577.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-11-20", "value": 49622.1, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 49622.1, "maxval": 49622.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-11-20", "value": 25976.5, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 25976.5, "maxval": 25976.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-11-20", "value": 11712.6, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 11712.6, "maxval": 11712.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-12-08", "value": 58145.9, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 58145.9, "maxval": 58145.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2015-12-08", "value": 6270.6, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6270.6, "maxval": 6270.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-01-06", "value": 11357.0, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11357.0, "maxval": 11357.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-01-06", "value": 4687.4, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4687.4, "maxval": 4687.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-02-24", "value": 21409.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21409.5, "maxval": 21409.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-04-05", "value": 69618.1, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 69618.1, "maxval": 69618.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-04-10", "value": 7270.8, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7270.8, "maxval": 7270.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-04-10", "value": 7157.5, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 7157.5, "maxval": 7157.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2016-04-10", "value": 5615.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5615.5, "maxval": 5615.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2017-03-10", "value": 16401.7, "name": "ES FF (1 hour) noop", "url": "https://arxiv.org/abs/1703.03864v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16401.7, "maxval": 16401.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Evolution Strategies as a Scalable Alternative to Reinforcement Learning", "label": "ES FF (1 hour) noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" }, { "date": "2017-07-21", "value": 35050.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 35050.0, "maxval": 35050.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Yars Revenge) SOLVED" } ], "name": "Atari 2600 Yars Revenge", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 5650.0, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" }, { "measures": [ { "date": "2015-09-22", "value": 23633.0, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 23633.0, "maxval": 23633.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-09-22", "value": 15917.5, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": 15917.5, "maxval": 15917.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-11-20", "value": 42214.0, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 42214.0, "maxval": 42214.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-11-20", "value": 35338.5, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": 35338.5, "maxval": 35338.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-11-20", "value": 33996.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 33996.0, "maxval": 33996.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-12-08", "value": 34415.0, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 34415.0, "maxval": 34415.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2015-12-08", "value": 27510.0, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 27510.0, "maxval": 27510.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-01-06", "value": 31286.5, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 31286.5, "maxval": 31286.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-01-06", "value": 23666.5, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23666.5, "maxval": 23666.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-02-24", "value": 11099.0, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11099.0, "maxval": 11099.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-04-05", "value": 41324.5, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 41324.5, "maxval": 41324.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-04-10", "value": 233021.5, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 233021.5, "maxval": 233021.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-04-10", "value": 56533.0, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 56533.0, "maxval": 56533.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2016-04-10", "value": 36242.5, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36242.5, "maxval": 36242.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Defender) SOLVED" }, { "date": "2017-07-21", "value": 47092.0, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 47092.0, "maxval": 47092.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Defender) SOLVED" } ], "name": "Atari 2600 Defender", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 18688.9, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" }, { "measures": [ { "date": "2015-09-22", "value": -5.6, "name": "DQN noop", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -5.6, "maxval": -5.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-09-22", "value": -6.0, "name": "DQN hs", "url": "https://arxiv.org/abs/1509.06461v1", "min_date": "2015-02-26", "max_date": null, "algorithm_src_url": "https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf", "src_name": "Human-level control through deep reinforcement learning", "uncertainty": 0, "minval": -6.0, "maxval": -6.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DQN hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-11-20", "value": 4.4, "name": "Duel noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.4, "maxval": 4.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-11-20", "value": 4.0, "name": "Duel hs", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.0, "maxval": 4.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Duel hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-11-20", "value": -2.9, "name": "DDQN (tuned) noop", "url": "https://arxiv.org/abs/1511.06581v1", "min_date": "2015-12-08", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1509.06461v3", "src_name": "Deep Reinforcement Learning with Double Q-learning", "uncertainty": 0, "minval": -2.9, "maxval": -2.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "DDQN (tuned) noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-12-08", "value": 1.9, "name": "DDQN (tuned) hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.9, "maxval": 1.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "DDQN (tuned) hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2015-12-08", "value": -0.2, "name": "Prior+Duel hs", "url": "https://arxiv.org/abs/1509.06461v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": -0.2, "maxval": -0.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Reinforcement Learning with Double Q-learning", "label": "Prior+Duel hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-01-06", "value": 8.9, "name": "Prior noop", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.9, "maxval": 8.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-01-06", "value": 5.9, "name": "Prior hs", "url": "https://arxiv.org/abs/1511.05952", "min_date": "2015-11-18", "max_date": "2016-02-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.9, "maxval": 5.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Prioritized Experience Replay", "label": "Prior hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-02-24", "value": -2.5, "name": "DDQN+Pop-Art noop", "url": "https://arxiv.org/abs/1602.07714v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -2.5, "maxval": -2.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning functions across many orders of magnitudes", "label": "DDQN+Pop-Art noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-04-05", "value": 1.2, "name": "Prior+Duel noop", "url": "https://arxiv.org/abs/1511.06581v3", "min_date": "2015-11-18", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1511.05952", "src_name": "Prioritized Experience Replay", "uncertainty": 0, "minval": 1.2, "maxval": 1.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dueling Network Architectures for Deep Reinforcement Learning", "label": "Prior+Duel noop", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-04-10", "value": -8.3, "name": "A3C LSTM hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -8.3, "maxval": -8.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C LSTM hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-04-10", "value": -9.6, "name": "A3C FF (1 day) hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -9.6, "maxval": -9.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF (1 day) hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2016-04-10", "value": -9.7, "name": "A3C FF hs", "url": "https://arxiv.org/abs/1602.01783", "min_date": "2016-02-04", "max_date": "2016-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": -9.7, "maxval": -9.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Asynchronous Methods for Deep Reinforcement Learning", "label": "A3C FF hs", "metric": "Metric(Atari 2600 Surround) SOLVED" }, { "date": "2017-07-21", "value": 6.8, "name": "C51 noop", "url": "https://arxiv.org/abs/1707.06887v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.8, "maxval": 6.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Distributional Perspective on Reinforcement Learning", "label": "C51 noop", "metric": "Metric(Atari 2600 Surround) SOLVED" } ], "name": "Atari 2600 Surround", "solved": true, "url": null, "notes": "", "scale": "Score", "target": 6.5, "target_source": "https://arxiv.org/abs/1509.06461", "changeable": false, "graphed": true, "parent": "Problem(Simple video games)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/scrapers/atari.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/scrapers/atari.py#L2431" } ], "solved": false, "url": null }, { "name": "Speech Recognition", "attributes": [ "language", "agi" ], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [ { "date": "2011-08-31", "value": 16.1, "name": "CD-DNN", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/CD-DNN-HMM-SWB-Interspeech2011-Pub.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.1, "maxval": 16.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "CD-DNN", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2012-04-27", "value": 18.5, "name": "DNN-HMM", "url": "https://pdfs.semanticscholar.org/ce25/00257fda92338ec0a117bea1dbc0381d7c73.pdf?_ga=1.195375081.452266805.1483390947", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.5, "maxval": 18.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNN-HMM", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-23", "value": 12.6, "name": "HMM-DNN +sMBR", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.6, "maxval": 12.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-DNN +sMBR", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-23", "value": 11.5, "name": "CNN", "url": "http://www.cs.toronto.edu/~asamir/papers/icassp13_cnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11.5, "maxval": 11.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "CNN", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-25", "value": 12.9, "name": "DNN MMI", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.9, "maxval": 12.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNN MMI", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-25", "value": 12.9, "name": "DNN MPE", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.9, "maxval": 12.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNN MPE", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-25", "value": 12.9, "name": "DNN BMMI", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.9, "maxval": 12.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNN BMMI", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2013-08-25", "value": 12.6, "name": "DNN sMBR", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.6, "maxval": 12.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNN sMBR", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-06-23", "value": 15, "name": "DNN + Dropout", "url": "https://arxiv.org/abs/1406.7806v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 15, "maxval": 15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Building DNN Acoustic Models for Large Vocabulary Speech Recognition", "label": "DNN + Dropout", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-06-30", "value": 16, "name": "DNN", "url": "https://arxiv.org/abs/1406.7806v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16, "maxval": 16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Increasing Deep Neural Network Acoustic Model Size for Large Vocabulary Continuous Speech Recognition", "label": "DNN", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-08-23", "value": 10.4, "name": "CNN on MFSC/fbanks + 1 non-conv layer for FMLLR/I-Vectors concatenated in a DNN", "url": "http://www.mirlab.org/conference_papers/International_Conference/ICASSP%202014/papers/p5609-soltau.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10.4, "maxval": 10.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "CNN on MFSC/fbanks + 1 no...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-12-07", "value": 20, "name": "Deep Speech", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20, "maxval": 20, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "Deep Speech", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-12-07", "value": 12.6, "name": "Deep Speech + FSH", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.6, "maxval": 12.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "Deep Speech + FSH", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2014-12-23", "value": 12.6, "name": "CNN + Bi-RNN + CTC (speech to letters), 25.9% WER if trainedonlyon SWB", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.6, "maxval": 12.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "CNN + Bi-RNN + CTC (speec...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2015-05-21", "value": 8.0, "name": "IBM 2015", "url": "https://arxiv.org/abs/1505.05899", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.0, "maxval": 8.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The IBM 2015 English Conversational Telephone Speech Recognition System", "label": "IBM 2015", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2015-08-23", "value": 12.9, "name": "HMM-TDNN + pNorm + speed up/down speech", "url": "http://www.danielpovey.com/files/2015_interspeech_augmentation.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.9, "maxval": 12.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN + pNorm + speed ...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2015-08-23", "value": 11, "name": "HMM-TDNN + iVectors", "url": "http://speak.clsp.jhu.edu/uploads/publications/papers/1048_pdf.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11, "maxval": 11, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN + iVectors", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2015-09-23", "value": 12.2, "name": "Deep CNN (10 conv, 4 FC layers), multi-scale feature maps", "url": "https://arxiv.org/abs/1509.08967v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.2, "maxval": 12.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Very Deep Multilingual Convolutional Neural Networks for LVCSR", "label": "Deep CNN (10 conv, 4 FC l...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-04-27", "value": 6.9, "name": "IBM 2016", "url": "https://arxiv.org/abs/1604.08242v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.9, "maxval": 6.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The IBM 2016 English Conversational Telephone Speech Recognition System", "label": "IBM 2016", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-06-23", "value": 6.6, "name": "RNN + VGG + LSTM acoustic model trained on SWB+Fisher+CH, N-gram + \"model M\" + NNLM language model", "url": "https://arxiv.org/abs/1604.08242v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.6, "maxval": 6.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The IBM 2016 English Conversational Telephone Speech Recognition System", "label": "RNN + VGG + LSTM acoustic...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-09-23", "value": 9.2, "name": "HMM-TDNN trained with MMI + data augmentation (speed) + iVectors + 3 regularizations + Fisher (10% / 15.1% respectively trained on SWBD only)", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.2, "maxval": 9.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN trained with MMI...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-09-23", "value": 8.5, "name": "HMM-BLSTM trained with MMI + data augmentation (speed) + iVectors + 3 regularizations + Fisher", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.5, "maxval": 8.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-BLSTM trained with MM...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-09-23", "value": 6.3, "name": "VGG/Resnet/LACE/BiLSTM acoustic model trained on SWB+Fisher+CH, N-gram + RNNLM language model trained on Switchboard+Fisher+Gigaword+Broadcast", "url": "https://arxiv.org/abs/1609.03528v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.3, "maxval": 6.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Microsoft 2016 Conversational Speech Recognition System", "label": "VGG/Resnet/LACE/BiLSTM ac...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-10-17", "value": 6.6, "name": "CNN-LSTM", "url": "https://arxiv.org/abs/1610.05256", "min_date": "2016-10-17", "max_date": "2017-02-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.6, "maxval": 6.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Achieving Human Parity in Conversational Speech Recognition", "label": "CNN-LSTM", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2016-12-17", "value": 5.8, "name": "Microsoft 2016b", "url": "https://arxiv.org/abs/1610.05256", "min_date": "2016-10-17", "max_date": "2017-02-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.8, "maxval": 5.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Achieving Human Parity in Conversational Speech Recognition", "label": "Microsoft 2016b", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2017-02-17", "value": 6.9, "name": "RNNLM", "url": "https://arxiv.org/abs/1609.03528", "min_date": "2016-09-12", "max_date": "2017-01-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.9, "maxval": 6.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Microsoft 2016 Conversational Speech Recognition System", "label": "RNNLM", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2017-02-17", "value": 6.2, "name": "Microsoft 2016", "url": "https://arxiv.org/abs/1609.03528", "min_date": "2016-09-12", "max_date": "2017-01-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.2, "maxval": 6.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Microsoft 2016 Conversational Speech Recognition System", "label": "Microsoft 2016", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" }, { "date": "2017-03-23", "value": 5.5, "name": "ResNet + BiLSTMs acoustic model, with 40d FMLLR + i-Vector inputs, trained on SWB+Fisher+CH, n-gram + model-M + LSTM + Strided ( trous) convs-based LM trained on Switchboard+Fisher+Gigaword+Broadcast", "url": "https://arxiv.org/abs/1703.02136", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.5, "maxval": 5.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "English Conversational Telephone Speech Recognition by Humans and Machines", "label": "ResNet + BiLSTMs acoustic...", "metric": "Metric(Word error rate on Switchboard trained against the Hub5'00 dataset)SOLVED" } ], "name": "Word error rate on Switchboard trained against the Hub5'00 dataset", "solved": true, "url": null, "notes": "", "scale": "Percentage error", "target": 5.9, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L42" }, { "measures": [ { "date": "2015-08-23", "value": 8.01, "name": "HMM-(SAT)GMM", "url": "http://kaldi-asr.org/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 8.01, "maxval": 8.01, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Kaldi ASR", "label": "HMM-(SAT)GMM", "metric": "Metric(librispeech WER testclean) SOLVED" }, { "date": "2015-08-23", "value": 5.51, "name": "HMM-DNN + pNorm*", "url": "http://www.danielpovey.com/files/2015_icassp_librispeech.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.51, "maxval": 5.51, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-DNN + pNorm*", "metric": "Metric(librispeech WER testclean) SOLVED" }, { "date": "2015-08-23", "value": 4.83, "name": "HMM-TDNN + iVectors", "url": "http://speak.clsp.jhu.edu/uploads/publications/papers/1048_pdf.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.83, "maxval": 4.83, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN + iVectors", "metric": "Metric(librispeech WER testclean) SOLVED" }, { "date": "2015-12-23", "value": 5.33, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters trained on 11940h", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.33, "maxval": 5.33, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(librispeech WER testclean) SOLVED" }, { "date": "2016-09-23", "value": 4.28, "name": "HMM-TDNN trained with MMI + data augmentation (speed) + iVectors + 3 regularizations", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.28, "maxval": 4.28, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN trained with MMI...", "metric": "Metric(librispeech WER testclean) SOLVED" } ], "name": "librispeech WER testclean", "solved": true, "url": null, "notes": "", "scale": "Percentage error", "target": 5.83, "target_source": "http://arxiv.org/abs/1512.02595v1", "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L14" }, { "measures": [ { "date": "2015-08-23", "value": 22.49, "name": "HMM-(SAT)GMM", "url": "http://kaldi-asr.org/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22.49, "maxval": 22.49, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Kaldi ASR", "label": "HMM-(SAT)GMM", "metric": "Metric(librispeech WER testother) SOLVED" }, { "date": "2015-08-23", "value": 13.97, "name": "HMM-DNN + pNorm*", "url": "http://www.danielpovey.com/files/2015_icassp_librispeech.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13.97, "maxval": 13.97, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-DNN + pNorm*", "metric": "Metric(librispeech WER testother) SOLVED" }, { "date": "2015-08-23", "value": 12.51, "name": "TDNN + pNorm + speed up/down speech", "url": "http://www.danielpovey.com/files/2015_interspeech_augmentation.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.51, "maxval": 12.51, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "TDNN + pNorm + speed up/d...", "metric": "Metric(librispeech WER testother) SOLVED" }, { "date": "2015-12-23", "value": 13.25, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters trained on 11940h", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13.25, "maxval": 13.25, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(librispeech WER testother) SOLVED" } ], "name": "librispeech WER testother", "solved": true, "url": null, "notes": "", "scale": "Percentage error", "target": 12.69, "target_source": "http://arxiv.org/abs/1512.02595v1", "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L18" }, { "measures": [ { "date": "2014-08-23", "value": 5.6, "name": "CNN over RAW speech (wav)", "url": "http://infoscience.epfl.ch/record/203464/files/Palaz_Idiap-RR-18-2014.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.6, "maxval": 5.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "CNN over RAW speech (wav)", "metric": "Metric(wsj WER eval92) SOLVED" }, { "date": "2015-04-23", "value": 3.47, "name": "TC-DNN-BLSTM-DNN", "url": "https://arxiv.org/abs/1504.01482v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.47, "maxval": 3.47, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Recurrent Neural Networks for Acoustic Modelling", "label": "TC-DNN-BLSTM-DNN", "metric": "Metric(wsj WER eval92) SOLVED" }, { "date": "2015-08-23", "value": 3.63, "name": "test-set on open vocabulary (i.e. harder), model = HMM-DNN + pNorm*", "url": "http://www.danielpovey.com/files/2015_icassp_librispeech.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.63, "maxval": 3.63, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "test-set on open vocabula...", "metric": "Metric(wsj WER eval92) SOLVED" }, { "date": "2015-12-23", "value": 3.6, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.6, "maxval": 3.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(wsj WER eval92) SOLVED" } ], "name": "wsj WER eval92", "solved": true, "url": null, "notes": "", "scale": "Percentage error", "target": 5.03, "target_source": "http://arxiv.org/abs/1512.02595v1", "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L24" }, { "measures": [ { "date": "2015-08-23", "value": 5.66, "name": "test-set on open vocabulary (i.e. harder), model = HMM-DNN + pNorm*", "url": "http://www.danielpovey.com/files/2015_icassp_librispeech.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 5.66, "maxval": 5.66, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "test-set on open vocabula...", "metric": "Metric(wsj WER eval93) SOLVED" }, { "date": "2015-12-23", "value": 4.98, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.98, "maxval": 4.98, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(wsj WER eval93) SOLVED" } ], "name": "wsj WER eval93", "solved": true, "url": null, "notes": "", "scale": "Percentage error", "target": 8.08, "target_source": "http://arxiv.org/abs/1512.02595v1", "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L26" }, { "measures": [ { "date": "2013-08-23", "value": 18.4, "name": "HMM-DNN +sMBR", "url": "http://www.danielpovey.com/files/2013_interspeech_dnn.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 18.4, "maxval": 18.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-DNN +sMBR", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2014-06-23", "value": 19.1, "name": "DNN + Dropout", "url": "https://arxiv.org/abs/1406.7806v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.1, "maxval": 19.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Building DNN Acoustic Models for Large Vocabulary Speech Recognition", "label": "DNN + Dropout", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2014-12-23", "value": 16, "name": "CNN + Bi-RNN + CTC (speech to letters), 25.9% WER if trainedonlyon SWB", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16, "maxval": 16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "CNN + Bi-RNN + CTC (speec...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2015-08-23", "value": 19.3, "name": "HMM-TDNN + pNorm + speed up/down speech", "url": "http://www.danielpovey.com/files/2015_interspeech_augmentation.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 19.3, "maxval": 19.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN + pNorm + speed ...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2015-08-23", "value": 17.1, "name": "HMM-TDNN + iVectors", "url": "http://speak.clsp.jhu.edu/uploads/publications/papers/1048_pdf.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.1, "maxval": 17.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN + iVectors", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2016-06-23", "value": 12.2, "name": "RNN + VGG + LSTM acoustic model trained on SWB+Fisher+CH, N-gram + \"model M\" + NNLM language model", "url": "https://arxiv.org/abs/1604.08242v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.2, "maxval": 12.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The IBM 2016 English Conversational Telephone Speech Recognition System", "label": "RNN + VGG + LSTM acoustic...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2016-09-23", "value": 13.3, "name": "HMM-TDNN trained with MMI + data augmentation (speed) + iVectors + 3 regularizations + Fisher (10% / 15.1% respectively trained on SWBD only)", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13.3, "maxval": 13.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNN trained with MMI...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2016-09-23", "value": 13, "name": "HMM-BLSTM trained with MMI + data augmentation (speed) + iVectors + 3 regularizations + Fisher", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 13, "maxval": 13, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-BLSTM trained with MM...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2016-09-23", "value": 11.9, "name": "VGG/Resnet/LACE/BiLSTM acoustic model trained on SWB+Fisher+CH, N-gram + RNNLM language model trained on Switchboard+Fisher+Gigaword+Broadcast", "url": "https://arxiv.org/abs/1609.03528v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 11.9, "maxval": 11.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Microsoft 2016 Conversational Speech Recognition System", "label": "VGG/Resnet/LACE/BiLSTM ac...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" }, { "date": "2017-03-23", "value": 10.3, "name": "ResNet + BiLSTMs acoustic model, with 40d FMLLR + i-Vector inputs, trained on SWB+Fisher+CH, n-gram + model-M + LSTM + Strided ( trous) convs-based LM trained on Switchboard+Fisher+Gigaword+Broadcast", "url": "https://arxiv.org/abs/1703.02136", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 10.3, "maxval": 10.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "English Conversational Telephone Speech Recognition by Humans and Machines", "label": "ResNet + BiLSTMs acoustic...", "metric": "Metric(swb_hub_500 WER fullSWBCH) ?" } ], "name": "swb_hub_500 WER fullSWBCH", "solved": false, "url": null, "notes": "", "scale": "Percentage error", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L52" }, { "measures": [ { "date": "2016-09-23", "value": 9.8, "name": "HMM-TDNNtrained with MMI + data augmentation (speed) + iVectors + 3 regularizations + SWBD", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.8, "maxval": 9.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-TDNNtrained with MMI ...", "metric": "Metric(fisher WER) ?" }, { "date": "2016-09-23", "value": 9.6, "name": "HMM-BLSTMtrained with MMI + data augmentation (speed) + iVectors + 3 regularizations + SWBD", "url": "http://www.danielpovey.com/files/2016_interspeech_mmi.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 9.6, "maxval": 9.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "HMM-BLSTMtrained with MMI...", "metric": "Metric(fisher WER) ?" } ], "name": "fisher WER", "solved": false, "url": null, "notes": "", "scale": "Percentage error", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L55" }, { "measures": [ { "date": "2014-12-23", "value": 6.3, "name": "CNN + Bi-RNN + CTC (speech to letters)", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.3, "maxval": 6.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "CNN + Bi-RNN + CTC (speec...", "metric": "Metric(chime clean) ?" }, { "date": "2015-12-23", "value": 3.34, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.34, "maxval": 3.34, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(chime clean) ?" } ], "name": "chime clean", "solved": false, "url": null, "notes": "", "scale": "Percentage error", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L59" }, { "measures": [ { "date": "2014-12-23", "value": 67.94, "name": "CNN + Bi-RNN + CTC (speech to letters)", "url": "https://arxiv.org/abs/1412.5567", "min_date": "2014-12-17", "max_date": "2014-12-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.94, "maxval": 67.94, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech: Scaling up end-to-end speech recognition", "label": "CNN + Bi-RNN + CTC (speec...", "metric": "Metric(chime real) ?" }, { "date": "2015-12-23", "value": 21.79, "name": "9-layer model w/ 2 layers of 2D-invariant convolution & 7 recurrent layers, w/ 68M parameters", "url": "https://arxiv.org/abs/1512.02595v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.79, "maxval": 21.79, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin", "label": "9-layer model w/ 2 layers...", "metric": "Metric(chime real) ?" } ], "name": "chime real", "solved": false, "url": null, "notes": "", "scale": "Percentage error", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L61" }, { "measures": [ { "date": "2009-08-23", "value": 23, "name": "(first, modern) HMM-DBN", "url": "http://www.cs.toronto.edu/~asamir/papers/NIPS09.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23, "maxval": 23, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "(first, modern) HMM-DBN", "metric": "Metric(timit PER) ?" }, { "date": "2013-03-23", "value": 17.7, "name": "Bi-LSTM + skip connections w/ CTC", "url": "https://arxiv.org/abs/1303.5778v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.7, "maxval": 17.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Speech Recognition with Deep Recurrent Neural Networks", "label": "Bi-LSTM + skip connection...", "metric": "Metric(timit PER) ?" }, { "date": "2014-08-23", "value": 16.7, "name": "CNN in time and frequency + dropout, 17.6% w/o dropout", "url": "http://www.inf.u-szeged.hu/~tothl/pubs/ICASSP2014.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.7, "maxval": 16.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "CNN in time and frequency...", "metric": "Metric(timit PER) ?" }, { "date": "2015-06-23", "value": 17.6, "name": "Bi-RNN + Attention", "url": "https://arxiv.org/abs/1506.07503", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.6, "maxval": 17.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention-Based Models for Speech Recognition", "label": "Bi-RNN + Attention", "metric": "Metric(timit PER) ?" }, { "date": "2015-09-23", "value": 16.5, "name": "Hierarchical maxout CNN + Dropout", "url": "https://link.springer.com/content/pdf/10.1186%2Fs13636-015-0068-3.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.5, "maxval": 16.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": " ", "label": "Hierarchical maxout CNN +...", "metric": "Metric(timit PER) ?" }, { "date": "2016-03-23", "value": 17.3, "name": "RNN-CRF on 24(x3) MFSC", "url": "https://arxiv.org/abs/1603.00223", "min_date": "2016-03-01", "max_date": "2016-06-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.3, "maxval": 17.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Segmental Recurrent Neural Networks for End-to-end Speech Recognition", "label": "RNN-CRF on 24(x3) MFSC", "metric": "Metric(timit PER) ?" } ], "name": "timit PER", "solved": false, "url": null, "notes": "", "scale": "Percentage error", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Speech Recognition)", "target_label": null, "axis_label": "Percentage error", "data_path": "/home/pde/aip/data/wer.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/wer.py#L68" } ], "solved": true, "url": null }, { "name": "Detection of Instrumentals musical tracks", "attributes": [ "language", "agi" ], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [ { "date": "2013-10-17", "value": 17.3, "name": "Ghosal et al.", "url": "https://link.springer.com/article/10.1186/2193-1801-2-526", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.3, "maxval": 17.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A hierarchical approach for speech-instrumental-song classification | SpringerLink", "label": "Ghosal et al.", "metric": "Metric(Precision of Instrumentals detection reached when tested on SATIN (Bayle et al. 2017))not solved" }, { "date": "2014-09-30", "value": 29.8, "name": "VQMM", "url": "https://arxiv.org/abs/1410.0001", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 29.8, "maxval": 29.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On Evaluation Validity in Music Autotagging", "label": "VQMM", "metric": "Metric(Precision of Instrumentals detection reached when tested on SATIN (Bayle et al. 2017))not solved" }, { "date": "2014-09-30", "value": 12.5, "name": "SVMBFF", "url": "https://arxiv.org/abs/1410.0001", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 12.5, "maxval": 12.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "On Evaluation Validity in Music Autotagging", "label": "SVMBFF", "metric": "Metric(Precision of Instrumentals detection reached when tested on SATIN (Bayle et al. 2017))not solved" }, { "date": "2017-06-23", "value": 82.5, "name": "Bayle et al.", "url": "https://arxiv.org/abs/1706.07613", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 82.5, "maxval": 82.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Revisiting Autotagging Toward Faultless Instrumental Playlists Generation", "label": "Bayle et al.", "metric": "Metric(Precision of Instrumentals detection reached when tested on SATIN (Bayle et al. 2017))not solved" } ], "name": "Precision of Instrumentals detection reached when tested on SATIN (Bayle et al. 2017)", "solved": false, "url": null, "notes": "", "scale": "Percentage correct", "target": 99, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Detection of Instrumentals musical tracks)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/acoustics.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/acoustics.py#L59" } ], "solved": false, "url": null }, { "name": "Drawing pictures", "attributes": [ "vision", "agi" ], "subproblems": [], "superproblems": [ "Be able to generate complex scene e.g. a baboon receiving their degree at convocatoin." ], "metrics": [ { "measures": [ { "date": "2014-10-30", "value": 4.48, "name": "NICE", "url": "https://arxiv.org/abs/1410.8516", "min_date": "2014-10-30", "max_date": "2015-04-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.48, "maxval": 4.48, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "NICE: Non-linear Independent Components Estimation", "label": "NICE", "metric": "Metric(Generative models of CIFAR-10 images) ?" }, { "date": "2015-02-16", "value": 4.13, "name": "DRAW", "url": "https://arxiv.org/abs/1502.04623", "min_date": "2015-02-16", "max_date": "2015-05-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.13, "maxval": 4.13, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "DRAW: A Recurrent Neural Network For Image Generation", "label": "DRAW", "metric": "Metric(Generative models of CIFAR-10 images) ?" }, { "date": "2016-05-27", "value": 3.0, "name": "PixelRNN", "url": "https://arxiv.org/abs/1605.08803", "min_date": "2016-05-27", "max_date": "2017-02-27", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.0, "maxval": 3.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Density estimation using Real NVP", "label": "PixelRNN", "metric": "Metric(Generative models of CIFAR-10 images) ?" }, { "date": "2016-05-27", "value": 3.49, "name": "Real NVP", "url": "https://arxiv.org/abs/1605.08803", "min_date": "2016-05-27", "max_date": "2017-02-27", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.49, "maxval": 3.49, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Density estimation using Real NVP", "label": "Real NVP", "metric": "Metric(Generative models of CIFAR-10 images) ?" }, { "date": "2016-06-15", "value": 3.11, "name": "VAE with IAF", "url": "https://papers.nips.cc/paper/6581-improved-variational-inference-with-inverse-autoregressive-flow", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 3.11, "maxval": 3.11, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Improved Variational Inference with Inverse Autoregressive Flow", "label": "VAE with IAF", "metric": "Metric(Generative models of CIFAR-10 images) ?" }, { "date": "2016-11-04", "value": 2.92, "name": "PixelCNN++", "url": "https://openreview.net/forum?id=BJrFC6ceg", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 2.92, "maxval": 2.92, "opensource": false, "not_directly_comparable": false, "replicated_url": "https://github.com/openai/pixel-cnn", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Forum | OpenReview", "label": "PixelCNN++", "metric": "Metric(Generative models of CIFAR-10 images) ?" } ], "name": "Generative models of CIFAR-10 images", "solved": false, "url": null, "notes": "", "scale": "Model Entropy", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Drawing pictures)", "target_label": null, "axis_label": "Model entropy (bits per pixel)", "data_path": "/home/pde/aip/data/generative.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/generative.py#L31" } ], "solved": false, "url": null }, { "name": "Be able to generate complex scene e.g. a baboon receiving their degree at convocatoin.", "attributes": [ "vision", "world-modelling", "agi" ], "subproblems": [ "Drawing pictures" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Accurate modelling of human language.", "attributes": [ "language", "agi" ], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [ { "date": "2012-04-07", "value": 78.8, "name": "KN5+RNNME ensemble", "url": "http://www.fit.vutbr.cz/~imikolov/rnnlm/google.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.8, "maxval": 78.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "KN5+RNNME ensemble", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-04-07", "value": 125.7, "name": "KN5+cache baseline", "url": "http://www.fit.vutbr.cz/~imikolov/rnnlm/google.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 125.7, "maxval": 125.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "KN5+cache baseline", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-07-27", "value": 74.1, "name": "RNN-LDA+all", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2012/07/rnn_ctxt_TR.sav_.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.1, "maxval": 74.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNN-LDA+all", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-07-27", "value": 80.1, "name": "RNN-LDA ensemble", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2012/07/rnn_ctxt_TR.sav_.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 80.1, "maxval": 80.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNN-LDA ensemble", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-07-27", "value": 92.0, "name": "RNN-LDA LM+KN5+cache", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2012/07/rnn_ctxt_TR.sav_.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 92.0, "maxval": 92.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNN-LDA LM+KN5+cache", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-07-27", "value": 113.7, "name": "RNN-LDA LM", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2012/07/rnn_ctxt_TR.sav_.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 113.7, "maxval": 113.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNN-LDA LM", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2012-07-27", "value": 124.7, "name": "RNNLM", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2012/07/rnn_ctxt_TR.sav_.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 124.7, "maxval": 124.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNNLM", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2013-12-20", "value": 107.5, "name": "Deep RNN", "url": "https://arxiv.org/abs/1312.6026", "min_date": "2013-12-20", "max_date": "2014-04-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 107.5, "maxval": 107.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "How to Construct Deep Recurrent Neural Networks", "label": "Deep RNN", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2014-09-08", "value": 68.7, "name": "RNN Dropout Regularization", "url": "https://arxiv.org/abs/1409.2329v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.7, "maxval": 68.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Neural Network Regularization", "label": "RNN Dropout Regularization", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2016-08-11", "value": 71.3, "name": "RHN", "url": "https://arxiv.org/abs/1607.03474v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.3, "maxval": 71.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "RHN", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2016-09-26", "value": 70.9, "name": "Pointer Sentinel-LSTM", "url": "https://arxiv.org/abs/1609.07843v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.9, "maxval": 70.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Pointer Sentinel Mixture Models", "label": "Pointer Sentinel-LSTM", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2016-10-05", "value": 73.4, "name": "Variational LSTM", "url": "https://arxiv.org/abs/1512.05287v5", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.4, "maxval": 73.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Theoretically Grounded Application of Dropout in Recurrent Neural Networks", "label": "Variational LSTM", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2016-10-27", "value": 66, "name": "RHN+WT", "url": "https://arxiv.org/abs/1607.03474v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66, "maxval": 66, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "RHN+WT", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2016-10-27", "value": 68.5, "name": "RHN", "url": "https://arxiv.org/abs/1607.03474v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.5, "maxval": 68.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "RHN", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2017-02-15", "value": 62.4, "name": "Neural Architecture Search", "url": "https://arxiv.org/abs/1611.01578v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 62.4, "maxval": 62.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Architecture Search with Reinforcement Learning", "label": "Neural Architecture Search", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2017-03-03", "value": 65.4, "name": "RHN+WT", "url": "https://arxiv.org/abs/1607.03474v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65.4, "maxval": 65.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "RHN+WT", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2018-02-10", "value": 55.8, "name": "ENAS", "url": "https://arxiv.org/abs/1802.03268", "min_date": "2018-02-09", "max_date": "2018-02-12", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 55.8, "maxval": 55.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Efficient Neural Architecture Search via Parameter Sharing", "label": "ENAS", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2018-03-02", "value": 47.67, "name": "AWD-LSTM-MOS+de", "url": "https://arxiv.org/abs/1711.03953v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 47.67, "maxval": 47.67, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Breaking the Softmax Bottleneck: A High-Rank RNN Language Model", "label": "AWD-LSTM-MOS+de", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2018-09-18", "value": 46.54, "name": "FRAGE", "url": "https://arxiv.org/abs/1809.06858", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 46.54, "maxval": 46.54, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "FRAGE: Frequency-Agnostic Word Representation", "label": "FRAGE", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" }, { "date": "2019-02-14", "value": 35.76, "name": "GPT2 (zero shot)", "url": "https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 35.76, "maxval": 35.76, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Language Models are Unsupervised Multitask Learners", "label": "GPT2 (zero shot)", "metric": "Metric(Penn Treebank (Perplexity when parsing English sentences))?" } ], "name": "Penn Treebank (Perplexity when parsing English sentences)", "solved": false, "url": null, "notes": "", "scale": "Perplexity", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Accurate modelling of human language.)", "target_label": null, "axis_label": "Perplexity", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L32" }, { "measures": [ { "date": "2011-06-28", "value": 1.6, "name": "RNN", "url": "http://www.cs.utoronto.ca/~ilya/pubs/2011/LANG-RNN.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.6, "maxval": 1.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "RNN", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2013-08-04", "value": 1.67, "name": "RNN, LSTM", "url": "https://arxiv.org/abs/1308.0850", "min_date": "2013-08-04", "max_date": "2014-06-05", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.67, "maxval": 1.67, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Generating Sequences With Recurrent Neural Networks", "label": "RNN, LSTM", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2015-02-15", "value": 1.58, "name": "Gated Feedback RNN", "url": "https://arxiv.org/abs/1502.02367", "min_date": "2015-02-09", "max_date": "2015-06-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.58, "maxval": 1.58, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated Feedback Recurrent Neural Networks", "label": "Gated Feedback RNN", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2015-07-06", "value": 1.47, "name": "Grid LSTM", "url": "https://arxiv.org/abs/1507.01526", "min_date": "2015-07-06", "max_date": "2016-01-07", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.47, "maxval": 1.47, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Grid Long Short-Term Memory", "label": "Grid LSTM", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-07-12", "value": 1.32, "name": "Recurrent Highway Networks", "url": "https://arxiv.org/abs/1607.03474", "min_date": "2016-07-12", "max_date": "2017-07-04", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.32, "maxval": 1.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "Recurrent Highway Networks", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-08-11", "value": 1.42, "name": "RHN", "url": "https://arxiv.org/abs/1607.03474v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.42, "maxval": 1.42, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "RHN", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-09-06", "value": 1.32, "name": " Hierarchical Multiscale RNN", "url": "https://arxiv.org/abs/1609.01704", "min_date": "2016-09-06", "max_date": "2017-03-09", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.32, "maxval": 1.32, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Hierarchical Multiscale Recurrent Neural Networks", "label": " Hierarchical Multiscale ...", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-09-27", "value": 1.39, "name": "Hypernetworks", "url": "https://arxiv.org/abs/1609.09106", "min_date": "2016-09-27", "max_date": "2016-12-01", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.39, "maxval": 1.39, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "HyperNetworks", "label": "Hypernetworks", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-10-19", "value": 1.37, "name": "Surprisal-Driven Feedback RNN", "url": "https://arxiv.org/abs/1608.06027", "min_date": "2016-08-22", "max_date": "2016-10-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.37, "maxval": 1.37, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Surprisal-Driven Feedback in Recurrent Networks", "label": "Surprisal-Driven Feedback...", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2016-10-31", "value": 1.313, "name": "Surprisal-Driven Zoneout", "url": "https://pdfs.semanticscholar.org/e9bc/83f9ff502bec9cffb750468f76fdfcf5dd05.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.313, "maxval": 1.313, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "Surprisal-Driven Zoneout", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2017-01-06", "value": 1.31, "name": "ByteNet", "url": "https://arxiv.org/abs/1610.10099", "min_date": "2016-10-31", "max_date": "2017-03-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.31, "maxval": 1.31, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Machine Translation in Linear Time", "label": "ByteNet", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" }, { "date": "2017-03-03", "value": 1.27, "name": "Large RHN depth 10", "url": "https://arxiv.org/abs/1607.03474v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 1.27, "maxval": 1.27, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Recurrent Highway Networks", "label": "Large RHN depth 10", "metric": "Metric(Hutter Prize (bits per character to encode English text))SOLVED" } ], "name": "Hutter Prize (bits per character to encode English text)", "solved": true, "url": null, "notes": "", "scale": "Model Entropy", "target": 1.3, "target_source": "http://languagelog.ldc.upenn.edu/myl/Shannon1950.pdf", "changeable": false, "graphed": true, "parent": "Problem(Accurate modelling of human language.)", "target_label": "Region of human performance", "axis_label": "Model Entropy", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L361" }, { "measures": [ { "date": "2017-02-16", "value": 49.0, "name": "GA + feat.", "url": "https://arxiv.org/abs/1610.08431v3", "min_date": "2016-12-01", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1606.01549v2", "src_name": "Gated-Attention Readers for Text Comprehension", "uncertainty": 0, "minval": 49.0, "maxval": 49.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Broad Context Language Modeling as Reading Comprehension", "label": "GA + feat.", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2017-02-16", "value": 44.5, "name": "AS + feat.", "url": "https://arxiv.org/abs/1610.08431v3", "min_date": "2016-03-04", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1603.01547", "src_name": "Text Understanding with the Attention Sum Reader Network", "uncertainty": 0, "minval": 44.5, "maxval": 44.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Broad Context Language Modeling as Reading Comprehension", "label": "AS + feat.", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2017-02-16", "value": 32.1, "name": "Modified Stanford", "url": "https://arxiv.org/abs/1610.08431v3", "min_date": "2016-06-09", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1606.02858", "src_name": "A Thorough Examination of the CNN/Daily Mail Reading Comprehension Task", "uncertainty": 0, "minval": 32.1, "maxval": 32.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Broad Context Language Modeling as Reading Comprehension", "label": "Modified Stanford", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2017-02-16", "value": 21.7, "name": "Stanford Reader", "url": "https://arxiv.org/abs/1610.08431v3", "min_date": "2016-06-09", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1606.02858", "src_name": "A Thorough Examination of the CNN/Daily Mail Reading Comprehension Task", "uncertainty": 0, "minval": 21.7, "maxval": 21.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Broad Context Language Modeling as Reading Comprehension", "label": "Stanford Reader", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2017-03-07", "value": 51.6, "name": "GA+MAGE (48)", "url": "https://arxiv.org/abs/1703.02620v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 51.6, "maxval": 51.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Linguistic Knowledge as Memory for Recurrent Neural Networks", "label": "GA+MAGE (48)", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2017-03-07", "value": 51.6, "name": "GA+MAGE (48)", "url": "https://arxiv.org/abs/1703.02620v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 51.6, "maxval": 51.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Linguistic Knowledge as Memory for Recurrent Neural Networks", "label": "GA+MAGE (48)", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2018-10-05", "value": 60.22, "name": "AttSum-Feat+L1", "url": "https://arxiv.org/abs/1810.02891", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 60.22, "maxval": 60.22, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Entity Tracking Improves Cloze-style Reading Comprehension", "label": "AttSum-Feat+L1", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2019-02-14", "value": 63.4, "name": "GPT2 (zero shot)", "url": "https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.4, "maxval": 63.4, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Language Models are Unsupervised Multitask Learners", "label": "GPT2 (zero shot)", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" }, { "date": "2019-09-19", "value": 66.5, "name": "Megatron-LM (zero shot)", "url": "https://arxiv.org/abs/1909.08053v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.5, "maxval": 66.5, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism", "label": "Megatron-LM (zero shot)", "metric": "Metric(LAMBADA prediction of words in discourse) not solved" } ], "name": "LAMBADA prediction of words in discourse", "solved": false, "url": "https://arxiv.org/abs/1606.06031", "notes": "", "scale": "Percentage correct", "target": 86, "target_source": "https://arxiv.org/abs/1610.08431v3", "changeable": false, "graphed": true, "parent": "Problem(Accurate modelling of human language.)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L67" } ], "solved": false, "url": null }, { "name": "Conduct arbitrary sustained, probing conversation", "attributes": [ "agi", "language", "world-modelling", "communication" ], "subproblems": [ "Turing test for casual conversation", "Language comprehension and question-answering" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Turing test for casual conversation", "attributes": [ "agi", "language", "world-modelling", "communication" ], "subproblems": [], "superproblems": [ "Conduct arbitrary sustained, probing conversation" ], "metrics": [ { "measures": [ { "date": "2014-11-15", "value": 89.2, "name": "Rose 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.2, "maxval": 89.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Rose 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2014-11-15", "value": 88.3, "name": "Izar 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 88.3, "maxval": 88.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Izar 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2014-11-15", "value": 88.3, "name": "Misuku 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 88.3, "maxval": 88.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Misuku 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2014-11-15", "value": 81.67, "name": "Uberbot 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.67, "maxval": 81.67, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Uberbot 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2014-11-15", "value": 80.83, "name": "Tutor 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 80.83, "maxval": 80.83, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Tutor 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2014-11-15", "value": 76.7, "name": "The Professor 2014", "url": "http://www.aisb.org.uk/events/loebner-prize#contest2014", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.7, "maxval": 76.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "The Professor 2014", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2015-09-19", "value": 83.3, "name": "Mitsuku 2015", "url": "http://www.aisb.org.uk/events/loebner-prize#Results15", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 83.3, "maxval": 83.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Mitsuku 2015", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2015-09-19", "value": 80, "name": "Lisa 2015", "url": "http://www.aisb.org.uk/events/loebner-prize#Results15", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 80, "maxval": 80, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Lisa 2015", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2015-09-19", "value": 76.7, "name": "Izar 2015", "url": "http://www.aisb.org.uk/events/loebner-prize#Results15", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.7, "maxval": 76.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Izar 2015", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2015-09-19", "value": 75, "name": "Rose 2015", "url": "http://www.aisb.org.uk/events/loebner-prize#Results15", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75, "maxval": 75, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Rose 2015", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2016-09-17", "value": 90, "name": "Mitsuku 2016", "url": "http://www.aisb.org.uk/events/loebner-prize#Results16", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 90, "maxval": 90, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Mitsuku 2016", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2016-09-17", "value": 78.3, "name": "Tutor 2016", "url": "http://www.aisb.org.uk/events/loebner-prize#Results16", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.3, "maxval": 78.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Tutor 2016", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2016-09-17", "value": 77.5, "name": "Rose 2016", "url": "http://www.aisb.org.uk/events/loebner-prize#Results16", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.5, "maxval": 77.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Rose 2016", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2016-09-17", "value": 77.5, "name": "Arckon 2016", "url": "http://www.aisb.org.uk/events/loebner-prize#Results16", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.5, "maxval": 77.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Arckon 2016", "metric": "Metric(The Loebner Prize scored selection answers) not solved" }, { "date": "2016-09-17", "value": 76.7, "name": "Katie 2016", "url": "http://www.aisb.org.uk/events/loebner-prize#Results16", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.7, "maxval": 76.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "AISB - The Society for the Study of Artificial Intelligence and Simulation of Behaviour - Loebner Prize", "label": "Katie 2016", "metric": "Metric(The Loebner Prize scored selection answers) not solved" } ], "name": "The Loebner Prize scored selection answers", "solved": false, "url": "http://www.aisb.org.uk/events/loebner-prize", "notes": "\nThe Loebner Prize is an actual enactment of the Turing Test. Importantly, judges are instructed to engage in casual, natural\nconversation rather than deliberately probing to determine if participants are \"intelligent\" (Brian Christian, The Most Human Human).\nThis makes it considerably easier than a probing Turing Test, and it is close to being solved. \n\nHowever these aren't scores for the full Loebner Turing Test; since 2014 the Loebner prize has scored its entrants by\ngiving them a corpus of conversation and scoring their answers. We use these numbers because they remove variability\nin the behaviour of the judges. Unfortunately, these questions change from year to year (and have to, since \nentrants will test with last year's data).\n", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": true, "graphed": true, "parent": "Problem(Turing test for casual conversation)", "target_label": "Completely plausible answers", "axis_label": "Percentage of answers rated plausible\n(each year is a different test)", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L106" } ], "solved": false, "url": null }, { "name": "Language comprehension and question-answering", "attributes": [ "language", "world-modelling", "agi" ], "subproblems": [], "superproblems": [ "Conduct arbitrary sustained, probing conversation" ], "metrics": [ { "measures": [ { "date": "2015-02-19", "value": 93.3, "name": "MemNN-AM+NG+NL (1k + strong supervision)", "url": "https://arxiv.org/abs/1502.05698v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 93.3, "maxval": 93.3, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": true, "algorithms": [], "offset": [ 2, 5 ], "notes": "", "withdrawn": false, "papername": "Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks", "label": "MemNN-AM+NG+NL (1k + strong supervision)", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2015-03-31", "value": 93.4, "name": "MemN2N-PE+LS+RN", "url": "https://arxiv.org/abs/1503.08895", "min_date": "2015-03-31", "max_date": "2015-11-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 93.4, "maxval": 93.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "End-To-End Memory Networks", "label": "MemN2N-PE+LS+RN", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2016-01-05", "value": 96.2, "name": "DNC", "url": "https://www.gwern.net/docs/2016-graves.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 96.2, "maxval": 96.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "DNC", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2016-06-30", "value": 97.2, "name": "DMN+", "url": "https://arxiv.org/abs/1607.00036", "min_date": "2016-06-30", "max_date": "2017-03-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 97.2, "maxval": 97.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dynamic Neural Turing Machine with Soft and Hard Addressing Schemes", "label": "DMN+", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2016-09-27", "value": 97.1, "name": "SDNC", "url": "https://arxiv.org/abs/1606.04582v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 97.1, "maxval": 97.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Query-Reduction Networks for Question Answering", "label": "SDNC", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2016-12-09", "value": 99.7, "name": "QRN", "url": "https://arxiv.org/abs/1606.04582v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 99.7, "maxval": 99.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": [ 2, 3 ], "notes": "", "withdrawn": false, "papername": "Query-Reduction Networks for Question Answering", "label": "QRN", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" }, { "date": "2016-12-12", "value": 99.5, "name": "EntNet", "url": "https://arxiv.org/abs/1612.03969", "min_date": "2016-12-12", "max_date": "2017-05-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 99.5, "maxval": 99.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Tracking the World State with Recurrent Entity Networks", "label": "EntNet", "metric": "Metric(bAbi 20 QA (10k training examples)) SOLVED" } ], "name": "bAbi 20 QA (10k training examples)", "solved": true, "url": "http://fb.ai/babi", "notes": "", "scale": "Percentage correct", "target": 99, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": "Excellent performance", "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L149" }, { "measures": [ { "date": "2015-03-31", "value": 86.1, "name": "MemN2N-PE+LS+RN", "url": "https://arxiv.org/abs/1503.08895", "min_date": "2015-03-31", "max_date": "2015-11-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 86.1, "maxval": 86.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "End-To-End Memory Networks", "label": "MemN2N-PE+LS+RN", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" }, { "date": "2015-06-24", "value": 93.6, "name": "DMN", "url": "https://arxiv.org/abs/1506.07285", "min_date": "2015-06-24", "max_date": "2016-03-05", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 93.6, "maxval": 93.6, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": [ 3, -2 ], "notes": "", "withdrawn": false, "papername": "Ask Me Anything: Dynamic Memory Networks for Natural Language Processing", "label": "DMN", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" }, { "date": "2016-12-09", "value": 90.1, "name": "QRN", "url": "https://arxiv.org/abs/1606.04582v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 90.1, "maxval": 90.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Query-Reduction Networks for Question Answering", "label": "QRN", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" }, { "date": "2016-12-09", "value": 66.8, "name": "DMN+", "url": "https://arxiv.org/abs/1606.04582v4", "min_date": "2016-06-30", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1607.00036", "src_name": "Dynamic Neural Turing Machine with Soft and Hard Addressing Schemes", "uncertainty": 0, "minval": 66.8, "maxval": 66.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "https://github.com/therne/dmn-tensorflow", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Query-Reduction Networks for Question Answering", "label": "DMN+", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" }, { "date": "2016-12-12", "value": 89.1, "name": "EntNet", "url": "https://arxiv.org/abs/1612.03969", "min_date": "2016-12-12", "max_date": "2017-05-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.1, "maxval": 89.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Tracking the World State with Recurrent Entity Networks", "label": "EntNet", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" }, { "date": "2017-03-07", "value": 91.3, "name": "GA+MAGE (16)", "url": "https://arxiv.org/abs/1703.02620v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 91.3, "maxval": 91.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Linguistic Knowledge as Memory for Recurrent Neural Networks", "label": "GA+MAGE (16)", "metric": "Metric(bAbi 20 QA (1k training examples)) not solved" } ], "name": "bAbi 20 QA (1k training examples)", "solved": false, "url": "http://fb.ai/babi", "notes": "\nA synthetic environment inspired by text adventures and SHRDLU, which enables generation\nof ground truths, describing sentences, and inferential questions. Includes:\nsupporting facts, relations, yes/no questions, counting, lists/sets, negation, indefiniteness,\nconference, conjunction, time, basic deduction and induction, reasoning about position, size,\npath finding and motivation.\n\nTable 3 of https://arxiv.org/abs/1502.05698 actually breaks this down into 20 submeasures\nbut initially we're lumping all of this together.\n\nOriginally \"solving\" bABI was defined as 95% accuracy (or perhaps) 95% accuracy on all submeasures,\nbut clearly humans and now algorithms are better than that.\n\nTODO: bAbi really needs to be decomposed into semi-supervised and unsupervised variants, and \nby amount of training data provided\n", "scale": "Percentage correct", "target": 99, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": "Excellent performance", "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L151" }, { "measures": [ { "date": "2013-10-01", "value": 69.16, "name": "SW+D+RTE", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/MCTest_EMNLP2013.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.16, "maxval": 69.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "MCTest: A Challenge Dataset for the Open-Domain Machine Comprehension of Text", "label": "SW+D+RTE", "metric": "Metric(Reading comprehension MCTest-160-all) ?" }, { "date": "2015-07-26", "value": 75.27, "name": "Wang-et-al", "url": "https://arxiv.org/abs/1603.08884", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.27, "maxval": 75.27, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Parallel-Hierarchical Model for Machine Comprehension on Sparse Data", "label": "Wang-et-al", "metric": "Metric(Reading comprehension MCTest-160-all) ?" }, { "date": "2015-07-26", "value": 73.27, "name": "Narasimhan-model3", "url": "https://people.csail.mit.edu/regina/my_papers/MCDR15.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.27, "maxval": 73.27, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Machine Comprehension with Discourse Relations", "label": "Narasimhan-model3", "metric": "Metric(Reading comprehension MCTest-160-all) ?" }, { "date": "2016-03-29", "value": 74.58, "name": "Parallel-Hierarchical", "url": "https://arxiv.org/abs/1603.08884", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.58, "maxval": 74.58, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Parallel-Hierarchical Model for Machine Comprehension on Sparse Data", "label": "Parallel-Hierarchical", "metric": "Metric(Reading comprehension MCTest-160-all) ?" } ], "name": "Reading comprehension MCTest-160-all", "solved": false, "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/MCTest_EMNLP2013.pdf", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L160" }, { "measures": [ { "date": "2013-10-01", "value": 63.33, "name": "SW+D+RTE", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/MCTest_EMNLP2013.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.33, "maxval": 63.33, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "MCTest: A Challenge Dataset for the Open-Domain Machine Comprehension of Text", "label": "SW+D+RTE", "metric": "Metric(Reading comprehension MCTest-500-all) ?" }, { "date": "2015-07-26", "value": 69.94, "name": "Wang-et-al", "url": "https://arxiv.org/abs/1603.08884", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.94, "maxval": 69.94, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Parallel-Hierarchical Model for Machine Comprehension on Sparse Data", "label": "Wang-et-al", "metric": "Metric(Reading comprehension MCTest-500-all) ?" }, { "date": "2015-07-26", "value": 67.83, "name": "LSSVM", "url": "https://pdfs.semanticscholar.org/f26e/088bc4659a9b7fce28b6604d26de779bcf93.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.83, "maxval": 67.83, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning Answer-Entailing Structures for Machine Comprehension", "label": "LSSVM", "metric": "Metric(Reading comprehension MCTest-500-all) ?" }, { "date": "2015-07-26", "value": 63.75, "name": "Narasimhan-model3", "url": "https://people.csail.mit.edu/regina/my_papers/MCDR15.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.75, "maxval": 63.75, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Machine Comprehension with Discourse Relations", "label": "Narasimhan-model3", "metric": "Metric(Reading comprehension MCTest-500-all) ?" }, { "date": "2016-03-29", "value": 71.0, "name": "Parallel-Hierarchical", "url": "https://arxiv.org/abs/1603.08884", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.0, "maxval": 71.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Parallel-Hierarchical Model for Machine Comprehension on Sparse Data", "label": "Parallel-Hierarchical", "metric": "Metric(Reading comprehension MCTest-500-all) ?" } ], "name": "Reading comprehension MCTest-500-all", "solved": false, "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/MCTest_EMNLP2013.pdf", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L167" }, { "measures": [ { "date": "2016-03-04", "value": 68.9, "name": "AS reader (avg)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.9, "maxval": 68.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (avg)", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-03-04", "value": 67.5, "name": "AS reader (greedy)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.5, "maxval": 67.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (greedy)", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-06-05", "value": 69.4, "name": "GA reader", "url": "https://arxiv.org/abs/1606.01549v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.4, "maxval": 69.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA reader", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-06-07", "value": 67.4, "name": "EpiReader", "url": "https://arxiv.org/abs/1606.02270", "min_date": "2016-06-07", "max_date": "2016-06-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.4, "maxval": 67.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Natural Language Comprehension with the EpiReader", "label": "EpiReader", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-07-17", "value": 65.7, "name": "CAS reader", "url": "https://arxiv.org/abs/1607.02250v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 65.7, "maxval": 65.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Consensus Attention-based Neural Networks for Chinese Reading Comprehension", "label": "CAS reader", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-08-04", "value": 69.4, "name": "AoA reader", "url": "https://arxiv.org/abs/1607.04423", "min_date": "2016-07-15", "max_date": "2017-06-06", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.4, "maxval": 69.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention-over-Attention Neural Networks for Reading Comprehension", "label": "AoA reader", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-12-01", "value": 71.9, "name": "NSE", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": "2016-07-04", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1607.04315", "src_name": "Neural Semantic Encoders", "uncertainty": 0, "minval": 71.9, "maxval": 71.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "NSE", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2016-12-01", "value": 70.7, "name": "GA +feature, fix L(w)", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.7, "maxval": 70.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA +feature, fix L(w)", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2017-10-07", "value": 70.0, "name": "DIM Reader", "url": "http://www.cips-cl.org/static/anthology/CCL-2017/CCL-17-075.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.0, "maxval": 70.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "DIM Reader: Dual Interaction Mode for Machine Comprehension", "label": "DIM Reader", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" }, { "date": "2019-02-14", "value": 93.3, "name": "GPT2 (zero shot)", "url": "https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 93.3, "maxval": 93.3, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Language Models are Unsupervised Multitask Learners", "label": "GPT2 (zero shot)", "metric": "Metric(bAbi Children's Book comprehension CBtest CN) SOLVED" } ], "name": "bAbi Children's Book comprehension CBtest CN", "solved": true, "url": "http://fb.ai/babi", "notes": "", "scale": "Percentage correct", "target": 81.6, "target_source": "https://arxiv.org/abs/1511.02301", "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L240" }, { "measures": [ { "date": "2015-06-10", "value": 63.8, "name": "Impatient reader", "url": "https://arxiv.org/abs/1506.03340", "min_date": "2015-06-10", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.8, "maxval": 63.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Teaching Machines to Read and Comprehend", "label": "Impatient reader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2015-06-10", "value": 63.0, "name": "Attentive reader", "url": "https://arxiv.org/abs/1506.03340", "min_date": "2015-06-10", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 63.0, "maxval": 63.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Teaching Machines to Read and Comprehend", "label": "Attentive reader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-03-04", "value": 75.4, "name": "AS reader (avg)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.4, "maxval": 75.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (avg)", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-03-04", "value": 74.8, "name": "AS reader (greedy)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.8, "maxval": 74.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (greedy)", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-06-05", "value": 77.4, "name": "GA reader", "url": "https://arxiv.org/abs/1606.01549v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.4, "maxval": 77.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA reader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-06-07", "value": 75.7, "name": "AIA", "url": "https://arxiv.org/abs/1606.02245v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.7, "maxval": 75.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Iterative Alternating Neural Attention for Machine Reading", "label": "AIA", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-06-07", "value": 74.0, "name": "EpiReader", "url": "https://arxiv.org/abs/1606.02270", "min_date": "2016-06-07", "max_date": "2016-06-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.0, "maxval": 74.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Natural Language Comprehension with the EpiReader", "label": "EpiReader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-07-17", "value": 70.0, "name": "CAS reader", "url": "https://arxiv.org/abs/1607.02250v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.0, "maxval": 70.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Consensus Attention-based Neural Networks for Chinese Reading Comprehension", "label": "CAS reader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-08-04", "value": 74.4, "name": "AoA reader", "url": "https://arxiv.org/abs/1607.04423", "min_date": "2016-07-15", "max_date": "2017-06-06", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.4, "maxval": 74.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention-over-Attention Neural Networks for Reading Comprehension", "label": "AoA reader", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-08-08", "value": 77.6, "name": "Attentive+relabling+ensemble", "url": "https://arxiv.org/abs/1606.02858", "min_date": "2016-06-09", "max_date": "2016-08-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.6, "maxval": 77.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Thorough Examination of the CNN/Daily Mail Reading Comprehension Task", "label": "Attentive+relabling+ensem...", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-09-17", "value": 74.7, "name": "ReasoNet", "url": "https://arxiv.org/abs/1609.05284v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.7, "maxval": 74.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReasoNet: Learning to Stop Reading in Machine Comprehension", "label": "ReasoNet", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-11-09", "value": 76.1, "name": "AIA", "url": "https://arxiv.org/abs/1606.02245v4", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.1, "maxval": 76.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Iterative Alternating Neural Attention for Machine Reading", "label": "AIA", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2016-12-01", "value": 77.9, "name": "GA update L(w)", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.9, "maxval": 77.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA update L(w)", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2017-03-07", "value": 78.6, "name": "GA+MAGE (32)", "url": "https://arxiv.org/abs/1703.02620v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.6, "maxval": 78.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Linguistic Knowledge as Memory for Recurrent Neural Networks", "label": "GA+MAGE (32)", "metric": "Metric(CNN Comprehension test) ?" }, { "date": "2017-10-07", "value": 74.4, "name": "DIM Reader", "url": "http://www.cips-cl.org/static/anthology/CCL-2017/CCL-17-075.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.4, "maxval": 74.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "DIM Reader: Dual Interaction Mode for Machine Comprehension", "label": "DIM Reader", "metric": "Metric(CNN Comprehension test) ?" } ], "name": "CNN Comprehension test", "solved": false, "url": "https://github.com/deepmind/rc-data/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L238" }, { "measures": [ { "date": "2015-06-10", "value": 69.0, "name": "Attentive reader", "url": "https://arxiv.org/abs/1506.03340", "min_date": "2015-06-10", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.0, "maxval": 69.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Teaching Machines to Read and Comprehend", "label": "Attentive reader", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2015-06-10", "value": 68.0, "name": "Impatient reader", "url": "https://arxiv.org/abs/1506.03340", "min_date": "2015-06-10", "max_date": "2015-11-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.0, "maxval": 68.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Teaching Machines to Read and Comprehend", "label": "Impatient reader", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-03-04", "value": 77.7, "name": "AS reader (greedy)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.7, "maxval": 77.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (greedy)", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-03-04", "value": 77.1, "name": "AS reader (avg)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.1, "maxval": 77.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (avg)", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-06-05", "value": 78.1, "name": "GA reader", "url": "https://arxiv.org/abs/1606.01549v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.1, "maxval": 78.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA reader", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-08-08", "value": 79.2, "name": "Attentive+relabling+ensemble", "url": "https://arxiv.org/abs/1606.02858", "min_date": "2016-06-09", "max_date": "2016-08-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.2, "maxval": 79.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Thorough Examination of the CNN/Daily Mail Reading Comprehension Task", "label": "Attentive+relabling+ensem...", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-09-17", "value": 76.6, "name": "ReasoNet", "url": "https://arxiv.org/abs/1609.05284v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.6, "maxval": 76.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReasoNet: Learning to Stop Reading in Machine Comprehension", "label": "ReasoNet", "metric": "Metric(Daily Mail Comprehension test) ?" }, { "date": "2016-12-01", "value": 80.9, "name": "GA update L(w)", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 80.9, "maxval": 80.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA update L(w)", "metric": "Metric(Daily Mail Comprehension test) ?" } ], "name": "Daily Mail Comprehension test", "solved": false, "url": "https://github.com/deepmind/rc-data/", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L231" }, { "measures": [ { "date": "2016-11-04", "value": 71.625, "name": "Dynamic Coattention Networks (ensemble)", "url": "https://arxiv.org/abs/1611.01604v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.625, "maxval": 71.625, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dynamic Coattention Networks For Question Answering", "label": "Dynamic Coattention Netwo...", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-11-04", "value": 66.233, "name": "Dynamic Coattention Networks (single model)", "url": "https://arxiv.org/abs/1611.01604v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 66.233, "maxval": 66.233, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dynamic Coattention Networks For Question Answering", "label": "Dynamic Coattention Netwo...", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-11-07", "value": 67.901, "name": "Match-LSTM+Ans-Ptr", "url": "https://arxiv.org/abs/1608.07905v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 67.901, "maxval": 67.901, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Machine Comprehension Using Match-LSTM and Answer Pointer", "label": "Match-LSTM+Ans-Ptr", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-11-29", "value": 68.478, "name": "BiDAF (single model)", "url": "https://arxiv.org/abs/1611.01603", "min_date": "2016-11-05", "max_date": "2017-02-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.478, "maxval": 68.478, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Bidirectional Attention Flow for Machine Comprehension", "label": "BiDAF (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-12-13", "value": 73.765, "name": "MPM (ensemble)", "url": "https://arxiv.org/abs/1612.04211", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.765, "maxval": 73.765, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multi-Perspective Context Matching for Machine Comprehension", "label": "MPM (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-12-13", "value": 70.387, "name": "MPM (single model)", "url": "https://arxiv.org/abs/1612.04211", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.387, "maxval": 70.387, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multi-Perspective Context Matching for Machine Comprehension", "label": "MPM (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-12-29", "value": 70.849, "name": "FastQAExt", "url": "https://arxiv.org/abs/1703.04816", "min_date": "2017-03-14", "max_date": "2017-06-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.849, "maxval": 70.849, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making Neural QA as Simple as Possible but not Simpler", "label": "FastQAExt", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2016-12-29", "value": 68.436, "name": "FastQA", "url": "https://arxiv.org/abs/1703.04816", "min_date": "2017-03-14", "max_date": "2017-06-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.436, "maxval": 68.436, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making Neural QA as Simple as Possible but not Simpler", "label": "FastQA", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-02-24", "value": 73.744, "name": "BiDAF (ensemble)", "url": "https://arxiv.org/abs/1611.01603", "min_date": "2016-11-05", "max_date": "2017-02-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.744, "maxval": 73.744, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Bidirectional Attention Flow for Machine Comprehension", "label": "BiDAF (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-03-08", "value": 76.922, "name": "r-net (ensemble)", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2017/05/r-net.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 76.922, "maxval": 76.922, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "r-net (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-03-08", "value": 74.614, "name": "r-net (single model)", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2017/05/r-net.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.614, "maxval": 74.614, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "r-net (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-03-31", "value": 70.733, "name": "Document Reader (single model)", "url": "https://arxiv.org/abs/1704.00051", "min_date": "2017-03-31", "max_date": "2017-04-28", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.733, "maxval": 70.733, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Reading Wikipedia to Answer Open-Domain Questions", "label": "Document Reader (single m...", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-04-20", "value": 73.723, "name": "SEDT+BiDAF (ensemble)", "url": "https://arxiv.org/abs/1703.00572", "min_date": "2017-03-02", "max_date": "2017-04-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.723, "maxval": 73.723, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Structural Embedding of Syntactic Trees for Machine Comprehension", "label": "SEDT+BiDAF (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-04-20", "value": 68.478, "name": "SEDT+BiDAF (single model)", "url": "https://arxiv.org/abs/1703.00572", "min_date": "2017-03-02", "max_date": "2017-04-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 68.478, "maxval": 68.478, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Structural Embedding of Syntactic Trees for Machine Comprehension", "label": "SEDT+BiDAF (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-04-24", "value": 70.639, "name": "Ruminating Reader (single model)", "url": "https://arxiv.org/abs/1704.07415", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.639, "maxval": 70.639, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Ruminating Reader: Reasoning with Gated Multi-Hop Attention", "label": "Ruminating Reader (single...", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-05-08", "value": 73.754, "name": "Mnemonic reader (ensemble)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.754, "maxval": 73.754, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "Mnemonic reader (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-05-08", "value": 69.863, "name": "Mnemonic reader (single model)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.863, "maxval": 69.863, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "Mnemonic reader (single m...", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-05-31", "value": 73.01, "name": "jNet (ensemble)", "url": "https://arxiv.org/abs/1703.04617", "min_date": "2017-05-01", "max_date": "2017-03-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.01, "maxval": 73.01, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploring Question Understanding and Adaptation in Neural-Network-Based Question Answering", "label": "jNet (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-05-31", "value": 70.849, "name": "RaSoR (single model)", "url": "https://arxiv.org/abs/1611.01436", "min_date": "2017-05-01", "max_date": "2017-03-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.849, "maxval": 70.849, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning Recurrent Span Representations for Extractive Question Answering", "label": "RaSoR (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-05-31", "value": 70.607, "name": "jNet (single model)", "url": "https://arxiv.org/abs/1703.04617", "min_date": "2017-05-01", "max_date": "2017-03-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.607, "maxval": 70.607, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploring Question Understanding and Adaptation in Neural-Network-Based Question Answering", "label": "jNet (single model)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-06-20", "value": 73.4, "name": "ReasoNet ensemble", "url": "https://arxiv.org/abs/1609.05284v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 73.4, "maxval": 73.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReasoNet: Learning to Stop Reading in Machine Comprehension", "label": "ReasoNet ensemble", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-07-28", "value": 75.37, "name": "MEMEN", "url": "https://arxiv.org/abs/1707.09098v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.37, "maxval": 75.37, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "MEMEN: Multi-layer Embedding with Memory Networks for Machine Comprehension", "label": "MEMEN", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-08-16", "value": 78.706, "name": "DCN+ (ensemble)", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.706, "maxval": 78.706, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "DCN+ (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-08-21", "value": 77.678, "name": "RMR (ensemble)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.678, "maxval": 77.678, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "RMR (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2017-09-20", "value": 78.842, "name": "AIR-FusionNet (ensemble)", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.842, "maxval": 78.842, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "AIR-FusionNet (ensemble)", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2018-09-26", "value": 85.954, "name": "nlnet ensemble", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 85.954, "maxval": 85.954, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "nlnet ensemble", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2018-10-11", "value": 87.4, "name": "BERT+TriviaQA", "url": "https://arxiv.org/abs/1810.04805", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 87.4, "maxval": 87.4, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "label": "BERT+TriviaQA", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2019-03-05", "value": 86.673, "name": "BERT+ngram+sst", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 86.673, "maxval": 86.673, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "BERT+ngram+sst", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" }, { "date": "2019-09-25", "value": 89.7, "name": "XLNet", "url": "https://arxiv.org/abs/1906.08237", "min_date": "2019-06-19", "max_date": "2020-01-02", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.7, "maxval": 89.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "XLNet: Generalized Autoregressive Pretraining for Language Understanding", "label": "XLNet", "metric": "Metric(Stanford Question Answering Dataset EM test) SOLVED" } ], "name": "Stanford Question Answering Dataset EM test", "solved": true, "url": "https://stanford-qa.com/", "notes": "", "scale": "Score", "target": 82.304, "target_source": "http://arxiv.org/abs/1606.05250", "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L324" }, { "measures": [ { "date": "2016-11-04", "value": 80.383, "name": "Dynamic Coattention Networks (ensemble)", "url": "https://arxiv.org/abs/1611.01604v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 80.383, "maxval": 80.383, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dynamic Coattention Networks For Question Answering", "label": "Dynamic Coattention Netwo...", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-11-04", "value": 75.896, "name": "Dynamic Coattention Networks (single model)", "url": "https://arxiv.org/abs/1611.01604v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 75.896, "maxval": 75.896, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Dynamic Coattention Networks For Question Answering", "label": "Dynamic Coattention Netwo...", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-11-07", "value": 77.022, "name": "Match-LSTM+Ans-Ptr", "url": "https://arxiv.org/abs/1608.07905v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.022, "maxval": 77.022, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Machine Comprehension Using Match-LSTM and Answer Pointer", "label": "Match-LSTM+Ans-Ptr", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-11-29", "value": 77.971, "name": "BiDAF (single model)", "url": "https://arxiv.org/abs/1611.01603", "min_date": "2016-11-05", "max_date": "2017-02-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.971, "maxval": 77.971, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Bidirectional Attention Flow for Machine Comprehension", "label": "BiDAF (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-12-13", "value": 81.257, "name": "MPM (ensemble)", "url": "https://arxiv.org/abs/1612.04211", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.257, "maxval": 81.257, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multi-Perspective Context Matching for Machine Comprehension", "label": "MPM (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-12-13", "value": 78.784, "name": "MPM (single model)", "url": "https://arxiv.org/abs/1612.04211", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.784, "maxval": 78.784, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Multi-Perspective Context Matching for Machine Comprehension", "label": "MPM (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-12-29", "value": 78.857, "name": "FastQAExt", "url": "https://arxiv.org/abs/1703.04816", "min_date": "2017-03-14", "max_date": "2017-06-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.857, "maxval": 78.857, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making Neural QA as Simple as Possible but not Simpler", "label": "FastQAExt", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2016-12-29", "value": 77.07, "name": "FastQA", "url": "https://arxiv.org/abs/1703.04816", "min_date": "2017-03-14", "max_date": "2017-06-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.07, "maxval": 77.07, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Making Neural QA as Simple as Possible but not Simpler", "label": "FastQA", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-02-24", "value": 81.525, "name": "BiDAF (ensemble)", "url": "https://arxiv.org/abs/1611.01603", "min_date": "2016-11-05", "max_date": "2017-02-24", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.525, "maxval": 81.525, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Bidirectional Attention Flow for Machine Comprehension", "label": "BiDAF (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-03-08", "value": 84.006, "name": "r-net (ensemble)", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2017/05/r-net.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 84.006, "maxval": 84.006, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "r-net (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-03-08", "value": 82.458, "name": "r-net (single model)", "url": "https://www.microsoft.com/en-us/research/wp-content/uploads/2017/05/r-net.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 82.458, "maxval": 82.458, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": null, "label": "r-net (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-03-31", "value": 79.353, "name": "Document Reader (single model)", "url": "https://arxiv.org/abs/1704.00051", "min_date": "2017-03-31", "max_date": "2017-04-28", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.353, "maxval": 79.353, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Reading Wikipedia to Answer Open-Domain Questions", "label": "Document Reader (single m...", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-04-20", "value": 81.53, "name": "SEDT+BiDAF (ensemble)", "url": "https://arxiv.org/abs/1703.00572", "min_date": "2017-03-02", "max_date": "2017-04-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.53, "maxval": 81.53, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Structural Embedding of Syntactic Trees for Machine Comprehension", "label": "SEDT+BiDAF (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-04-20", "value": 77.971, "name": "SEDT+BiDAF (single model)", "url": "https://arxiv.org/abs/1703.00572", "min_date": "2017-03-02", "max_date": "2017-04-20", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 77.971, "maxval": 77.971, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Structural Embedding of Syntactic Trees for Machine Comprehension", "label": "SEDT+BiDAF (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-04-24", "value": 79.821, "name": "Ruminating Reader (single model)", "url": "https://arxiv.org/abs/1704.07415", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.821, "maxval": 79.821, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Ruminating Reader: Reasoning with Gated Multi-Hop Attention", "label": "Ruminating Reader (single...", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-05-08", "value": 81.863, "name": "Mnemonic reader (ensemble)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.863, "maxval": 81.863, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "Mnemonic reader (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-05-08", "value": 79.207, "name": "Mnemonic reader (single model)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.207, "maxval": 79.207, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "Mnemonic reader (single m...", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-05-31", "value": 81.517, "name": "jNet (ensemble)", "url": "https://arxiv.org/abs/1703.04617", "min_date": "2017-05-01", "max_date": "2017-03-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 81.517, "maxval": 81.517, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploring Question Understanding and Adaptation in Neural-Network-Based Question Answering", "label": "jNet (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-05-31", "value": 79.456, "name": "jNet (single model)", "url": "https://arxiv.org/abs/1703.04617", "min_date": "2017-05-01", "max_date": "2017-03-25", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.456, "maxval": 79.456, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Exploring Question Understanding and Adaptation in Neural-Network-Based Question Answering", "label": "jNet (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-05-31", "value": 78.741, "name": "RaSoR (single model)", "url": "https://arxiv.org/abs/1611.01436", "min_date": "2017-05-01", "max_date": "2017-03-17", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 78.741, "maxval": 78.741, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Learning Recurrent Span Representations for Extractive Question Answering", "label": "RaSoR (single model)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-06-20", "value": 82.9, "name": "ReasoNet ensemble", "url": "https://arxiv.org/abs/1609.05284v3", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 82.9, "maxval": 82.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "ReasoNet: Learning to Stop Reading in Machine Comprehension", "label": "ReasoNet ensemble", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-07-28", "value": 82.66, "name": "MEMEN", "url": "https://arxiv.org/abs/1707.09098v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 82.66, "maxval": 82.66, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "MEMEN: Multi-layer Embedding with Memory Networks for Machine Comprehension", "label": "MEMEN", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-08-16", "value": 85.619, "name": "DCN+ (ensemble)", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 85.619, "maxval": 85.619, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "DCN+ (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-08-21", "value": 84.888, "name": "RMR (ensemble)", "url": "https://arxiv.org/abs/1705.02798", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 84.888, "maxval": 84.888, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Mnemonic Reader for Machine Comprehension", "label": "RMR (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2017-09-20", "value": 85.936, "name": "AIR-FusionNet (ensemble)", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 85.936, "maxval": 85.936, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "AIR-FusionNet (ensemble)", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2018-09-26", "value": 91.677, "name": "nlnet ensemble", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 91.677, "maxval": 91.677, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "nlnet ensemble", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2018-10-11", "value": 93.2, "name": "BERT+TriviaQA", "url": "https://arxiv.org/abs/1810.04805", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 93.2, "maxval": 93.2, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "label": "BERT+TriviaQA", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2019-03-05", "value": 89.147, "name": "BERT+ngram+sst", "url": "https://rajpurkar.github.io/SQuAD-explorer/", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.147, "maxval": 89.147, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The Stanford Question Answering Dataset", "label": "BERT+ngram+sst", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" }, { "date": "2019-09-25", "value": 95.1, "name": "XLNet", "url": "https://arxiv.org/abs/1906.08237", "min_date": "2019-06-19", "max_date": "2020-01-02", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 95.1, "maxval": 95.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "XLNet: Generalized Autoregressive Pretraining for Language Understanding", "label": "XLNet", "metric": "Metric(Stanford Question Answering Dataset F1 test) SOLVED" } ], "name": "Stanford Question Answering Dataset F1 test", "solved": true, "url": "https://stanford-qa.com/", "notes": "", "scale": "Score", "target": 91.221, "target_source": "http://arxiv.org/abs/1606.05250", "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L323" }, { "measures": [ { "date": "2016-03-04", "value": 71.0, "name": "AS reader (greedy)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.0, "maxval": 71.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (greedy)", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-03-04", "value": 70.6, "name": "AS reader (avg)", "url": "https://arxiv.org/abs/1603.01547v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 70.6, "maxval": 70.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Text Understanding with the Attention Sum Reader Network", "label": "AS reader (avg)", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-06-05", "value": 71.9, "name": "GA reader", "url": "https://arxiv.org/abs/1606.01549v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.9, "maxval": 71.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA reader", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-06-07", "value": 72.0, "name": "AIA", "url": "https://arxiv.org/abs/1606.02245v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.0, "maxval": 72.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Iterative Alternating Neural Attention for Machine Reading", "label": "AIA", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-06-07", "value": 71.0, "name": "AIA", "url": "https://arxiv.org/abs/1606.02245v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.0, "maxval": 71.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Iterative Alternating Neural Attention for Machine Reading", "label": "AIA", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-06-07", "value": 69.7, "name": "EpiReader", "url": "https://arxiv.org/abs/1606.02270", "min_date": "2016-06-07", "max_date": "2016-06-10", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.7, "maxval": 69.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Natural Language Comprehension with the EpiReader", "label": "EpiReader", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-07-17", "value": 69.2, "name": "CAS reader", "url": "https://arxiv.org/abs/1607.02250v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.2, "maxval": 69.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Consensus Attention-based Neural Networks for Chinese Reading Comprehension", "label": "CAS reader", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-08-04", "value": 72.0, "name": "AoA reader", "url": "https://arxiv.org/abs/1607.04423", "min_date": "2016-07-15", "max_date": "2017-06-06", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.0, "maxval": 72.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention-over-Attention Neural Networks for Reading Comprehension", "label": "AoA reader", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-12-01", "value": 74.9, "name": "GA +feature, fix L(w)", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 74.9, "maxval": 74.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "GA +feature, fix L(w)", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2016-12-01", "value": 73.2, "name": "NSE", "url": "https://arxiv.org/abs/1606.01549v2", "min_date": "2016-07-04", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1607.04315", "src_name": "Neural Semantic Encoders", "uncertainty": 0, "minval": 73.2, "maxval": 73.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Gated-Attention Readers for Text Comprehension", "label": "NSE", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2017-10-07", "value": 72.2, "name": "DIM Reader", "url": "http://www.cips-cl.org/static/anthology/CCL-2017/CCL-17-075.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 72.2, "maxval": 72.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "DIM Reader: Dual Interaction Mode for Machine Comprehension", "label": "DIM Reader", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2018-10-05", "value": 79.4, "name": "AttSum-Feat+L2", "url": "https://arxiv.org/abs/1810.02891", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 79.4, "maxval": 79.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Entity Tracking Improves Cloze-style Reading Comprehension", "label": "AttSum-Feat+L2", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" }, { "date": "2019-02-14", "value": 89.05, "name": "GPT2 (zero shot)", "url": "https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 89.05, "maxval": 89.05, "opensource": false, "not_directly_comparable": true, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Language Models are Unsupervised Multitask Learners", "label": "GPT2 (zero shot)", "metric": "Metric(bAbi Children's Book comprehension CBtest NE) SOLVED" } ], "name": "bAbi Children's Book comprehension CBtest NE", "solved": true, "url": "http://fb.ai/babi", "notes": "", "scale": "Percentage correct", "target": 81.6, "target_source": "https://arxiv.org/abs/1511.02301", "changeable": false, "graphed": true, "parent": "Problem(Language comprehension and question-answering)", "target_label": null, "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L239" } ], "solved": false, "url": null }, { "name": "Translation between human langauges", "attributes": [ "agi", "language" ], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [ { "date": "2014-02-24", "value": 37, "name": "PBMT", "url": "http://www.anthology.aclweb.org/W/W14/W14-33.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 37, "maxval": 37, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Edinburgh\u2019s phrase-based machine translation systems for WMT-14", "label": "PBMT", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2014-09-01", "value": 36.15, "name": "RNN-search50*", "url": "https://arxiv.org/abs/1409.0473", "min_date": "2014-09-01", "max_date": "2016-05-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36.15, "maxval": 36.15, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Machine Translation by Jointly Learning to Align and Translate", "label": "RNN-search50*", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2014-09-10", "value": 36.5, "name": "SMT+LSTM5", "url": "https://arxiv.org/abs/1409.3215v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 36.5, "maxval": 36.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Sequence to Sequence Learning with Neural Networks", "label": "SMT+LSTM5", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2014-09-10", "value": 34.81, "name": "LSTM", "url": "https://arxiv.org/abs/1409.3215v1", "min_date": null, "max_date": null, "algorithm_src_url": "http://www.bioinf.jku.at/publications/older/2604.pdf", "src_name": "http://www.bioinf.jku.at/publications/older/2604.pdf", "uncertainty": 0, "minval": 34.81, "maxval": 34.81, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Sequence to Sequence Learning with Neural Networks", "label": "LSTM", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2014-10-30", "value": 37.5, "name": "LSTM6 + PosUnk", "url": "https://arxiv.org/abs/1410.8206", "min_date": "2014-10-30", "max_date": "2015-05-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 37.5, "maxval": 37.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Addressing the Rare Word Problem in Neural Machine Translation", "label": "LSTM6 + PosUnk", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2015-07-11", "value": 28.45, "name": "RNNsearch-50", "url": "https://arxiv.org/abs/1409.0473", "min_date": "2014-09-01", "max_date": "2016-05-19", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.45, "maxval": 28.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Machine Translation by Jointly Learning to Align and Translate", "label": "RNNsearch-50", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2016-07-23", "value": 39.2, "name": "Deep-Att + PosUnk", "url": "https://arxiv.org/abs/1606.04199", "min_date": "2016-06-14", "max_date": "2016-07-23", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 39.2, "maxval": 39.2, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Recurrent Models with Fast-Forward Connections for Neural Machine Translation", "label": "Deep-Att + PosUnk", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2016-09-26", "value": 39.92, "name": "GNMT+RL", "url": "https://arxiv.org/abs/1609.08144", "min_date": "2016-09-26", "max_date": "2016-10-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 39.92, "maxval": 39.92, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation", "label": "GNMT+RL", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2017-01-23", "value": 40.56, "name": "MoE 2048", "url": "https://arxiv.org/abs/1701.06538", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 40.56, "maxval": 40.56, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer", "label": "MoE 2048", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2017-05-12", "value": 41.29, "name": "ConvS2S (ensemble)", "url": "https://arxiv.org/abs/1705.03122v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 41.29, "maxval": 41.29, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Convolutional Sequence to Sequence Learning", "label": "ConvS2S (ensemble)", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2017-05-12", "value": 40.46, "name": "ConvS2S", "url": "https://arxiv.org/abs/1705.03122v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 40.46, "maxval": 40.46, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Convolutional Sequence to Sequence Learning", "label": "ConvS2S", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2017-06-21", "value": 41, "name": "Transformer (big)", "url": "https://arxiv.org/abs/1706.03762", "min_date": "2017-06-12", "max_date": "2017-06-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 41, "maxval": 41, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention Is All You Need", "label": "Transformer (big)", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" }, { "date": "2017-06-21", "value": 38.1, "name": "Transformer", "url": "https://arxiv.org/abs/1706.03762", "min_date": "2017-06-12", "max_date": "2017-06-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 38.1, "maxval": 38.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention Is All You Need", "label": "Transformer", "metric": "Metric(news-test-2014 En-Fr BLEU) not solved" } ], "name": "news-test-2014 En-Fr BLEU", "solved": false, "url": "http://aclweb.org/anthology/P/P02/P02-1040.pdf", "notes": "", "scale": "BLEU score", "target": 50, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Translation between human langauges)", "target_label": "Identical to professional human translations", "axis_label": "BLEU score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L378" }, { "measures": [ { "date": "2014-02-24", "value": 20.7, "name": "PBMT", "url": "http://www.anthology.aclweb.org/W/W14/W14-33.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20.7, "maxval": 20.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Edinburgh\u2019s phrase-based machine translation systems for WMT-14", "label": "PBMT", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2016-07-14", "value": 17.93, "name": "NSE-NSE", "url": "https://arxiv.org/abs/1607.04315v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 17.93, "maxval": 17.93, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Semantic Encoders", "label": "NSE-NSE", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2016-07-23", "value": 20.7, "name": "Deep-Att", "url": "https://arxiv.org/abs/1606.04199", "min_date": "2016-06-14", "max_date": "2016-07-23", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 20.7, "maxval": 20.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Deep Recurrent Models with Fast-Forward Connections for Neural Machine Translation", "label": "Deep-Att", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2016-09-26", "value": 26.3, "name": "GNMT+RL", "url": "https://arxiv.org/abs/1609.08144", "min_date": "2016-09-26", "max_date": "2016-10-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.3, "maxval": 26.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation", "label": "GNMT+RL", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-01-06", "value": 23.75, "name": "ByteNet", "url": "https://arxiv.org/abs/1610.10099", "min_date": "2016-10-31", "max_date": "2017-03-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23.75, "maxval": 23.75, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Machine Translation in Linear Time", "label": "ByteNet", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-01-23", "value": 26.03, "name": "MoE 2048", "url": "https://arxiv.org/abs/1701.06538", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.03, "maxval": 26.03, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer", "label": "MoE 2048", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-05-12", "value": 26.36, "name": "ConvS2S (ensemble)", "url": "https://arxiv.org/abs/1705.03122v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.36, "maxval": 26.36, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Convolutional Sequence to Sequence Learning", "label": "ConvS2S (ensemble)", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-05-12", "value": 25.16, "name": "ConvS2S", "url": "https://arxiv.org/abs/1705.03122v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 25.16, "maxval": 25.16, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Convolutional Sequence to Sequence Learning", "label": "ConvS2S", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-06-12", "value": 26.1, "name": "SliceNet", "url": "https://arxiv.org/abs/1706.03059", "min_date": "2017-06-09", "max_date": "2017-06-16", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.1, "maxval": 26.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Depthwise Separable Convolutions for Neural Machine Translation", "label": "SliceNet", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-06-21", "value": 28.4, "name": "Transformer (big)", "url": "https://arxiv.org/abs/1706.03762", "min_date": "2017-06-12", "max_date": "2017-06-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.4, "maxval": 28.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention Is All You Need", "label": "Transformer (big)", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-06-21", "value": 27.3, "name": "Transformer", "url": "https://arxiv.org/abs/1706.03762", "min_date": "2017-06-12", "max_date": "2017-06-30", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 27.3, "maxval": 27.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Attention Is All You Need", "label": "Transformer", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2017-09-25", "value": 27.9, "name": "Transformer+BR-CSGAN", "url": "https://arxiv.org/abs/1703.04887", "min_date": "2017-03-15", "max_date": "2018-04-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 27.9, "maxval": 27.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Improving Neural Machine Translation with Conditional Sequence Generative Adversarial Nets", "label": "Transformer+BR-CSGAN", "metric": "Metric(news-test-2014 En-De BLEU) not solved" }, { "date": "2018-09-18", "value": 28.36, "name": "FRAGE", "url": "https://arxiv.org/abs/1809.06858", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.36, "maxval": 28.36, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "FRAGE: Frequency-Agnostic Word Representation", "label": "FRAGE", "metric": "Metric(news-test-2014 En-De BLEU) not solved" } ], "name": "news-test-2014 En-De BLEU", "solved": false, "url": "http://aclweb.org/anthology/P/P02/P02-1040.pdf", "notes": "", "scale": "BLEU score", "target": 50, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Translation between human langauges)", "target_label": "Identical to professional human translations", "axis_label": "BLEU score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L381" }, { "measures": [ { "date": "2015-09-17", "value": 24.1, "name": "S2Tree+5gram NPLM", "url": "http://aclweb.org/anthology/W15-3024.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 24.1, "maxval": 24.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Edinburgh's Syntax-Based Systems at WMT 2015", "label": "S2Tree+5gram NPLM", "metric": "Metric(news-test-2015 En-De BLEU) not solved" }, { "date": "2016-03-19", "value": 23.45, "name": "Enc-Dec Att (char)", "url": "https://arxiv.org/abs/1603.06147v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 23.45, "maxval": 23.45, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Character-level Decoder without Explicit Segmentation for Neural Machine Translation", "label": "Enc-Dec Att (char)", "metric": "Metric(news-test-2015 En-De BLEU) not solved" }, { "date": "2016-03-19", "value": 21.72, "name": "Enc-Dec Att (BPE)", "url": "https://arxiv.org/abs/1603.06147v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 21.72, "maxval": 21.72, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Character-level Decoder without Explicit Segmentation for Neural Machine Translation", "label": "Enc-Dec Att (BPE)", "metric": "Metric(news-test-2015 En-De BLEU) not solved" }, { "date": "2017-01-06", "value": 26.26, "name": "ByteNet", "url": "https://arxiv.org/abs/1610.10099", "min_date": "2016-10-31", "max_date": "2017-03-15", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 26.26, "maxval": 26.26, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Neural Machine Translation in Linear Time", "label": "ByteNet", "metric": "Metric(news-test-2015 En-De BLEU) not solved" } ], "name": "news-test-2015 En-De BLEU", "solved": false, "url": null, "notes": "", "scale": "BLEU score", "target": 50, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Translation between human langauges)", "target_label": "Identical to professional human translations", "axis_label": "BLEU score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L359" }, { "measures": [ { "date": "2016-07-11", "value": 28.9, "name": "GRU BPE90k", "url": "http://www.statmt.org/wmt16/pdf/W16-2320.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 28.9, "maxval": 28.9, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "The QT21/HimL Combined Machine Translation System", "label": "GRU BPE90k", "metric": "Metric(news-test-2016 En-Ro BLEU) not solved" }, { "date": "2017-05-12", "value": 29.88, "name": "ConvS2S BPE40k", "url": "https://arxiv.org/abs/1705.03122v2", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 29.88, "maxval": 29.88, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Convolutional Sequence to Sequence Learning", "label": "ConvS2S BPE40k", "metric": "Metric(news-test-2016 En-Ro BLEU) not solved" } ], "name": "news-test-2016 En-Ro BLEU", "solved": false, "url": "http://www.statmt.org/wmt16/book.pdf", "notes": "", "scale": "BLEU score", "target": 50, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Translation between human langauges)", "target_label": "Identical to professional human translations", "axis_label": "BLEU score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L384" }, { "measures": [ { "date": "2017-09-25", "value": 22.89, "name": "Transformer+BR-CSGAN", "url": "https://arxiv.org/abs/1703.04887", "min_date": "2017-03-15", "max_date": "2018-04-08", "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 22.89, "maxval": 22.89, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Improving Neural Machine Translation with Conditional Sequence Generative Adversarial Nets", "label": "Transformer+BR-CSGAN", "metric": "Metric(LDC En-De BLEU) not solved" } ], "name": "LDC En-De BLEU", "solved": false, "url": "https://arxiv.org/pdf/1703.04887.pdf", "notes": "", "scale": "BLEU score", "target": 50, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Translation between human langauges)", "target_label": "Identical to professional human translations", "axis_label": "BLEU score", "data_path": "/home/pde/aip/data/language.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/language.py#L376" } ], "solved": false, "url": null }, { "name": "Read a scientific or technical paper, and comprehend its contents", "attributes": [ "language", "world-modelling", "super" ], "subproblems": [ "Extract major numerical results or progress claims from a STEM paper" ], "superproblems": [ "Solve vaguely or under-constrained technical problems" ], "metrics": [], "solved": false, "url": null }, { "name": "Extract major numerical results or progress claims from a STEM paper", "attributes": [ "language", "world-modelling", "agi" ], "subproblems": [], "superproblems": [ "Read a scientific or technical paper, and comprehend its contents" ], "metrics": [ { "measures": [], "name": "Automatically find new relevant ML results on arXiv", "solved": false, "url": null, "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Extract major numerical results or progress claims from a STEM paper)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L12" } ], "solved": false, "url": null, "notes": "\nThis metric is the ability to automatically update the ipython Notebook you are reading by spotting results in pdfs uploaded to arxiv.org.\nPull requests demonstrating solutions are welcome :)\n" }, { "name": "Given an arbitrary technical problem, solve it as well as a typical professional in that field", "attributes": [ "language", "world-modelling" ], "subproblems": [ "Writing software from specifications", "Solve vaguely or under-constrained technical problems", "Solve technical problems with clear constraints (proofs, circuit design, aerofoil design, etc)" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Writing software from specifications", "attributes": [], "subproblems": [], "superproblems": [ "Given an arbitrary technical problem, solve it as well as a typical professional in that field" ], "metrics": [ { "measures": [], "name": "Card2Code", "solved": false, "url": "https://github.com/deepmind/card2code", "notes": "", "scale": "Percentage correct", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Writing software from specifications)", "target_label": null, "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L22" } ], "solved": false, "url": null }, { "name": "Solve vaguely or under-constrained technical problems", "attributes": [], "subproblems": [ "Read a scientific or technical paper, and comprehend its contents", "Write computer programs from specifications", "Answering Science Exam Questions" ], "superproblems": [ "Given an arbitrary technical problem, solve it as well as a typical professional in that field" ], "metrics": [], "solved": false, "url": null }, { "name": "Solve technical problems with clear constraints (proofs, circuit design, aerofoil design, etc)", "attributes": [], "subproblems": [ "Given examples of proofs, find correct proofs of simple mathematical theorems", "Given desired circuit characteristics, and many examples, design new circuits to spec" ], "superproblems": [ "Given an arbitrary technical problem, solve it as well as a typical professional in that field" ], "metrics": [], "solved": false, "url": null }, { "name": "Given examples of proofs, find correct proofs of simple mathematical theorems", "attributes": [ "agi", "math" ], "subproblems": [], "superproblems": [ "Solve technical problems with clear constraints (proofs, circuit design, aerofoil design, etc)" ], "metrics": [ { "measures": [], "name": "HolStep", "solved": false, "url": "https://arxiv.org/abs/1703.00426", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Given examples of proofs, find correct proofs of simple mathematical theorems)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L39" } ], "solved": false, "url": null }, { "name": "Given desired circuit characteristics, and many examples, design new circuits to spec", "attributes": [ "agi", "math" ], "subproblems": [], "superproblems": [ "Solve technical problems with clear constraints (proofs, circuit design, aerofoil design, etc)" ], "metrics": [], "solved": false, "url": null }, { "name": "Write computer programs from specifications", "attributes": [], "subproblems": [ "Parse and implement complex conditional expressions" ], "superproblems": [ "Solve vaguely or under-constrained technical problems" ], "metrics": [ { "measures": [ { "date": "2016-03-22", "value": 4.8, "name": "LPN", "url": "https://arxiv.org/abs/1603.06744v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 4.8, "maxval": 4.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Latent Predictor Networks for Code Generation", "label": "LPN", "metric": "Metric(Card2Code MTG accuracy) not solved" } ], "name": "Card2Code MTG accuracy", "solved": false, "url": "https://github.com/deepmind/card2code", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Write computer programs from specifications)", "target_label": "Bug-free card implementation", "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/stem.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L53" }, { "measures": [ { "date": "2016-03-22", "value": 6.1, "name": "LPN", "url": "https://arxiv.org/abs/1603.06744v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 6.1, "maxval": 6.1, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Latent Predictor Networks for Code Generation", "label": "LPN", "metric": "Metric(Card2Code Hearthstone accuracy) not solved" }, { "date": "2017-04-06", "value": 16.7, "name": "SNM -frontier embed", "url": "https://arxiv.org/abs/1704.01696v1", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 16.7, "maxval": 16.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Syntactic Neural Model for General-Purpose Code Generation", "label": "SNM -frontier embed", "metric": "Metric(Card2Code Hearthstone accuracy) not solved" }, { "date": "2017-04-06", "value": 13.6, "name": "Seq2Tree-Unk", "url": "https://arxiv.org/abs/1704.01696v1", "min_date": "2016-01-06", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1601.01280v1", "src_name": "Language to Logical Form with Neural Attention", "uncertainty": 0, "minval": 13.6, "maxval": 13.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Syntactic Neural Model for General-Purpose Code Generation", "label": "Seq2Tree-Unk", "metric": "Metric(Card2Code Hearthstone accuracy) not solved" }, { "date": "2017-04-06", "value": 1.5, "name": "NMT", "url": "https://arxiv.org/abs/1704.01696v1", "min_date": "2014-09-01", "max_date": null, "algorithm_src_url": "https://arxiv.org/abs/1409.0473v1", "src_name": "Neural Machine Translation by Jointly Learning to Align and Translate", "uncertainty": 0, "minval": 1.5, "maxval": 1.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "A Syntactic Neural Model for General-Purpose Code Generation", "label": "NMT", "metric": "Metric(Card2Code Hearthstone accuracy) not solved" } ], "name": "Card2Code Hearthstone accuracy", "solved": false, "url": "https://github.com/deepmind/card2code", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Write computer programs from specifications)", "target_label": "Bug-free card implementation", "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/stem.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L58" } ], "solved": false, "url": null }, { "name": "Parse and implement complex conditional expressions", "attributes": [], "subproblems": [], "superproblems": [ "Write computer programs from specifications" ], "metrics": [], "solved": false, "url": null }, { "name": "Answering Science Exam Questions", "attributes": [ "science", "qa" ], "subproblems": [], "superproblems": [ "Solve vaguely or under-constrained technical problems" ], "metrics": [ { "measures": [ { "date": "2016-02-12", "value": 71.3, "name": "Aristo (ALL)", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 71.3, "maxval": 71.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "Aristo (ALL)", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 60.7, "name": "PMI", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 60.7, "maxval": 60.7, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "PMI", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 60.6, "name": "IR", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 60.6, "maxval": 60.6, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "IR", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 55.4, "name": "SVM", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 55.4, "maxval": 55.4, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "SVM", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 54.3, "name": "RULE", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 54.3, "maxval": 54.3, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "RULE", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 47.5, "name": "Praline", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": "2015-11-17", "max_date": null, "algorithm_src_url": "http://aclweb.org/anthology/D15-1080", "src_name": "http://aclweb.org/anthology/D15-1080", "uncertainty": 0, "minval": 47.5, "maxval": 47.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "Praline", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-02-12", "value": 43.8, "name": "ILP", "url": "https://pdfs.semanticscholar.org/478b/4a5123bd5fda98bb35e6317d7f3555fec97d.pdf", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 43.8, "maxval": 43.8, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Combining Retrieval, Statistics, and Inference to Answer Elementary Science Questions", "label": "ILP", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-04-20", "value": 69.0, "name": "TableILP+IR+PMI", "url": "https://arxiv.org/abs/1604.06076", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 69.0, "maxval": 69.0, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Question Answering via Integer Programming over Semi-Structured Knowledge", "label": "TableILP+IR+PMI", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" }, { "date": "2016-04-20", "value": 61.5, "name": "TableILP", "url": "https://arxiv.org/abs/1604.06076", "min_date": null, "max_date": null, "algorithm_src_url": "", "src_name": null, "uncertainty": 0, "minval": 61.5, "maxval": 61.5, "opensource": false, "not_directly_comparable": false, "replicated_url": "", "long_label": false, "algorithms": [], "offset": null, "notes": "", "withdrawn": false, "papername": "Question Answering via Integer Programming over Semi-Structured Knowledge", "label": "TableILP", "metric": "Metric(NY Regents 4th Grade Science Exams) not solved" } ], "name": "NY Regents 4th Grade Science Exams", "solved": false, "url": "http://www.nysedregents.org/Grade4/Science/home.html", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": true, "parent": "Problem(Answering Science Exam Questions)", "target_label": "Perfect score", "axis_label": "Percentage correct", "data_path": "/home/pde/aip/data/stem.py", "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L77" }, { "measures": [], "name": "Elementery Non-Diagram Multiple Choice (NDMC) Science Exam accuracy", "solved": false, "url": "http://data.allenai.org/ai2-science-questions/", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Answering Science Exam Questions)", "target_label": "Perfect Score", "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L79" }, { "measures": [], "name": "Elementery Diagram Multiple Choice (DMC) Science Exam accuracy", "solved": false, "url": "http://data.allenai.org/ai2-science-questions/", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Answering Science Exam Questions)", "target_label": "Perfect Score", "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L82" }, { "measures": [], "name": "Middle School Non-Diagram Multiple Choice (NDMC) Science Exam accuracy", "solved": false, "url": "http://data.allenai.org/ai2-science-questions/", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Answering Science Exam Questions)", "target_label": "Perfect Score", "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L85" }, { "measures": [], "name": "Middle School Diagram Multiple Choice (DMC) Science Exam accuracy", "solved": false, "url": "http://data.allenai.org/ai2-science-questions/", "notes": "", "scale": "Percentage correct", "target": 100, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Answering Science Exam Questions)", "target_label": "Perfect Score", "axis_label": "Percentage correct", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master//home/pde/aip/data/stem.py#L88" } ], "solved": false, "url": null }, { "name": "Building systems that solve a wide range of diverse problems, rather than just specific ones", "attributes": [], "subproblems": [ "Transfer learning: apply relevant knowledge from a prior setting to a new slightly different one" ], "superproblems": [], "metrics": [ { "measures": [], "name": "Solve all other solved problems in this document, with a single system", "solved": false, "url": null, "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Building systems that solve a wide range of diverse problems, rather than just specific ones)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L2" } ], "solved": false, "url": null }, { "name": "Transfer learning: apply relevant knowledge from a prior setting to a new slightly different one", "attributes": [], "subproblems": [ "Transfer of learning within simple arcade game paradigms" ], "superproblems": [ "Building systems that solve a wide range of diverse problems, rather than just specific ones" ], "metrics": [], "solved": false, "url": null }, { "name": "Transfer of learning within simple arcade game paradigms", "attributes": [], "subproblems": [], "superproblems": [ "Transfer learning: apply relevant knowledge from a prior setting to a new slightly different one" ], "metrics": [], "solved": false, "url": null }, { "name": "One shot learning: ingest important truths from a single example", "attributes": [ "agi", "world-modelling" ], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Correctly identify when an answer to a classification problem is uncertain", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null, "notes": "Humans can usually tell when they don't know something. Present ML classifiers do not have this ability." }, { "name": "Learn a several tasks without undermining performance on a first task, avoiding catastrophic forgetting", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1612.00796" }, { "name": "Resistance to adversarial examples", "attributes": [ "safety", "agi", "security" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1312.6199", "notes": "\nWe know that humans have significant resistance to adversarial examples. Although methods like camouflage sometimes\nwork to fool us into thinking one thing is another, those\n" }, { "name": "Scalable supervision of a learning system", "attributes": [ "safety", "agi" ], "subproblems": [ "Cooperative inverse reinforcement learning of objective functions" ], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.06565" }, { "name": "Cooperative inverse reinforcement learning of objective functions", "attributes": [ "safety", "agi" ], "subproblems": [], "superproblems": [ "Scalable supervision of a learning system" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.03137", "notes": "This is tagged agi because most humans are able to learn ethics from their surrounding community" }, { "name": "Safe exploration", "attributes": [ "safety", "agi", "world-modelling" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.06565", "notes": "\nSometimes, even doing something once is catastrophic. In such situations, how can an RL agent or some other AI system\nlearn about the catastrophic consequences without even taking the action once? This is an ability that most humans acquire\nat some point between childhood and adolescence.\n" }, { "name": "Avoiding reward hacking", "attributes": [ "safety" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.06565", "notes": "\nHumans have only partial resistance to reward hacking.\nAddiction seems to be one failure to exhibit this resistance.\nAvoiding learning something because it might make us feel bad, or even building elaborate systems of self-deception, are also sometimes\nseen in humans. So this problem is not tagged \"agi\".\n" }, { "name": "Avoiding undesirable side effects", "attributes": [ "safety" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.06565", "nodes": "\nMany important constraints on good behaviour will not be explicitly\nencoded in goal specification, either because they are too hard to capture\nor simply because there are so many of them and they are hard to enumerate\n" }, { "name": "Function correctly in novel environments (robustness to distributional change)", "attributes": [ "safety", "agi" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": "https://arxiv.org/abs/1606.06565" }, { "name": "Know how to prevent an autonomous AI agent from reproducing itself an unbounded number of times", "attributes": [ "safety" ], "subproblems": [], "superproblems": [ "Know how to build general AI agents that will behave as expected" ], "metrics": [], "solved": false, "url": null }, { "name": "Know how to build general AI agents that will behave as expected", "attributes": [], "subproblems": [ "Resistance to adversarial examples", "Scalable supervision of a learning system", "Safe exploration", "Avoiding reward hacking", "Avoiding undesirable side effects", "Function correctly in novel environments (robustness to distributional change)", "Know how to prevent an autonomous AI agent from reproducing itself an unbounded number of times" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Detect security-related bugs in codebases", "attributes": [ "safety", "security", "unsafe" ], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Deploy automated defensive security tools to protect valuable systems", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null, "notes": "\nIt is clearly important is ensuring that the state of the art in defensive technology is deployed everywhere\nthat matters, including systems that perform important functions or have sensitive data on them (smartphones, for instance), and \nsystems that have signifcant computational resources. This \"Problem\" isn't \n" }, { "name": "Pedestrian, bicycle & obstacle detection", "attributes": [ "safety", "vision" ], "subproblems": [], "superproblems": [ "Image classification" ], "metrics": [ { "measures": [], "name": "Caltech Pedestrians USA", "solved": false, "url": "http://www.vision.caltech.edu/Image_Datasets/CaltechPedestrians/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Pedestrian, bicycle & obstacle detection)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L9" }, { "measures": [], "name": "INRIA persons", "solved": false, "url": "http://pascal.inrialpes.fr/data/human/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Pedestrian, bicycle & obstacle detection)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L11" }, { "measures": [], "name": "ETH Pedestrian", "solved": false, "url": "http://www.vision.ee.ethz.ch/~aess/dataset/", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Pedestrian, bicycle & obstacle detection)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L13" }, { "measures": [], "name": "TUD-Brussels Pedestrian", "solved": false, "url": "http://www.d2.mpi-inf.mpg.de/tud-brussels", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Pedestrian, bicycle & obstacle detection)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L15" }, { "measures": [], "name": "Damiler Pedestrian", "solved": false, "url": "http://www.gavrila.net/Datasets/Daimler_Pedestrian_Benchmark_D/Daimler_Mono_Ped__Detection_Be/daimler_mono_ped__detection_be.html", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Pedestrian, bicycle & obstacle detection)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L17" } ], "solved": false, "url": null }, { "name": "Modify arbitrary ML systems in order to be able to provide comprehensible human explanations of their decisions", "attributes": [], "subproblems": [ "Provide mathematical or technical explanations of decisions from classifiers" ], "superproblems": [], "metrics": [], "solved": false, "url": null }, { "name": "Provide mathematical or technical explanations of decisions from classifiers", "attributes": [], "subproblems": [], "superproblems": [ "Modify arbitrary ML systems in order to be able to provide comprehensible human explanations of their decisions" ], "metrics": [], "solved": false, "url": null, "notes": "\nProviding explanations with techniques such as monte carlo analysis may in general\nbe easier than providing robust ones in natural language (since those may or may not\nexist in all cases)\n" }, { "name": "Build systems which can recognise and avoid biases decision making", "attributes": [ "safety" ], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null, "notes": "\nLegally institutionalised protected categories represent only the most extreme and socially recognised\nforms of biased decisionmaking. Attentive human decision makers are sometime capable of recognising\nand avoiding many more subtle biases. This problem tracks AI systems' ability to do likewise.\n" }, { "name": "Train ML classifiers in a manner that corrects for the impact of omitted-variable bias on certain groups", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [], "name": "Adjust prediction models to have constant false-positive rates", "solved": true, "url": "https://arxiv.org/abs/1610.02413", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Train ML classifiers in a manner that corrects for the impact of omitted-variable bias on certain groups)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L18" }, { "measures": [], "name": "Adjust prediction models tos have constant false-positive and -negative rates", "solved": true, "url": "http://www.jmlr.org/proceedings/papers/v28/zemel13.pdf", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Train ML classifiers in a manner that corrects for the impact of omitted-variable bias on certain groups)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L19" } ], "solved": true, "url": null, "notes": "\nSeveral standards are available for avoiding classification biases.\n\nThey include holding false-positive / false adverse prediction rates constant across protected categories (which roughly maps \nto \"equal opportunity\"), holding both false-positive and false-negative rates equal (\"demographic parity\"), and ensuring\nthat the fraction of each protected group that receives a given prediction is constant across all groups \n(roughly equivalent to \"affirmative action\")." }, { "name": "Train machine learning systems on private user data, without transferring sensitive facts into the model", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [ { "measures": [], "name": "Federated Learning (distributed training with thresholded updates to models)", "solved": true, "url": "https://arxiv.org/abs/1602.05629", "notes": "", "scale": "Score", "target": null, "target_source": null, "changeable": false, "graphed": false, "parent": "Problem(Train machine learning systems on private user data, without transferring sensitive facts into the model)", "target_label": null, "axis_label": "Score", "data_path": null, "data_url": "https://github.com/AI-metrics/AI-metrics/edit/master/#L2" } ], "solved": false, "url": null }, { "name": "Fairness in machine learning towards people with a preference for privacy", "attributes": [], "subproblems": [], "superproblems": [], "metrics": [], "solved": false, "url": null, "notes": "\nPeople who care strongly about their own privacy take many measures to obfuscate their tracks through\ntechnological society, including using fictitious names, email addresses, etc in their routine dealings with\ncorporations, installing software to block or send inacurate data to online trackers. Like many other groups,\nthese people may be subject to unfairly adverse algorithmic decisionmaking. Treating them as a protected\ngroup will be more difficult, because they are in many respects harder to identify.\n" } ] }