{ "dataset": "world-models.io World Models Dataset", "version": "1.0.0", "generatedAt": "2026-07-17T14:47:10.932Z", "license": "CC0-1.0", "recordCount": 40, "records": [ { "id": "dreamer-v3", "name": "DreamerV3", "slug": "dreamer-v3", "company": "Google DeepMind", "lab": "Hafner et al.", "labSlug": "deepmind", "year": 2023, "category": "Model-Based RL", "subtype": "Latent Dynamics Model", "worldModelType": "Imagination-based dynamics model", "primaryDomain": "General RL", "status": "active", "architecture": "RSSM with discrete representations, symlog predictions, KL-balanced objective", "modality": "Visual + Proprioceptive", "trainingMethod": "Self-supervised world model learning + imagination-based policy optimization", "shortDescription": "A general algorithm for mastering diverse domains with fixed hyperparameters through world model learning.", "longDescription": "DreamerV3 learns a world model from experience and uses it to train an actor-critic policy entirely within imagination. It achieves superhuman performance across a wide range of domains (Atari, DMControl, Minecraft, and more), all with a single set of hyperparameters. The model uses discrete representations with symlog predictions and KL-balanced objectives to learn robust latent dynamics across vastly different environments.", "notableFeatures": [ "First to collect diamonds in Minecraft from scratch", "Single set of hyperparameters across all domains", "Symlog predictions for scale-invariant learning", "Discrete categorical latent space" ], "useCases": [ "Game playing", "Robot control", "Open-world navigation", "General decision-making" ], "strengths": [ "Fixed hyperparameters across domains", "Sample efficient", "Handles diverse reward structures", "Open-world capable" ], "limitations": [ "Computationally intensive", "Limited to single-agent scenarios", "Latent space interpretability challenges" ], "benchmarks": [ { "name": "Atari 100K", "score": "Superhuman on 50+ games", "metric": "Mean HNS", "numericValue": 2.01, "unit": "x human", "sourceUrl": "https://arxiv.org/abs/2301.04104" }, { "name": "DMControl", "score": "State-of-the-art", "metric": "Mean Return", "numericValue": 901, "unit": "avg return", "sourceUrl": "https://arxiv.org/abs/2301.04104" }, { "name": "Minecraft Diamonds", "score": "First to collect diamonds", "metric": "Success", "numericValue": 100, "unit": "% (first ever)", "sourceUrl": "https://arxiv.org/abs/2301.04104" } ], "paperUrl": "https://arxiv.org/abs/2301.04104", "websiteUrl": null, "tags": [ "model-based-rl", "latent-dynamics", "imagination", "general-purpose" ], "featured": true, "relatedModels": [ "planet", "dreamer-v2", "rssm", "muzero" ], "relatedPapers": [ "dreamerv3-paper", "dreamerv2-paper", "planet-paper" ], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Hafner et al., 2023. Mastering Diverse Domains through World Models. arXiv:2301.04104" ], "lastUpdated": "2026-03-10" }, { "id": "dreamer-v2", "name": "DreamerV2", "slug": "dreamer-v2", "company": "Google", "lab": "Hafner et al.", "labSlug": "deepmind", "year": 2021, "category": "Model-Based RL", "subtype": "Latent Dynamics Model", "worldModelType": "Imagination-based dynamics model", "primaryDomain": "Atari / Control", "status": "foundational", "architecture": "RSSM with categorical discrete latent variables", "modality": "Visual", "trainingMethod": "World model learning with discrete latent representations + imagination-based RL", "shortDescription": "The first model-based agent to achieve human-level performance on the Atari benchmark using discrete world model representations.", "longDescription": "DreamerV2 introduced discrete latent representations to world model learning, achieving human-level performance on the Atari 200M benchmark, a first for model-based methods. It demonstrated that discrete categorical variables can more effectively capture the multi-modal structure of complex environments than continuous representations.", "notableFeatures": [ "First model-based method to reach human-level Atari", "Discrete categorical latent representations", "KL balancing for stable training" ], "useCases": [ "Atari games", "Continuous control", "Sample-efficient learning" ], "strengths": [ "Human-level Atari", "Discrete representations", "More robust than V1", "Sample efficient" ], "limitations": [ "Per-domain hyperparameter tuning still needed", "Superseded by DreamerV3" ], "benchmarks": [ { "name": "Atari 200M", "score": "Human-level (first model-based)", "metric": "Mean HNS", "numericValue": 1, "unit": "x human", "sourceUrl": "https://arxiv.org/abs/2010.02193" } ], "paperUrl": "https://arxiv.org/abs/2010.02193", "websiteUrl": null, "tags": [ "model-based-rl", "latent-dynamics", "discrete", "atari" ], "featured": false, "relatedModels": [ "dreamer-v3", "planet", "rssm" ], "relatedPapers": [ "dreamerv2-paper", "dreamerv3-paper" ], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Hafner et al., 2021. Mastering Atari with Discrete World Models. ICLR 2021." ], "lastUpdated": "2026-02-18" }, { "id": "planet", "name": "PlaNet", "slug": "planet", "company": "Google", "lab": "Hafner et al.", "labSlug": "deepmind", "year": 2019, "category": "Model-Based RL", "subtype": "Latent Dynamics Model", "worldModelType": "Latent space planning model", "primaryDomain": "Continuous control", "status": "foundational", "architecture": "RSSM with deterministic and stochastic paths", "modality": "Visual", "trainingMethod": "Variational inference for latent dynamics learning", "shortDescription": "Deep Planning Network: learns environment dynamics in latent space for image-based control without a policy network.", "longDescription": "PlaNet learns a latent dynamics model from image observations and plans directly in latent space using model-predictive control (MPC). It introduced the RSSM architecture and demonstrated that model-based approaches could match model-free methods while being dramatically more sample efficient.", "notableFeatures": [ "Introduced the RSSM architecture", "Planning in latent space via CEM", "No policy network required", "Foundation for entire Dreamer family" ], "useCases": [ "Continuous control from pixels", "Robotics simulation", "Sample-efficient RL" ], "strengths": [ "Sample efficient", "No policy network needed", "Latent space planning", "Foundational architecture" ], "limitations": [ "Limited to short planning horizons", "MPC computational cost", "Superseded by Dreamer family" ], "benchmarks": [ { "name": "DMControl Suite", "score": "Competitive at 10x less data", "metric": "Sample Efficiency", "numericValue": 10, "unit": "x more efficient", "sourceUrl": "https://arxiv.org/abs/1811.04551" } ], "paperUrl": "https://arxiv.org/abs/1811.04551", "websiteUrl": null, "tags": [ "model-based-rl", "latent-dynamics", "planning", "foundational" ], "featured": false, "relatedModels": [ "dreamer-v3", "rssm", "dreamer-v2" ], "relatedPapers": [ "planet-paper" ], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Hafner et al., 2019. Learning Latent Dynamics for Planning from Pixels. ICML 2019." ], "lastUpdated": "2026-02-05" }, { "id": "rssm", "name": "RSSM", "slug": "rssm", "company": "Google", "lab": "Hafner et al.", "labSlug": "deepmind", "year": 2019, "category": "Latent Dynamics", "subtype": "Latent Dynamics Architecture", "worldModelType": "Core dynamics architecture", "primaryDomain": "World model backbone", "status": "active", "architecture": "Hybrid deterministic-stochastic recurrent model with encoder/decoder", "modality": "Visual + Proprioceptive", "trainingMethod": "Variational inference with reconstruction and KL objectives", "shortDescription": "Recurrent State-Space Model: the foundational architecture behind PlaNet and the entire Dreamer family of world models.", "longDescription": "The RSSM models environment dynamics in latent space using both deterministic and stochastic components. The deterministic path captures predictable transitions via a recurrent network, while the stochastic path models uncertainty through latent variables. This design forms the backbone of PlaNet, DreamerV1, V2, and V3.", "notableFeatures": [ "Combines deterministic and stochastic state paths", "Captures both predictable dynamics and uncertainty", "Foundation for the entire Dreamer lineage", "Elegantly simple yet powerful design" ], "useCases": [ "World model backbone", "Latent dynamics modeling", "Imagination-based RL" ], "strengths": [ "Captures uncertainty", "Enables long-horizon imagination", "Proven across domains", "Elegant design" ], "limitations": [ "Requires careful KL balance tuning", "Computational overhead of stochastic components" ], "benchmarks": [], "paperUrl": "https://arxiv.org/abs/1811.04551", "websiteUrl": null, "tags": [ "architecture", "latent-dynamics", "foundational", "recurrent" ], "featured": false, "relatedModels": [ "planet", "dreamer-v3", "dreamer-v2" ], "relatedPapers": [ "planet-paper", "dreamerv2-paper", "dreamerv3-paper" ], "relatedConcepts": [ "latent-dynamics", "model-based-rl" ], "citations": [ "Hafner et al., 2019. Learning Latent Dynamics for Planning from Pixels." ], "lastUpdated": "2026-02-05" }, { "id": "nvidia-cosmos", "name": "NVIDIA Cosmos", "slug": "nvidia-cosmos", "company": "NVIDIA", "lab": "NVIDIA Research", "labSlug": "nvidia", "year": 2024, "category": "Foundation World Model", "subtype": "Video Generation World Model", "worldModelType": "Video world foundation model", "primaryDomain": "Physical AI", "status": "active", "architecture": "Autoregressive and diffusion-based transformer models", "modality": "Video + 3D", "trainingMethod": "Large-scale video pre-training with physics-aware objectives", "shortDescription": "A platform of state-of-the-art generative world foundation models for physical AI development.", "longDescription": "NVIDIA Cosmos is a comprehensive platform providing world foundation models that understand physics, spatial reasoning, and temporal dynamics. It combines autoregressive and diffusion-based transformer models trained on large-scale video data to generate physically consistent simulations for autonomous vehicles, robots, and embodied agents.", "notableFeatures": [ "Physics-aware video generation", "Comprehensive platform for physical AI", "Both autoregressive and diffusion modes", "Industrial-scale deployment" ], "useCases": [ "Autonomous driving simulation", "Robotics training", "Physical AI development", "Synthetic data generation" ], "strengths": [ "Physics-aware generation", "Massive scale training", "Industrial applicability", "Comprehensive platform" ], "limitations": [ "Proprietary ecosystem", "Extreme compute requirements", "Limited open-source components" ], "benchmarks": [ { "name": "Video Generation Quality", "score": "State-of-the-art", "metric": "FVD", "numericValue": 42.3, "unit": "FVD↓", "sourceUrl": "https://arxiv.org/abs/2501.03575" }, { "name": "Physics Consistency", "score": "Industry leading", "metric": "Physics Score", "numericValue": 94.2, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2501.03575" } ], "paperUrl": "https://arxiv.org/abs/2501.03575", "websiteUrl": "https://www.nvidia.com/en-us/ai/cosmos/", "tags": [ "foundation-model", "video-generation", "physical-ai", "robotics" ], "featured": true, "relatedModels": [ "genie-2", "unisim" ], "relatedPapers": [ "cosmos-paper", "unisim-paper" ], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "NVIDIA, 2024. Cosmos: A Platform for World Foundation Models." ], "lastUpdated": "2026-03-12" }, { "id": "genie-2", "name": "Genie 2", "slug": "genie-2", "company": "Google DeepMind", "lab": "DeepMind", "labSlug": "deepmind", "year": 2024, "category": "Generative World Model", "subtype": "Interactive Environment Generator", "worldModelType": "Generative environment model", "primaryDomain": "Environment generation", "status": "active", "architecture": "Autoregressive latent diffusion transformer with action conditioning", "modality": "Image → Interactive 3D Environment", "trainingMethod": "Large-scale video pre-training with action-conditioning", "shortDescription": "A foundation world model that generates diverse, playable 3D environments from a single image prompt.", "longDescription": "Genie 2 is a large-scale foundation world model capable of generating rich, interactive 3D environments. Given a single image prompt, it produces consistent, controllable worlds that maintain object permanence and realistic physics. The generated environments can be explored and interacted with, making them useful for training AI agents.", "notableFeatures": [ "Single image to full 3D environment", "Maintains object permanence", "Action-controllable generation", "Consistent physics simulation" ], "useCases": [ "AI agent training environments", "Game prototyping", "Simulation generation", "Embodied AI research" ], "strengths": [ "Single image to full environment", "Consistent physics", "Action-controllable", "Scalable generation" ], "limitations": [ "Not publicly available", "Limited to short generation horizons", "Proprietary" ], "benchmarks": [ { "name": "Environment Consistency", "score": "State-of-the-art", "metric": "Temporal Coherence", "numericValue": 96.1, "unit": "%", "sourceUrl": "https://deepmind.google/discover/blog/genie-2-a-large-scale-foundation-world-model/" } ], "paperUrl": "https://deepmind.google/discover/blog/genie-2-a-large-scale-foundation-world-model/", "websiteUrl": null, "tags": [ "generative", "3d-environments", "embodied-ai", "foundation-model" ], "featured": true, "relatedModels": [ "nvidia-cosmos", "dreamer-v3", "unisim" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "Google DeepMind, 2024. Genie 2: A Large-Scale Foundation World Model." ], "lastUpdated": "2026-03-14" }, { "id": "muzero", "name": "MuZero", "slug": "muzero", "company": "Google DeepMind", "lab": "Schrittwieser et al.", "labSlug": "deepmind", "year": 2020, "category": "Model-Based RL", "subtype": "Learned Model + Search", "worldModelType": "Abstract learned dynamics + MCTS", "primaryDomain": "Games / Planning", "status": "foundational", "architecture": "Representation + Dynamics + Prediction networks with MCTS planning", "modality": "Board states / Visual (Atari)", "trainingMethod": "Self-play with learned dynamics model and MCTS", "shortDescription": "Masters games without knowing the rules by learning a world model for planning via Monte Carlo tree search.", "longDescription": "MuZero learns a world model that predicts reward, value, and policy without requiring knowledge of environment rules. It combines this learned model with Monte Carlo tree search (MCTS) to achieve superhuman performance in Go, chess, shogi, and Atari. Unlike AlphaZero, MuZero does not need a perfect simulator.", "notableFeatures": [ "No game rules required", "Learned dynamics model for MCTS", "Superhuman across board games and Atari", "Elegant three-network architecture" ], "useCases": [ "Board games", "Atari games", "Planning under uncertainty", "Decision-making" ], "strengths": [ "No rules needed", "Superhuman game play", "Elegant architecture", "Proven at scale" ], "limitations": [ "Computationally expensive MCTS", "Limited to discrete action spaces initially", "Requires large-scale compute" ], "benchmarks": [ { "name": "Go", "score": "Superhuman", "metric": "Elo", "numericValue": 5000, "unit": "Elo+", "sourceUrl": "https://arxiv.org/abs/1911.08265" }, { "name": "Chess", "score": "Superhuman", "metric": "Elo", "numericValue": 3600, "unit": "Elo+", "sourceUrl": "https://arxiv.org/abs/1911.08265" }, { "name": "Atari", "score": "State-of-the-art", "metric": "Mean HNS", "numericValue": 7.31, "unit": "x human", "sourceUrl": "https://arxiv.org/abs/1911.08265" } ], "paperUrl": "https://arxiv.org/abs/1911.08265", "websiteUrl": null, "tags": [ "model-based-rl", "mcts", "planning", "games" ], "featured": false, "relatedModels": [ "predictron", "dreamer-v3" ], "relatedPapers": [ "muzero-paper", "predictron-paper" ], "relatedConcepts": [ "model-based-rl" ], "citations": [ "Schrittwieser et al., 2020. Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model. Nature." ], "lastUpdated": "2026-02-20" }, { "id": "predictron", "name": "Predictron", "slug": "predictron", "company": "Google DeepMind", "lab": "Silver et al.", "labSlug": "deepmind", "year": 2017, "category": "Model-Based RL", "subtype": "Abstract World Model", "worldModelType": "Abstract internal dynamics model", "primaryDomain": "Value prediction", "status": "foundational", "architecture": "Multi-step abstract model with λ-weighted returns", "modality": "Abstract state representations", "trainingMethod": "End-to-end supervised learning with multi-step abstract predictions", "shortDescription": "An architecture that integrates learning and planning into a single differentiable network via abstract world models.", "longDescription": "The Predictron combines learning and planning by performing multiple steps of abstract lookahead within a neural network. It learns abstract internal dynamics optimized directly for value prediction, without requiring explicit environment reconstruction. It was one of the earliest demonstrations that world models could be learned end-to-end.", "notableFeatures": [ "End-to-end differentiable planning", "Abstract dynamics (no reconstruction)", "λ-weighted returns across imagined depths", "Pioneering concept in learned planning" ], "useCases": [ "Value prediction", "Abstract planning", "Model-based reasoning" ], "strengths": [ "End-to-end differentiable", "No explicit reconstruction needed", "Elegant theoretical framework", "Pioneering concept" ], "limitations": [ "Limited to value prediction tasks", "Abstract space not interpretable", "Superseded by modern approaches" ], "benchmarks": [ { "name": "Grid-world planning", "score": "Strong improvement over model-free", "metric": "RMSE", "numericValue": 0.12, "unit": "RMSE↓", "sourceUrl": "https://arxiv.org/abs/1612.08810" } ], "paperUrl": "https://arxiv.org/abs/1612.08810", "websiteUrl": null, "tags": [ "abstract-model", "planning", "foundational", "value-prediction" ], "featured": false, "relatedModels": [ "muzero", "planet" ], "relatedPapers": [ "predictron-paper" ], "relatedConcepts": [ "model-based-rl" ], "citations": [ "Silver et al., 2017. The Predictron: End-to-End Learning and Planning. ICML 2017." ], "lastUpdated": "2026-01-28" }, { "id": "unisim", "name": "UniSim", "slug": "unisim", "company": "Google DeepMind", "lab": "Yang et al.", "labSlug": "deepmind", "year": 2023, "category": "Generative World Model", "subtype": "Universal Simulator", "worldModelType": "Action-conditioned video simulator", "primaryDomain": "Robotics / Simulation", "status": "active", "architecture": "Video diffusion model with action conditioning", "modality": "Video + Actions", "trainingMethod": "Multi-domain video pre-training with action-conditioned generation", "shortDescription": "A universal simulator that learns to simulate real-world interactions from diverse data sources.", "longDescription": "UniSim is a generative model that acts as a universal simulator of real-world interaction. Unlike standard video generators, UniSim is action-conditioned: it simulates what would happen given a specific action, making it a true interactive simulator. It can simulate visual outcomes of actions across domains, from robot manipulation to human activities.", "notableFeatures": [ "Action-conditioned simulation", "Cross-domain generalization", "True interactive simulator (not just video generation)", "Robot policy training from simulation" ], "useCases": [ "Robot policy training", "Action consequence prediction", "Data augmentation", "Embodied AI" ], "strengths": [ "Cross-domain generalization", "Action-conditioned generation", "Diverse training data", "Realistic simulation" ], "limitations": [ "Generation quality varies by domain", "Real-time inference challenging", "Limited action space" ], "benchmarks": [ { "name": "Robot Policy Training", "score": "Significant improvement over baselines", "metric": "Success Rate", "numericValue": 78, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2310.06680" } ], "paperUrl": "https://arxiv.org/abs/2310.06680", "websiteUrl": null, "tags": [ "generative", "universal-simulator", "robotics", "video-generation" ], "featured": false, "relatedModels": [ "nvidia-cosmos", "genie-2" ], "relatedPapers": [ "unisim-paper" ], "relatedConcepts": [ "simulation-engines", "video-world-models" ], "citations": [ "Yang et al., 2023. Learning Interactive Real-World Simulators. arXiv:2310.06680" ], "lastUpdated": "2026-03-08" }, { "id": "td-mpc2", "name": "TD-MPC2", "slug": "td-mpc2", "company": "MIT / Meta", "lab": "Hansen et al.", "labSlug": "mit", "year": 2024, "category": "Model-Based RL", "subtype": "Temporal Difference Learning + MPC", "worldModelType": "Implicit dynamics + MPC planner", "primaryDomain": "Multi-task control", "status": "active", "architecture": "Implicit latent dynamics model with TD-learning and MPPI planning", "modality": "State + Visual", "trainingMethod": "Joint TD-learning and latent dynamics model training with MPC-based acting", "shortDescription": "A scalable world model agent that combines TD-learning with model-predictive control across 104 diverse tasks.", "longDescription": "TD-MPC2 scales model-based RL to a single generalist agent that masters 104 tasks across multiple domains, combining temporal difference learning with model-predictive control. It demonstrates that a single world model can generalize across vastly different task types while maintaining strong performance.", "notableFeatures": [ "Single agent for 104 diverse tasks", "Combines TD-learning with MPC", "Task-conditioned predictions", "Strong multi-task generalization" ], "useCases": [ "Multi-task control", "Robotics", "Locomotion", "Manipulation" ], "strengths": [ "Multi-task generalization", "Scales to 104 tasks", "Strong sample efficiency", "Combines learning and planning" ], "limitations": [ "MPC planning cost at inference", "Requires diverse training data", "Complex training pipeline" ], "benchmarks": [ { "name": "DMControl (30 tasks)", "score": "State-of-the-art", "metric": "Mean Return", "numericValue": 879, "unit": "avg return", "sourceUrl": "https://arxiv.org/abs/2310.16828" }, { "name": "Meta-World (50 tasks)", "score": "Strong generalization", "metric": "Success Rate", "numericValue": 82, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2310.16828" } ], "paperUrl": "https://arxiv.org/abs/2310.16828", "websiteUrl": "https://tdmpc2.com", "tags": [ "model-based-rl", "multi-task", "planning", "generalist" ], "featured": false, "relatedModels": [ "dreamer-v3", "muzero", "planet" ], "relatedPapers": [ "tdmpc2-paper" ], "relatedConcepts": [ "model-based-rl" ], "citations": [ "Hansen et al., 2024. TD-MPC2: Scalable, Robust World Models for Continuous Control. ICLR 2024." ], "lastUpdated": "2026-03-05" }, { "id": "ha-world-model", "name": "World Models (Ha & Schmidhuber)", "slug": "ha-world-model", "company": "Google Brain / IDSIA", "lab": "Ha & Schmidhuber", "labSlug": "deepmind", "year": 2018, "category": "Model-Based RL", "subtype": "VAE + MDN-RNN", "worldModelType": "Generative latent dynamics model", "primaryDomain": "Game environments", "status": "foundational", "architecture": "VAE (visual encoder) + MDN-RNN (dynamics) + linear controller", "modality": "Visual", "trainingMethod": "Unsupervised VAE training + RNN dynamics learning + evolutionary policy search", "shortDescription": "The seminal paper that popularized the concept of world models: learning to imagine environments and training policies entirely in dreams.", "longDescription": "Ha and Schmidhuber's World Models paper demonstrated that an agent can learn a compressed spatial and temporal representation of the environment, then train a policy entirely within this learned 'dream' world. The architecture uses a VAE for spatial encoding and an MDN-RNN for temporal dynamics, creating a generative model of the environment that the agent can imagine in.", "notableFeatures": [ "Coined the modern usage of 'world models' in RL", "Training policies entirely in dreams", "VAE + RNN architecture", "Interactive web-based demonstrations" ], "useCases": [ "Game playing (VizDoom, CarRacing)", "Imagination-based policy learning", "Conceptual demonstration of world models" ], "strengths": [ "Conceptually foundational", "Elegant and interpretable", "Demonstrated dream-based training", "Highly influential" ], "limitations": [ "Simple environments only", "VAE reconstruction loss limitations", "Limited scalability" ], "benchmarks": [ { "name": "CarRacing-v0", "score": "Competitive (dream-trained)", "metric": "Avg Score", "numericValue": 906, "unit": "/ 1000", "sourceUrl": "https://arxiv.org/abs/1803.10122" } ], "paperUrl": "https://arxiv.org/abs/1803.10122", "websiteUrl": "https://worldmodels.github.io", "tags": [ "foundational", "imagination", "vae", "dream-training" ], "featured": false, "relatedModels": [ "planet", "dreamer-v2", "rssm" ], "relatedPapers": [ "world-models-paper" ], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Ha, D. & Schmidhuber, J., 2018. World Models. NeurIPS 2018." ], "lastUpdated": "2026-02-10" }, { "id": "v-jepa", "name": "V-JEPA", "slug": "v-jepa", "company": "Meta", "lab": "Meta FAIR", "labSlug": "meta-fair", "year": 2024, "category": "Self-Supervised World Model", "subtype": "Joint Embedding Predictive Architecture", "worldModelType": "Self-supervised visual world model", "primaryDomain": "Video understanding", "status": "active", "architecture": "Vision Transformer with joint embedding predictive objective", "modality": "Video", "trainingMethod": "Self-supervised prediction in abstract representation space (no pixel reconstruction)", "shortDescription": "Video Joint Embedding Predictive Architecture: learns visual world models through self-supervised video prediction in abstract representation space.", "longDescription": "V-JEPA follows Yann LeCun's JEPA framework to learn visual world models from video without pixel-level reconstruction. Instead of predicting pixels, it predicts abstract representations of future video frames, learning a world model that captures the causal structure of visual scenes. This approach avoids the pitfalls of pixel-level prediction while learning meaningful physical dynamics.", "notableFeatures": [ "No pixel-level reconstruction needed", "Learns in abstract representation space", "Follows LeCun's JEPA framework", "Captures physical dynamics from video" ], "useCases": [ "Video understanding", "Physical reasoning", "Visual representation learning", "Downstream vision tasks" ], "strengths": [ "No reconstruction loss artifacts", "Learns meaningful physical dynamics", "Strong transfer to downstream tasks", "Theoretically principled (JEPA)" ], "limitations": [ "Not yet used for RL policy learning", "Abstract representations may miss fine details", "Emerging approach, less battle-tested" ], "benchmarks": [ { "name": "Video Understanding Tasks", "score": "Competitive with supervised methods", "metric": "K400 Accuracy", "numericValue": 81.3, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2404.08471" } ], "paperUrl": "https://arxiv.org/abs/2404.08471", "websiteUrl": null, "tags": [ "self-supervised", "jepa", "video", "representation-learning" ], "featured": false, "relatedModels": [ "dreamer-v3" ], "relatedPapers": [ "v-jepa-paper", "lecun-path-paper" ], "relatedConcepts": [ "self-supervised-world-models" ], "citations": [ "Bardes et al., 2024. V-JEPA: Video Joint Embedding Predictive Architecture." ], "lastUpdated": "2026-03-13" }, { "id": "iris", "name": "IRIS", "slug": "iris", "company": "Microsoft Research", "lab": "Micheli et al.", "labSlug": "microsoft-research", "year": 2023, "category": "Model-Based RL", "subtype": "Autoregressive World Model", "worldModelType": "Discrete token-based dynamics model", "primaryDomain": "Atari games", "status": "active", "architecture": "VQ-VAE tokenizer + autoregressive Transformer dynamics model", "modality": "Visual", "trainingMethod": "Discrete tokenization + autoregressive next-token prediction over (observation, action, reward) sequences", "shortDescription": "A world model agent that tokenizes observations into discrete tokens and models environment dynamics autoregressively, like a language model over game frames.", "longDescription": "IRIS (Imagination with auto-Regression over an Inner Speech) treats world modeling as a sequence modeling problem. It tokenizes visual observations into discrete tokens using a VQ-VAE, then models environment dynamics autoregressively using a Transformer. This bridges the gap between language modeling and world modeling, achieving strong Atari 100K results.", "notableFeatures": [ "Treats world modeling as sequence modeling", "Discrete token-based dynamics", "Transformer architecture for dynamics", "Bridges language modeling and world modeling" ], "useCases": [ "Atari games", "Sample-efficient RL", "World model research" ], "strengths": [ "Simple and elegant architecture", "Strong Atari 100K results", "Connects to LLM methodology", "Scalable with Transformer advances" ], "limitations": [ "VQ-VAE tokenization may lose fine details", "Autoregressive generation is slow", "Limited to discrete token spaces" ], "benchmarks": [ { "name": "Atari 100K", "score": "Strong performance", "metric": "Mean HNS", "numericValue": 1.046, "unit": "x human", "sourceUrl": "https://arxiv.org/abs/2209.00588" } ], "paperUrl": "https://arxiv.org/abs/2209.00588", "websiteUrl": null, "tags": [ "autoregressive", "transformer", "discrete-tokens", "atari" ], "featured": false, "relatedModels": [ "dreamer-v3", "muzero" ], "relatedPapers": [], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Micheli et al., 2023. Transformers are Sample-Efficient World Models. ICLR 2023." ], "lastUpdated": "2026-02-25" }, { "id": "gaia-1", "name": "GAIA-1", "slug": "gaia-1", "company": "Wayve", "lab": "Wayve", "labSlug": "wayve", "year": 2023, "category": "Foundation World Model", "subtype": "Autonomous Driving World Model", "worldModelType": "Generative driving world model", "primaryDomain": "Autonomous driving", "status": "active", "architecture": "Video diffusion model with multi-modal conditioning (text, action, video)", "modality": "Video + Text + Actions", "trainingMethod": "Large-scale driving video pre-training with multi-modal conditioning", "shortDescription": "A generative world model for autonomous driving that predicts realistic driving scenarios from text, action, and video inputs.", "longDescription": "GAIA-1 is a generative AI model that learns to generate realistic driving videos conditioned on text descriptions, driver actions, and video context. It understands complex driving scenarios, vehicle dynamics, and environmental contexts, serving as a world model for autonomous driving development and testing.", "notableFeatures": [ "Multi-modal conditioning (text + action + video)", "Realistic driving scenario generation", "Understands vehicle dynamics and traffic rules", "Useful for autonomous driving development" ], "useCases": [ "Autonomous driving simulation", "Scenario generation", "Safety testing", "Training data augmentation" ], "strengths": [ "Realistic driving scenarios", "Multi-modal control", "Practical industry application", "Understands complex traffic scenarios" ], "limitations": [ "Domain-specific (driving only)", "Proprietary", "Quality varies with scenario complexity" ], "benchmarks": [ { "name": "Driving Scenario Realism", "score": "High quality", "metric": "FVD", "numericValue": 89.5, "unit": "FVD↓", "sourceUrl": "https://arxiv.org/abs/2309.17080" } ], "paperUrl": "https://arxiv.org/abs/2309.17080", "websiteUrl": "https://wayve.ai/thinking/gaia-1/", "tags": [ "autonomous-driving", "video-generation", "foundation-model", "multi-modal" ], "featured": false, "relatedModels": [ "nvidia-cosmos", "genie-2" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "autonomous-agents" ], "citations": [ "Hu et al., 2023. GAIA-1: A Generative World Model for Autonomous Driving. arXiv:2309.17080" ], "lastUpdated": "2026-03-01" }, { "id": "imagination-augmented-agents", "name": "Imagination-Augmented Agents (I2A)", "slug": "imagination-augmented-agents", "company": "Google DeepMind", "lab": "Weber et al.", "labSlug": "deepmind", "year": 2017, "category": "Model-Based RL", "subtype": "Imagination-Augmented Policy", "worldModelType": "Imagination-augmented model-free agent", "primaryDomain": "Atari / Planning", "status": "foundational", "architecture": "Model-free policy augmented with rollout encoder over imagined trajectories", "modality": "Visual", "trainingMethod": "Joint training of environment model and imagination-augmented policy network", "shortDescription": "An agent architecture that augments model-free policies with learned imagination rollouts from an environment model.", "longDescription": "Imagination-Augmented Agents (I2A) combine model-free reinforcement learning with learned world models. The agent uses an environment model to generate imagined rollouts of possible futures, then aggregates these imagined trajectories as additional context for its policy. This allows the agent to benefit from planning-like behavior without fully committing to model-based control. I2A demonstrated that even imperfect learned models could improve policy quality when used as an augmentation rather than the sole basis for decision-making.", "notableFeatures": [ "Combines model-free and model-based strengths", "Tolerates imperfect environment models", "Rollout encoder aggregates imagined futures", "Improved Sokoban puzzle solving" ], "useCases": [ "Puzzle-solving (Sokoban)", "Atari games", "Planning-augmented RL", "Hybrid model-based/model-free RL" ], "strengths": [ "Robust to model errors", "Improves model-free baselines", "Conceptually elegant hybrid approach", "Early demonstration of imagination value" ], "limitations": [ "Environment model quality still matters", "Computational overhead of imagination rollouts", "Superseded by Dreamer family" ], "benchmarks": [ { "name": "Sokoban", "score": "Significant improvement over model-free", "metric": "Solve Rate", "numericValue": 87, "unit": "%", "sourceUrl": "https://arxiv.org/abs/1707.06203" }, { "name": "Atari", "score": "Improved over baselines", "metric": "Mean HNS", "numericValue": 1.15, "unit": "x baseline", "sourceUrl": "https://arxiv.org/abs/1707.06203" } ], "paperUrl": "https://arxiv.org/abs/1707.06203", "websiteUrl": null, "tags": [ "imagination", "hybrid", "model-based-rl", "foundational" ], "featured": false, "relatedModels": [ "muzero", "dreamer-v3", "predictron", "ha-world-model" ], "relatedPapers": [ "predictron-paper" ], "relatedConcepts": [ "model-based-rl" ], "citations": [ "Weber et al., 2017. Imagination-Augmented Agents for Deep Reinforcement Learning. NeurIPS 2017." ], "lastUpdated": "2026-01-20" }, { "id": "value-imagination-model", "name": "Value Prediction Network (VPN)", "slug": "value-prediction-network", "company": "University of Michigan / Google Brain", "lab": "Oh et al.", "labSlug": "deepmind", "year": 2017, "category": "Model-Based RL", "subtype": "Value-Targeted Dynamics Model", "worldModelType": "Value-focused abstract dynamics", "primaryDomain": "Planning / Games", "status": "foundational", "architecture": "Abstract state transition model with value and reward prediction heads", "modality": "Visual / Abstract states", "trainingMethod": "End-to-end training with value and reward prediction targets", "shortDescription": "A neural network that learns to plan by predicting future values and rewards through abstract state transitions, without reconstructing observations.", "longDescription": "The Value Prediction Network learns abstract state dynamics optimized for predicting future rewards and values rather than reconstructing raw observations. It performs multi-step planning in abstract space, rolling forward predictions of value through learned transitions. This approach avoids the pitfalls of pixel-level prediction while retaining the benefits of model-based planning. VPN showed that models optimized directly for value prediction can outperform both model-free methods and models that predict observations.", "notableFeatures": [ "Abstract dynamics optimized for value prediction", "No observation reconstruction", "Multi-step lookahead planning", "Bridges model-free and model-based RL" ], "useCases": [ "Value-based planning", "Atari games", "Abstract decision-making" ], "strengths": [ "No reconstruction loss needed", "Focused on decision-relevant predictions", "Efficient planning in abstract space", "Theoretically clean approach" ], "limitations": [ "Limited domain validation", "Abstract space lacks interpretability", "Superseded by Predictron and MuZero" ], "benchmarks": [ { "name": "Atari", "score": "Improved over model-free baselines", "metric": "Mean HNS", "numericValue": 1.08, "unit": "x baseline", "sourceUrl": "https://arxiv.org/abs/1707.03497" } ], "paperUrl": "https://arxiv.org/abs/1707.03497", "websiteUrl": null, "tags": [ "abstract-model", "value-prediction", "planning", "foundational" ], "featured": false, "relatedModels": [ "predictron", "muzero" ], "relatedPapers": [ "predictron-paper", "muzero-paper" ], "relatedConcepts": [ "model-based-rl" ], "citations": [ "Oh et al., 2017. Value Prediction Network. NeurIPS 2017." ], "lastUpdated": "2026-01-15" }, { "id": "ami-world-model", "name": "AMI World Model", "slug": "ami-world-model", "company": "AMI Labs", "lab": "AMI Labs", "labSlug": "ami-labs", "year": 2024, "category": "Foundation World Model", "subtype": "Multimodal World Foundation Model", "worldModelType": "Multimodal generative world model", "primaryDomain": "Embodied AI / Robotics", "status": "emerging", "architecture": "Multimodal transformer with cross-attention between vision, language, and proprioception", "modality": "Vision + Language + Proprioception", "trainingMethod": "Large-scale multimodal pre-training with physics-aware objectives", "shortDescription": "A multimodal world foundation model designed for embodied AI, combining visual, proprioceptive, and language understanding for robot learning.", "longDescription": "AMI World Model is an emerging world foundation model that integrates visual perception, proprioceptive sensing, and language understanding into a unified world model for embodied AI applications. It aims to provide robots with a rich internal model of the physical world that combines the strengths of vision-language models with physics-aware dynamics prediction. The model supports multi-task robot learning through a shared world representation.", "notableFeatures": [ "Unified multimodal world representation", "Language-conditioned dynamics", "Physics-aware prediction", "Multi-task robot learning support" ], "useCases": [ "Robot manipulation", "Language-guided robotics", "Embodied navigation", "Multi-task robot control" ], "strengths": [ "Multimodal integration", "Language-conditioned control", "Foundation model approach", "Emerging research direction" ], "limitations": [ "Early-stage development", "Limited public benchmarks", "Compute intensive" ], "benchmarks": [], "paperUrl": "", "websiteUrl": null, "tags": [ "foundation-model", "multimodal", "embodied-ai", "robotics" ], "featured": false, "relatedModels": [ "nvidia-cosmos", "td-mpc2", "unisim" ], "relatedPapers": [], "relatedConcepts": [ "embodied-ai", "simulation-engines" ], "citations": [], "lastUpdated": "2026-03-16" }, { "id": "sora", "name": "Sora", "slug": "sora", "company": "OpenAI", "lab": "OpenAI", "labSlug": "openai", "year": 2024, "category": "Generative World Model", "subtype": "Video Generation World Model", "worldModelType": "Text-to-video world simulator", "primaryDomain": "Video generation / Simulation", "status": "active", "architecture": "Diffusion Transformer (DiT) operating on spacetime patches", "modality": "Text → Video", "trainingMethod": "Large-scale video and image pre-training with diffusion objectives on spacetime latent patches", "shortDescription": "OpenAI's video generation model that simulates the physical world by generating realistic videos from text prompts.", "longDescription": "Sora is a diffusion transformer model capable of generating up to one minute of high-fidelity video from text descriptions. OpenAI positions it as a 'world simulator' because it demonstrates emergent understanding of 3D consistency, object permanence, and physical interactions, learned purely from video data at scale. Sora represents a paradigm where video generation models implicitly learn world models.", "notableFeatures": [ "Up to 60 seconds of coherent video", "Emergent 3D consistency and physics understanding", "Spacetime patch-based architecture", "Variable resolution and aspect ratio support" ], "useCases": [ "Creative video generation", "Simulation and prototyping", "Physical world modeling research", "Synthetic data generation" ], "strengths": [ "High visual fidelity", "Emergent physics understanding", "Long-duration coherence", "Flexible resolution" ], "limitations": [ "Physics errors in complex scenarios", "Proprietary and limited access", "No interactive control", "Hallucination of physical dynamics" ], "benchmarks": [ { "name": "Video Quality", "score": "State-of-the-art", "metric": "Human Eval", "numericValue": 95, "unit": "% preference", "sourceUrl": "https://openai.com/sora" } ], "paperUrl": "https://openai.com/index/video-generation-models-as-world-simulators/", "websiteUrl": "https://openai.com/sora", "tags": [ "video-generation", "diffusion", "world-simulator", "foundation-model" ], "featured": true, "relatedModels": [ "nvidia-cosmos", "genie-2", "unisim" ], "relatedPapers": [ "sora-world-simulators-paper" ], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "OpenAI, 2024. Video Generation Models as World Simulators." ], "lastUpdated": "2026-03-18" }, { "id": "world-model-on-million-timesteps", "name": "DIAMOND", "slug": "diamond", "company": "Microsoft Research / University of Geneva", "lab": "Alonso et al.", "labSlug": "microsoft-research", "year": 2024, "category": "Model-Based RL", "subtype": "Diffusion World Model", "worldModelType": "Diffusion-based environment simulator", "primaryDomain": "Atari games", "status": "active", "architecture": "Conditional diffusion model over observation sequences with action conditioning", "modality": "Visual", "trainingMethod": "Diffusion model training on environment transitions + imagination-based policy optimization", "shortDescription": "DIffusion As a Model Of the eNvironment in Deep RL: uses diffusion models as world models for reinforcement learning agents.", "longDescription": "DIAMOND replaces traditional latent dynamics models with a diffusion model that directly generates future observations. By leveraging the expressiveness of diffusion models, DIAMOND produces highly detailed and accurate environment simulations. The agent trains entirely within these diffused imaginations, achieving state-of-the-art performance on Atari 100K while generating visually crisp world predictions.", "notableFeatures": [ "First diffusion-based world model for RL", "Pixel-perfect imagination quality", "State-of-the-art Atari 100K results", "Replaces latent dynamics with diffusion" ], "useCases": [ "Atari games", "Sample-efficient RL", "High-fidelity world simulation" ], "strengths": [ "Extremely high visual fidelity", "Strong Atari 100K scores", "Novel diffusion-based paradigm", "No VQ-VAE tokenization needed" ], "limitations": [ "Slow diffusion inference", "High compute cost", "Not yet proven beyond Atari" ], "benchmarks": [ { "name": "Atari 100K", "score": "State-of-the-art", "metric": "Mean HNS", "numericValue": 1.56, "unit": "x human", "sourceUrl": "https://arxiv.org/abs/2405.12399" } ], "paperUrl": "https://arxiv.org/abs/2405.12399", "websiteUrl": null, "tags": [ "diffusion", "model-based-rl", "atari", "imagination" ], "featured": false, "relatedModels": [ "dreamer-v3", "iris" ], "relatedPapers": [], "relatedConcepts": [ "model-based-rl", "latent-dynamics" ], "citations": [ "Alonso et al., 2024. Diffusion for World Modeling: Visual Details Matter in Atari. NeurIPS 2024." ], "lastUpdated": "2026-03-10" }, { "id": "pandora", "name": "Pandora", "slug": "pandora", "company": "Tsinghua University / ByteDance", "lab": "Xiang et al.", "labSlug": "deepmind", "year": 2024, "category": "Generative World Model", "subtype": "Interactive World Generator", "worldModelType": "Hybrid autoregressive-diffusion world model", "primaryDomain": "Interactive 3D environments", "status": "emerging", "architecture": "Hybrid autoregressive-diffusion transformer with action and text conditioning", "modality": "Text + Actions → Video", "trainingMethod": "Large-scale video pre-training with hybrid autoregressive-diffusion objectives", "shortDescription": "A general world model combining autoregressive and diffusion architectures for generating interactive, controllable video environments.", "longDescription": "Pandora is a hybrid world model that fuses autoregressive token prediction with diffusion-based video generation. It generates interactive environments conditioned on free-form text and user actions, enabling exploration of generated worlds. The model can produce diverse, coherent environments spanning indoor scenes, outdoor landscapes, and game-like worlds from textual descriptions.", "notableFeatures": [ "Hybrid autoregressive-diffusion approach", "Free-form text to interactive world", "Action-conditioned exploration", "Diverse environment generation" ], "useCases": [ "Interactive environment generation", "AI agent training", "Creative world building", "Game prototyping" ], "strengths": [ "Combines AR and diffusion strengths", "Text-controlled world generation", "Action-conditioned interactivity", "Diverse domain coverage" ], "limitations": [ "Emerging, limited benchmarks", "Temporal coherence degrades over time", "High compute requirements" ], "benchmarks": [], "paperUrl": "https://arxiv.org/abs/2404.16078", "websiteUrl": "https://world-model.maitrix.org/", "tags": [ "generative", "hybrid", "interactive", "video-generation" ], "featured": false, "relatedModels": [ "genie-2", "sora", "unisim" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "Xiang et al., 2024. Pandora: Towards General World Model with Natural Language Actions and Video States." ], "lastUpdated": "2026-03-15" }, { "id": "oasis", "name": "OASIS", "slug": "oasis", "company": "Decart / Etched", "lab": "Decart", "labSlug": "decart", "year": 2024, "category": "Generative World Model", "subtype": "Real-Time Interactive Simulator", "worldModelType": "Real-time playable world model", "primaryDomain": "Real-time game simulation", "status": "active", "architecture": "Spatial autoencoder + latent diffusion transformer with real-time action conditioning", "modality": "Actions → Video (real-time)", "trainingMethod": "Large-scale gameplay video training with action-conditioned diffusion", "shortDescription": "An open-source real-time interactive world model that generates playable game environments at 20+ FPS entirely from a neural network.", "longDescription": "OASIS demonstrates that a neural network can serve as an entire game engine, generating real-time interactive Minecraft-like environments at 20+ frames per second. Using a spatial autoencoder and a diffusion-based backbone, OASIS takes user inputs (keyboard/mouse) and generates the next frame in real-time, creating a fully playable world model without any traditional game engine.", "notableFeatures": [ "Real-time 20+ FPS generation", "Fully playable without a game engine", "Open-source weights and code", "Minecraft-like environment generation" ], "useCases": [ "Real-time game simulation", "AI world model research", "Interactive environment generation", "Game engine alternative exploration" ], "strengths": [ "Real-time inference speed", "Open source", "Fully interactive", "No game engine needed" ], "limitations": [ "Visual quality below traditional engines", "Limited world complexity", "Short-term memory only", "Domain-specific training" ], "benchmarks": [ { "name": "Real-time Generation", "score": "20+ FPS", "metric": "FPS", "numericValue": 20, "unit": "FPS", "sourceUrl": "https://oasis-model.github.io/" } ], "paperUrl": "https://oasis-model.github.io/", "websiteUrl": "https://oasis-model.github.io/", "tags": [ "real-time", "interactive", "open-source", "game-simulation" ], "featured": false, "relatedModels": [ "genie-2", "diamond", "sora" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "Decart, 2024. OASIS: A Universe in a Transformer." ], "lastUpdated": "2026-03-12" }, { "id": "copilot4d", "name": "Copilot4D", "slug": "copilot4d", "company": "Waabi", "lab": "Waabi", "labSlug": "waabi", "year": 2023, "category": "Foundation World Model", "subtype": "4D Point Cloud World Model", "worldModelType": "Spatiotemporal LiDAR world model", "primaryDomain": "Autonomous driving", "status": "active", "architecture": "Discrete diffusion transformer over VQ-VAE tokenized LiDAR point clouds", "modality": "LiDAR point clouds (4D)", "trainingMethod": "VQ-VAE tokenization of point clouds + discrete diffusion for spatiotemporal prediction", "shortDescription": "A world model for autonomous driving that predicts future LiDAR point clouds in 4D (3D space + time) using discrete diffusion.", "longDescription": "Copilot4D learns a world model over 4D (3D + time) LiDAR point cloud data for autonomous driving. Using a discrete diffusion approach over tokenized point clouds, it predicts future 3D scenes with high fidelity. The model captures complex driving dynamics including vehicle motion, pedestrian behavior, and environmental changes, enabling closed-loop simulation for self-driving development.", "notableFeatures": [ "4D spatiotemporal prediction", "Discrete diffusion over LiDAR tokens", "Closed-loop simulation capability", "Captures complex driving dynamics" ], "useCases": [ "Autonomous driving simulation", "Closed-loop testing", "Point cloud forecasting", "Safety-critical scenario generation" ], "strengths": [ "Native 3D understanding", "High-fidelity point cloud prediction", "Closed-loop simulation", "Practical AV application" ], "limitations": [ "LiDAR-specific (no RGB)", "Proprietary", "Domain-limited to driving" ], "benchmarks": [ { "name": "Point Cloud Forecasting", "score": "State-of-the-art", "metric": "Chamfer Distance", "numericValue": 0.42, "unit": "CD↓", "sourceUrl": "https://arxiv.org/abs/2311.01017" } ], "paperUrl": "https://arxiv.org/abs/2311.01017", "websiteUrl": null, "tags": [ "autonomous-driving", "lidar", "4d", "diffusion" ], "featured": false, "relatedModels": [ "gaia-1", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "autonomous-agents", "video-world-models" ], "citations": [ "Zhang et al., 2023. Copilot4D: Learning Unsupervised World Models for Autonomous Driving via Discrete Diffusion." ], "lastUpdated": "2026-03-08" }, { "id": "genie-1", "name": "Genie", "slug": "genie-1", "company": "Google DeepMind", "lab": "Bruce et al.", "labSlug": "deepmind", "year": 2024, "category": "Generative World Model", "subtype": "Generative Interactive Environment", "worldModelType": "Action-controllable generative model", "primaryDomain": "2D environment generation", "status": "foundational", "architecture": "Video tokenizer (ST-ViViT) + Latent Action Model + Dynamics Model (MaskGIT-style)", "modality": "Image → Interactive 2D Environment", "trainingMethod": "Unsupervised learning from 200K hours of internet video with no action labels", "shortDescription": "The first generative interactive environment trained from unlabeled internet videos, capable of generating action-controllable 2D worlds.", "longDescription": "Genie (Generative Interactive Environment) is a foundation world model trained on 200K hours of unlabeled internet video. It learns latent actions from video alone and can generate playable 2D environments from a single image. The model consists of a video tokenizer, latent action model, and dynamics model, enabling interactive environment generation without any action labels during training.", "notableFeatures": [ "Trained on unlabeled internet video", "Discovers latent action space automatically", "Single image to playable world", "11B parameter foundation model" ], "useCases": [ "2D environment generation", "AI agent training", "Creative game design", "World model research" ], "strengths": [ "No action labels required", "Massive scale training", "Generative and interactive", "Foundation for Genie 2" ], "limitations": [ "2D environments only", "Low resolution output", "Short generation horizons", "Not publicly available" ], "benchmarks": [ { "name": "Controllability", "score": "Strong action following", "metric": "Action Accuracy", "numericValue": 78.5, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2402.15391" } ], "paperUrl": "https://arxiv.org/abs/2402.15391", "websiteUrl": null, "tags": [ "generative", "unsupervised", "interactive", "foundation-model" ], "featured": false, "relatedModels": [ "genie-2", "sora", "oasis" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "Bruce et al., 2024. Genie: Generative Interactive Environments. ICML 2024." ], "lastUpdated": "2026-03-14" }, { "id": "jepa", "name": "I-JEPA", "slug": "i-jepa", "company": "Meta", "lab": "Meta FAIR", "labSlug": "meta-fair", "year": 2023, "category": "Self-Supervised World Model", "subtype": "Image Joint Embedding Predictive Architecture", "worldModelType": "Self-supervised visual world model", "primaryDomain": "Image understanding", "status": "active", "architecture": "Vision Transformer with asymmetric masking and prediction in representation space", "modality": "Image", "trainingMethod": "Self-supervised prediction of masked image regions in abstract representation space", "shortDescription": "Image Joint Embedding Predictive Architecture: learns visual representations by predicting abstract image regions without pixel reconstruction.", "longDescription": "I-JEPA is the image-domain instantiation of Yann LeCun's JEPA framework. Instead of reconstructing pixels or using contrastive objectives, I-JEPA predicts the abstract representation of a target image block from the representation of a context block. This approach learns semantic visual features that capture high-level structure without being distracted by pixel-level details, following LeCun's vision for non-generative self-supervised world models.", "notableFeatures": [ "No pixel reconstruction or data augmentation", "Predicts in abstract representation space", "Follows LeCun's JEPA framework", "Learns semantic rather than pixel-level features" ], "useCases": [ "Image classification", "Object detection", "Visual representation learning", "Transfer learning" ], "strengths": [ "Learns semantic features", "No reconstruction artifacts", "Scalable ViT-based architecture", "Strong linear probing results" ], "limitations": [ "Image-only (no temporal dynamics)", "Not yet applied to RL or robotics", "Requires large compute for pre-training" ], "benchmarks": [ { "name": "ImageNet Linear Probe", "score": "Competitive with MAE", "metric": "Top-1 Accuracy", "numericValue": 81.1, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2301.08243" } ], "paperUrl": "https://arxiv.org/abs/2301.08243", "websiteUrl": null, "tags": [ "self-supervised", "jepa", "image", "representation-learning" ], "featured": false, "relatedModels": [ "v-jepa" ], "relatedPapers": [ "lecun-path-paper" ], "relatedConcepts": [ "self-supervised-world-models" ], "citations": [ "Assran et al., 2023. Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture. CVPR 2023." ], "lastUpdated": "2026-03-10" }, { "id": "GameNGen", "name": "GameNGen", "slug": "gamengen", "company": "Google Research", "lab": "Valevski et al.", "labSlug": "google-research", "year": 2024, "category": "Generative World Model", "subtype": "Neural Game Engine", "worldModelType": "Diffusion-based neural game engine", "primaryDomain": "Real-time game simulation", "status": "active", "architecture": "Fine-tuned Stable Diffusion 1.4 with action and frame history conditioning", "modality": "Actions + Frame History → Next Frame", "trainingMethod": "Two-phase: (1) RL agent generates gameplay data, (2) Diffusion model fine-tuned on gameplay sequences", "shortDescription": "The first neural model to simulate a complex game (DOOM) in real-time at high quality, making the game engine itself a neural network.", "longDescription": "GameNGen demonstrates that a neural network can fully replace a traditional game engine. It simulates the classic game DOOM in real-time at over 20 FPS with visual quality nearly indistinguishable from the original game. The system uses a fine-tuned Stable Diffusion model conditioned on past frames and player actions, trained via a two-phase approach: first an RL agent generates gameplay data, then the diffusion model learns to simulate the game.", "notableFeatures": [ "Real-time DOOM simulation at 20+ FPS", "Nearly indistinguishable from original game", "No traditional game engine needed", "Based on Stable Diffusion architecture" ], "useCases": [ "Neural game engines", "Game simulation research", "World model demonstrations", "Interactive AI systems" ], "strengths": [ "Impressive visual fidelity", "Real-time performance", "Proof-of-concept for neural game engines", "Playable and interactive" ], "limitations": [ "Single game only", "No long-term memory/consistency", "Based on fine-tuning existing model", "Not generalizable yet" ], "benchmarks": [ { "name": "DOOM Simulation", "score": "Near-original quality", "metric": "LPIPS", "numericValue": 0.072, "unit": "LPIPS↓", "sourceUrl": "https://arxiv.org/abs/2408.14837" } ], "paperUrl": "https://arxiv.org/abs/2408.14837", "websiteUrl": null, "tags": [ "neural-game-engine", "diffusion", "real-time", "doom" ], "featured": false, "relatedModels": [ "oasis", "diamond", "genie-2" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "simulation-engines" ], "citations": [ "Valevski et al., 2024. Diffusion Models Are Real-Time Game Engines. arXiv:2408.14837" ], "lastUpdated": "2026-03-11" }, { "id": "emu-video", "name": "Emu Video", "slug": "emu-video", "company": "Meta", "lab": "Meta GenAI", "labSlug": "meta-fair", "year": 2023, "category": "Generative World Model", "subtype": "Video Generation Model", "worldModelType": "Factorized text-to-video model", "primaryDomain": "Video generation", "status": "active", "architecture": "Two-stage diffusion model: text-to-image + image-to-video", "modality": "Text → Image → Video", "trainingMethod": "Factorized training: image diffusion + video diffusion with image conditioning", "shortDescription": "Meta's efficient video generation model using a factorized approach: first generate an image, then animate it into a video.", "longDescription": "Emu Video simplifies text-to-video generation by factorizing it into two steps: text-to-image generation followed by image-to-video animation. This factored approach reduces the complexity of direct text-to-video generation while producing high-quality results. The model uses a diffusion-based architecture and achieves strong results compared to commercial video generation systems.", "notableFeatures": [ "Factorized two-stage approach", "High visual quality", "Simpler than end-to-end methods", "Strong human evaluation results" ], "useCases": [ "Video content creation", "Animation from images", "Creative tools", "Visual storytelling" ], "strengths": [ "Simplified architecture", "High quality output", "Efficient factored approach", "Strong human preference scores" ], "limitations": [ "Not action-conditioned", "Short video clips only", "No interactive control" ], "benchmarks": [ { "name": "Human Evaluation", "score": "Preferred over competitors", "metric": "Human Preference", "numericValue": 81, "unit": "% preference", "sourceUrl": "https://arxiv.org/abs/2311.10709" } ], "paperUrl": "https://arxiv.org/abs/2311.10709", "websiteUrl": null, "tags": [ "video-generation", "diffusion", "factorized", "text-to-video" ], "featured": false, "relatedModels": [ "sora", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models" ], "citations": [ "Girdhar et al., 2023. Emu Video: Factorizing Text-to-Video Generation by Explicit Image Conditioning. arXiv:2311.10709" ], "lastUpdated": "2026-03-05" }, { "id": "3d-vla", "name": "3D-VLA", "slug": "3d-vla", "company": "MIT / Tsinghua", "lab": "Zhen et al.", "labSlug": "mit", "year": 2024, "category": "Foundation World Model", "subtype": "3D Vision-Language-Action Model", "worldModelType": "3D-aware embodied world model", "primaryDomain": "Robotics / Embodied AI", "status": "emerging", "architecture": "3D-aware vision-language model with integrated world model and action decoder", "modality": "3D Point Clouds + Language + Actions", "trainingMethod": "Multi-task training: 3D understanding, world modeling, and action generation", "shortDescription": "A 3D vision-language-action model with a built-in world model for embodied AI, enabling 3D-aware reasoning, planning, and action generation.", "longDescription": "3D-VLA integrates 3D perception, language understanding, and action generation with an explicit world model component. The model can reason about 3D scenes, predict future states, and generate robot actions, all within a unified architecture. By incorporating a world model, 3D-VLA can plan ahead by imagining consequences of actions in 3D space before executing them.", "notableFeatures": [ "Unified 3D perception + world model + action", "3D-aware future prediction", "Language-conditioned planning", "Embodied task execution" ], "useCases": [ "Robot manipulation", "3D scene understanding", "Language-guided robotics", "Embodied planning" ], "strengths": [ "Unified architecture", "3D-native understanding", "Integrated world model for planning", "Language-conditioned" ], "limitations": [ "Early-stage research", "Limited real-robot validation", "High compute requirements" ], "benchmarks": [], "paperUrl": "https://arxiv.org/abs/2403.09631", "websiteUrl": null, "tags": [ "3d", "vision-language-action", "robotics", "embodied-ai" ], "featured": false, "relatedModels": [ "ami-world-model", "td-mpc2", "unisim" ], "relatedPapers": [], "relatedConcepts": [ "embodied-ai", "simulation-engines" ], "citations": [ "Zhen et al., 2024. 3D-VLA: A 3D Vision-Language-Action Generative World Model." ], "lastUpdated": "2026-03-16" }, { "id": "rt-2", "name": "RT-2", "slug": "rt-2", "company": "Google DeepMind", "lab": "Brohan et al.", "labSlug": "deepmind", "year": 2023, "category": "Foundation World Model", "subtype": "Vision-Language-Action Model", "worldModelType": "Web-knowledge transfer model for robotics", "primaryDomain": "Robotics", "status": "active", "architecture": "PaLI-X / PaLM-E backbone fine-tuned with action tokenization", "modality": "Visual + Language + Proprioceptive", "trainingMethod": "Co-fine-tuning on web VLM data + robot demonstration trajectories", "shortDescription": "A vision-language-action model that transfers web-scale knowledge directly to robot control.", "longDescription": "RT-2 (Robotic Transformer 2) is a vision-language-action (VLA) model that co-fine-tunes a large vision-language model on both web data and robotic trajectories. By tokenizing robot actions as text tokens, RT-2 enables a single model to perform visual question answering, image captioning, and robot control simultaneously. It demonstrates emergent capabilities such as reasoning about previously unseen objects, interpreting abstract instructions, and performing multi-step manipulation tasks, all without task-specific training.", "notableFeatures": [ "Emergent reasoning about novel objects", "Action tokenization within language model", "Zero-shot generalization to unseen instructions", "Combines web knowledge with physical grounding" ], "useCases": [ "Robotic manipulation", "Instruction-following robots", "Object reasoning", "Multi-step task execution" ], "strengths": [ "Web-scale knowledge transfer", "No task-specific modules needed", "Handles novel objects and instructions", "Unified architecture for perception and action" ], "limitations": [ "Requires large-scale compute", "Limited to manipulation tasks in evaluation", "Single-robot arm setting", "Inference latency for real-time control" ], "benchmarks": [ { "name": "Emergent Skills Eval", "score": "62% success on novel objects", "metric": "Success Rate", "numericValue": 62, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2307.15818" }, { "name": "Language Table", "score": "90% success", "metric": "Task Success", "numericValue": 90, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2307.15818" } ], "paperUrl": "https://arxiv.org/abs/2307.15818", "websiteUrl": "https://robotics-transformer2.github.io/", "tags": [ "robotics", "vision-language-action", "foundation-model", "embodied-ai", "transfer-learning" ], "featured": true, "relatedModels": [ "3d-vla", "ami-world-model", "td-mpc2", "unipi" ], "relatedPapers": [], "relatedConcepts": [ "embodied-ai", "foundation-models" ], "citations": [ "Brohan et al., 2023. RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control. arXiv:2307.15818" ], "lastUpdated": "2026-04-07" }, { "id": "lwm", "name": "Large World Model (LWM)", "slug": "lwm", "company": "UC Berkeley", "lab": "Liu et al.", "labSlug": "uc-berkeley", "year": 2024, "category": "Foundation World Model", "subtype": "Long-Context Multimodal Model", "worldModelType": "Million-length video-language world model", "primaryDomain": "Video Understanding", "status": "active", "architecture": "LLaMA-based transformer with RingAttention for million-token context", "modality": "Visual (Video) + Language", "trainingMethod": "Progressive context extension on interleaved video-text data", "shortDescription": "A foundation model trained on 1M+ interleaved video and language tokens for long-horizon world understanding.", "longDescription": "The Large World Model (LWM) extends the context length of multimodal transformers to over one million tokens, enabling processing of long videos interleaved with text. Built on the LLaMA architecture with RingAttention for efficient long-context training, LWM demonstrates that scaling sequence length (not just model size) unlocks emergent world understanding capabilities including long-horizon video comprehension, temporal reasoning, and cross-modal physical dynamics prediction.", "notableFeatures": [ "1M+ token context length", "RingAttention for memory-efficient training", "Interleaved video-text understanding", "Emergent temporal reasoning" ], "useCases": [ "Long video understanding", "Temporal event reasoning", "Cross-modal question answering", "Video summarization" ], "strengths": [ "Unprecedented context length for video", "Unified video-language architecture", "Emergent world understanding", "Open-source weights" ], "limitations": [ "High compute requirements", "Limited action/planning capabilities", "Primarily passive understanding, not generation" ], "benchmarks": [ { "name": "Long Video QA", "score": "State-of-the-art", "metric": "Accuracy", "sourceUrl": "https://arxiv.org/abs/2402.08855" } ], "paperUrl": "https://arxiv.org/abs/2402.08855", "websiteUrl": "https://largeworldmodel.github.io/", "tags": [ "long-context", "multimodal", "video-understanding", "foundation-model", "open-source" ], "featured": false, "relatedModels": [ "v-jepa", "sora", "nvidia-cosmos", "ami-world-model" ], "relatedPapers": [], "relatedConcepts": [ "foundation-models", "video-world-models" ], "citations": [ "Liu et al., 2024. World Model on Million-Length Video And Language With RingAttention. arXiv:2402.08855" ], "lastUpdated": "2026-04-07" }, { "id": "stable-video-diffusion", "name": "Stable Video Diffusion", "slug": "stable-video-diffusion", "company": "Stability AI", "lab": "Blattmann et al.", "labSlug": "stability-ai", "year": 2023, "category": "Generative World Model", "subtype": "Video Diffusion Model", "worldModelType": "Image-to-video diffusion model", "primaryDomain": "Video Generation", "status": "active", "architecture": "3D UNet latent diffusion with temporal attention layers", "modality": "Visual (Image → Video)", "trainingMethod": "Multi-stage training: image pretraining → video fine-tuning on curated dataset", "shortDescription": "An open-source foundation model for video generation, producing temporally consistent video from a single image.", "longDescription": "Stable Video Diffusion (SVD) is an open-source latent video diffusion model built upon the Stable Diffusion image model. It generates short, temporally coherent video sequences from a single conditioning image. SVD demonstrates strong motion dynamics and physical plausibility, making it a foundational building block for researchers working on video generation, world simulation, and controllable content creation. Its open weights have enabled rapid community adoption and downstream applications.", "notableFeatures": [ "Open-source weights", "Image-to-video generation", "Temporally consistent motion", "Multi-stage curation pipeline for training data" ], "useCases": [ "Creative video generation", "Animation from stills", "Research on video dynamics", "Downstream video editing" ], "strengths": [ "Open-source and widely adopted", "Strong temporal coherence", "Good physical plausibility for short clips", "Efficient latent space architecture" ], "limitations": [ "Short video duration (2-4 seconds)", "Limited resolution", "No text-to-video (image conditioning only)", "Occasional temporal artifacts" ], "benchmarks": [ { "name": "FVD (UCF-101)", "score": "Competitive with closed-source models", "metric": "Fréchet Video Distance", "sourceUrl": "https://arxiv.org/abs/2311.15127" } ], "paperUrl": "https://arxiv.org/abs/2311.15127", "websiteUrl": "https://stability.ai/stable-video", "tags": [ "video-generation", "diffusion", "open-source", "image-to-video", "foundation-model" ], "featured": false, "relatedModels": [ "sora", "genie-2", "emu-video", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "diffusion-models" ], "citations": [ "Blattmann et al., 2023. Stable Video Diffusion: Scaling Latent Video Diffusion Models to Large Datasets. arXiv:2311.15127" ], "lastUpdated": "2026-04-07" }, { "id": "mile", "name": "MILE", "slug": "mile", "company": "Wayve", "lab": "Hu et al.", "labSlug": "wayve", "year": 2022, "category": "Foundation World Model", "subtype": "Driving World Model", "worldModelType": "Model-based imitation learning for driving", "primaryDomain": "Autonomous Driving", "status": "active", "architecture": "Variational autoencoder with spatial-temporal transformer for dynamics prediction", "modality": "Visual (Multi-camera) + Ego-state", "trainingMethod": "Model-based imitation learning with future imagination rollouts", "shortDescription": "A world model for autonomous driving that jointly learns dynamics, perception, and planning through model-based imitation learning.", "longDescription": "MILE (Model-based Imitation LEarning) is a world model for autonomous driving that learns a joint representation of scene dynamics, semantic segmentation, and ego-vehicle planning from camera-only input. Unlike modular driving stacks, MILE uses a learned world model to imagine future driving scenarios and optimize planning within that imagination, achieving state-of-the-art performance on the CARLA benchmark. It pioneered the concept of world-model-based end-to-end driving.", "notableFeatures": [ "Joint perception-prediction-planning", "Imagination-based planning", "Camera-only input", "End-to-end differentiable" ], "useCases": [ "Autonomous driving", "Driving scene prediction", "End-to-end planning", "Simulation-based training" ], "strengths": [ "Unified world model for driving", "State-of-the-art on CARLA", "Camera-only (no LiDAR needed)", "Imagination enables counterfactual reasoning" ], "limitations": [ "Evaluated primarily in simulation", "Single-city training data", "Limited to urban driving scenarios" ], "benchmarks": [ { "name": "CARLA Benchmark", "score": "State-of-the-art", "metric": "Driving Score", "numericValue": 81, "unit": "DS", "sourceUrl": "https://arxiv.org/abs/2210.07729" } ], "paperUrl": "https://arxiv.org/abs/2210.07729", "websiteUrl": "https://wayve.ai/thinking/mile/", "tags": [ "autonomous-driving", "imitation-learning", "end-to-end", "world-model", "planning" ], "featured": false, "relatedModels": [ "gaia-1", "copilot4d", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "autonomous-driving", "model-based-planning" ], "citations": [ "Hu et al., 2022. Model-Based Imitation Learning for Urban Driving. NeurIPS 2022. arXiv:2210.07729" ], "lastUpdated": "2026-04-07" }, { "id": "steve-1", "name": "STEVE-1", "slug": "steve-1", "company": "UT Austin", "lab": "Lifshitz et al.", "labSlug": "ut-austin", "year": 2023, "category": "Generative World Model", "subtype": "Instruction-Following Game Agent", "worldModelType": "Text-conditioned generative agent in open-world", "primaryDomain": "Game AI / Open World", "status": "active", "architecture": "VPT backbone + MineCLIP goal encoder + latent-conditioned policy", "modality": "Visual + Language (instructions)", "trainingMethod": "Hindsight relabeling on unlabeled gameplay + CLIP-based instruction conditioning", "shortDescription": "An instruction-following agent for Minecraft that uses a generative world model to execute open-ended text commands.", "longDescription": "STEVE-1 is a generative agent for Minecraft that follows open-ended text and visual instructions. Built on top of a VPT (Video Pre-Training) backbone, STEVE-1 uses a latent-conditioned policy that translates CLIP-based goal embeddings into action sequences. It demonstrates that combining large-scale video pre-training with instruction-conditioned generation creates agents capable of executing diverse, compositional tasks in open-world environments.", "notableFeatures": [ "Open-ended instruction following", "Minecraft open-world navigation", "CLIP-based goal conditioning", "No reward function needed" ], "useCases": [ "Open-world game playing", "Instruction-following agents", "Creative task execution", "Embodied language grounding" ], "strengths": [ "Handles diverse open-ended instructions", "No reward engineering needed", "Scales with video pre-training data", "Compositional task understanding" ], "limitations": [ "Minecraft-specific", "Limited long-horizon planning", "Requires large pre-training corpus", "Short action horizons" ], "benchmarks": [ { "name": "Programmatic Eval", "score": "Strong instruction-following", "metric": "Task Completion", "sourceUrl": "https://arxiv.org/abs/2306.00937" } ], "paperUrl": "https://arxiv.org/abs/2306.00937", "websiteUrl": "https://sites.google.com/view/steve-1", "tags": [ "minecraft", "instruction-following", "open-world", "generative-agent", "video-pretraining" ], "featured": false, "relatedModels": [ "dreamer-v3", "genie-1", "oasis", "gamengen" ], "relatedPapers": [], "relatedConcepts": [ "open-world-ai", "instruction-following" ], "citations": [ "Lifshitz et al., 2023. STEVE-1: A Generative Model for Text-to-Behavior in Minecraft. arXiv:2306.00937" ], "lastUpdated": "2026-04-07" }, { "id": "gen-3-alpha", "name": "Gen-3 Alpha", "slug": "gen-3-alpha", "company": "Runway", "lab": "Runway Research", "labSlug": "runway", "year": 2024, "category": "Generative World Model", "subtype": "Video Generation Model", "worldModelType": "Controllable video generation with scene understanding", "primaryDomain": "Video Generation", "status": "active", "architecture": "Proprietary multimodal transformer with temporal consistency mechanisms", "modality": "Text + Image → Video", "trainingMethod": "Large-scale video-image joint training with human feedback alignment", "shortDescription": "Runway's next-generation video model with fine-grained control over motion, style, and composition.", "longDescription": "Gen-3 Alpha is Runway's most advanced video generation model, representing a major leap in controllable AI video synthesis. It generates high-fidelity video with fine-grained control over camera motion, character consistency, and scene dynamics. Gen-3 Alpha demonstrates emergent understanding of physical interactions, temporal coherence, and artistic style transfer. Trained on a combination of video and image data with novel architectures for temporal modeling, it sets a new standard for creative and industrial video AI applications.", "notableFeatures": [ "Fine-grained motion control", "Consistent character generation", "High temporal coherence", "Text and image conditioning" ], "useCases": [ "Creative video production", "Advertising content", "Film pre-visualization", "Interactive media" ], "strengths": [ "Industry-leading controllability", "Near-cinematic visual quality", "Versatile conditioning options", "Strong physical plausibility" ], "limitations": [ "Closed-source", "Limited video duration", "API-only access", "High generation cost" ], "benchmarks": [], "paperUrl": "", "websiteUrl": "https://runwayml.com/research/gen-3-alpha", "tags": [ "video-generation", "creative-ai", "controllable-generation", "text-to-video" ], "featured": false, "relatedModels": [ "sora", "stable-video-diffusion", "emu-video", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "video-world-models", "diffusion-models" ], "citations": [ "Runway Research, 2024. Gen-3 Alpha: Next-Generation Video Model." ], "lastUpdated": "2026-04-07" }, { "id": "genie-3", "name": "Genie 3", "slug": "genie-3", "company": "Google DeepMind", "lab": "DeepMind", "labSlug": "deepmind", "year": 2025, "category": "Generative World Model", "subtype": "Interactive 3D World Model", "worldModelType": "Real-time interactive 3D environment generation", "primaryDomain": "3D World Generation", "status": "active", "architecture": "Autoregressive latent world model with spatiotemporal transformer", "modality": "Text → Interactive 3D Environment", "trainingMethod": "Large-scale internet video and 3D data pre-training with action-conditioned generation", "shortDescription": "Google DeepMind's general-purpose world model that generates interactive 3D environments from text prompts in real time at 24fps.", "longDescription": "Genie 3 is a general-purpose world model from Google DeepMind that generates an unprecedented diversity of interactive environments from text prompts. Users can navigate generated worlds in real time at 24 frames per second, with consistency maintained for several minutes at 720p resolution. Building on Genie 1 and Genie 2, this third generation dramatically expands the scope of generated worlds, from natural landscapes and ecosystems to architectural spaces and fantastical environments. The model demonstrates emergent understanding of physics, lighting, object permanence, and spatial relationships. Project Genie, the public product built on Genie 3, was made available to Google AI Ultra subscribers in January 2026.", "notableFeatures": [ "Real-time interactive 3D world generation at 24fps", "Text-to-world prompt interface", "720p resolution with multi-minute consistency", "Emergent physics and lighting understanding", "Publicly available via Project Genie" ], "useCases": [ "Game prototyping", "Creative world-building", "Architectural visualization", "Training data generation", "Embodied AI research" ], "strengths": [ "First real-time interactive world model", "Unprecedented environment diversity", "Consumer-facing product", "Strong physical plausibility" ], "limitations": [ "Consistency degrades after ~1 minute", "Closed-source", "Limited to Google AI Ultra subscribers", "Cannot export generated worlds" ], "benchmarks": [ { "name": "Interactive World Rate", "score": "Real-time at 24fps", "metric": "Frame Rate", "numericValue": 24, "unit": "fps", "sourceUrl": "https://deepmind.google/blog/genie-3-a-new-frontier-for-world-models/" }, { "name": "World Consistency Horizon", "score": "Up to ~1 minute", "metric": "Stable Rollout", "numericValue": 60, "unit": "seconds", "sourceUrl": "https://www.gamesindustry.biz/googles-genie-3-world-models-start-to-break-down-after-around-a-minute" } ], "paperUrl": "https://deepmind.google/blog/genie-3-a-new-frontier-for-world-models/", "websiteUrl": "https://deepmind.google/models/genie/", "tags": [ "3d-generation", "interactive", "real-time", "text-to-world", "foundation-model" ], "featured": true, "relatedModels": [ "genie-2", "genie-1", "nvidia-cosmos", "oasis" ], "relatedPapers": [], "relatedConcepts": [ "generative-world-models", "3d-generation" ], "citations": [ "Parker-Holder & Fruchter, 2025. Genie 3: A New Frontier for World Models. Google DeepMind Blog." ], "lastUpdated": "2026-04-10" }, { "id": "v-jepa-2", "name": "V-JEPA 2", "slug": "v-jepa-2", "company": "Meta", "lab": "Meta FAIR", "labSlug": "meta-fair", "year": 2025, "category": "Self-Supervised World Model", "subtype": "Video Prediction Model", "worldModelType": "Joint-embedding predictive world model for video understanding and robot planning", "primaryDomain": "Physical Reasoning & Robotics", "status": "active", "architecture": "Vision Transformer with joint-embedding predictive architecture, latent space prediction", "modality": "Video → Latent Predictions", "trainingMethod": "Self-supervised learning via latent space prediction from video, no pixel reconstruction", "shortDescription": "Meta FAIR's self-supervised video world model achieving state-of-the-art visual understanding and enabling zero-shot robot control.", "longDescription": "V-JEPA 2 (Video Joint Embedding Predictive Architecture 2) is a self-supervised foundation world model from Meta FAIR that learns to understand, predict, and plan from video without relying on pixel-level reconstruction. By predicting in a learned latent space rather than pixel space, V-JEPA 2 avoids the computational overhead and irrelevant detail of generative approaches. It achieves state-of-the-art performance on visual understanding benchmarks and demonstrates zero-shot robot control capabilities, planning actions in new environments without task-specific fine-tuning. The model introduces new physical reasoning benchmarks and is fully open-source, marking a major step toward Yann LeCun's vision of world-model-based AI.", "notableFeatures": [ "State-of-the-art visual understanding without pixel reconstruction", "Zero-shot robot planning in unseen environments", "New physical reasoning benchmarks", "Fully open-source model and weights", "Energy-efficient compared to generative world models" ], "useCases": [ "Robot manipulation planning", "Physical scene understanding", "Video prediction", "Embodied AI", "Action planning" ], "strengths": [ "No pixel reconstruction overhead", "Zero-shot transfer to robotics", "Open-source", "Strong physical reasoning", "Scales efficiently" ], "limitations": [ "Limited to visual modality", "Requires large-scale video data", "Robot experiments limited to manipulation tasks" ], "benchmarks": [ { "name": "Physical Reasoning", "score": "State-of-the-art", "metric": "PhyBench Accuracy", "numericValue": 89.2, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2506.09985" } ], "paperUrl": "https://arxiv.org/abs/2506.09985", "websiteUrl": "https://ai.meta.com/research/vjepa/", "tags": [ "self-supervised", "jepa", "video-prediction", "robotics", "open-source", "physical-reasoning" ], "featured": true, "relatedModels": [ "v-jepa", "i-jepa", "ami-world-model" ], "relatedPapers": [], "relatedConcepts": [ "self-supervised-learning", "latent-dynamics", "embodied-ai" ], "citations": [ "Assran et al., 2025. V-JEPA 2: Self-Supervised Video Models Enable Understanding, Prediction and Planning. arXiv:2506.09985." ], "lastUpdated": "2026-04-10" }, { "id": "leworldmodel", "name": "LeWorldModel", "slug": "leworldmodel", "company": "Mila / NYU / Samsung SAIL", "lab": "Mila / NYU / Samsung SAIL", "labSlug": "mila-nyu-samsung-sail", "year": 2026, "category": "Self-Supervised World Model", "subtype": "Compact JEPA Model", "worldModelType": "Compact joint-embedding predictive world model for physical understanding", "primaryDomain": "Physical Reasoning", "status": "active", "architecture": "Compact Vision Transformer with JEPA and SigReg regularization", "modality": "Visual → Latent Predictions", "trainingMethod": "Self-supervised JEPA training with SigReg collapse prevention, single-GPU trainable", "shortDescription": "A compact 15M-parameter JEPA world model that learns real-world physics on a single GPU, solving the notorious representation collapse problem.", "longDescription": "LeWorldModel (LeWM) is a compact JEPA-based world model with only 15 million parameters that achieves physical understanding and planning up to 48× faster than foundation-model-based alternatives. Developed by researchers from Mila, NYU, and Samsung SAIL, it introduces SigReg, a novel regularization technique that solves the representation collapse problem that has plagued JEPA architectures. LeWM demonstrates that large-scale foundation models are not strictly necessary for physical reasoning: efficient, well-regularized architectures can learn meaningful world representations from pixels on commodity hardware.", "notableFeatures": [ "Only 15M parameters", "Trainable on a single GPU", "SigReg solves JEPA collapse problem", "48× faster planning than foundation models", "End-to-end from pixels" ], "useCases": [ "Physical scene understanding", "Efficient robot planning", "Resource-constrained deployment", "Research prototyping" ], "strengths": [ "Extremely compact", "Single-GPU training", "Solves collapse problem", "Fast inference", "Open research" ], "limitations": [ "Limited to simple physical scenarios", "Early-stage research", "Not yet tested at scale" ], "benchmarks": [ { "name": "Planning Efficiency", "score": "Up to 48x faster", "metric": "Planning Speedup", "numericValue": 48, "unit": "x", "sourceUrl": "https://arxiv.org/abs/2603.19312" }, { "name": "Training Footprint", "score": "Single-GPU trainable", "metric": "Hardware Requirement", "numericValue": 1, "unit": "GPU", "sourceUrl": "https://arxiv.org/abs/2603.19312" } ], "paperUrl": "https://arxiv.org/abs/2603.19312", "websiteUrl": null, "tags": [ "jepa", "compact", "efficient", "physical-reasoning", "sigreg", "single-gpu" ], "featured": false, "relatedModels": [ "v-jepa", "v-jepa-2", "i-jepa", "ami-world-model" ], "relatedPapers": [], "relatedConcepts": [ "self-supervised-learning", "latent-dynamics" ], "citations": [ "LeCun et al., 2026. LeWorldModel: Stable End-to-End Joint-Embedding Predictive Architecture from Pixels. arXiv:2603.19312." ], "lastUpdated": "2026-04-10" }, { "id": "pixverse-r1", "name": "PixVerse R1", "slug": "pixverse-r1", "company": "PixVerse", "lab": "PixVerse Research", "labSlug": "pixverse", "year": 2026, "category": "Generative World Model", "subtype": "Real-time Multiplayer World Model", "worldModelType": "Real-time interactive world generation with multi-user participation", "primaryDomain": "Interactive Entertainment", "status": "active", "architecture": "Real-time generative world model with multi-stream rendering", "modality": "Text/Image → Interactive Shared World", "trainingMethod": "Large-scale video and 3D data training with real-time streaming optimization", "shortDescription": "The first real-time world model supporting multi-user shared worlds with personalized avatars and no session limits.", "longDescription": "PixVerse R1 is a real-time interactive world model that introduces multi-user shared worlds, a first in the field. Users can enter AI-generated environments as personalized avatars and explore alongside others through a continuous shared livestream, with no session time limits. The model generates interactive worlds from text or image prompts while maintaining real-time consistency for multiple simultaneous participants. R1 represents a significant shift from single-user world models toward collaborative, persistent AI-generated environments.", "notableFeatures": [ "Multi-user shared worlds", "Personalized avatar generation", "No session time limits", "Real-time continuous livestream", "Text and image conditioning" ], "useCases": [ "Social virtual worlds", "Collaborative exploration", "Interactive entertainment", "Creative prototyping" ], "strengths": [ "First multiplayer world model", "No session limits", "Real-time generation", "Avatar personalization" ], "limitations": [ "Early-stage product", "Limited physical accuracy", "Closed-source" ], "benchmarks": [ { "name": "Shared World Sessions", "score": "Multi-user real time", "metric": "Concurrent Participation", "sourceUrl": "https://pixverse.ai" }, { "name": "Session Duration", "score": "No session limits", "metric": "Runtime Constraint", "sourceUrl": "https://pixverse.ai" } ], "paperUrl": "", "websiteUrl": "https://pixverse.ai", "tags": [ "real-time", "multiplayer", "interactive", "shared-worlds", "avatars" ], "featured": false, "relatedModels": [ "genie-3", "oasis", "genie-2" ], "relatedPapers": [], "relatedConcepts": [ "generative-world-models", "3d-generation" ], "citations": [ "PixVerse, 2026. PixVerse R1: Real-time World Model with Shared Worlds." ], "lastUpdated": "2026-04-10" }, { "id": "marble", "name": "Marble", "slug": "marble", "company": "World Labs", "lab": "World Labs", "labSlug": "world-labs", "year": 2026, "category": "Foundation World Model", "subtype": "Spatially Consistent 3D World Generator", "worldModelType": "Persistent multimodal 3D world model", "primaryDomain": "3D World Generation", "status": "active", "architecture": "Proprietary multimodal 3D world generation system", "modality": "Text/Image/Video/360 -> Persistent 3D World", "trainingMethod": "Large-scale multimodal world understanding and 3D scene generation", "shortDescription": "World Labs' multimodal world model for generating spatially consistent, persistent 3D environments from text, images, video, and 360 inputs.", "longDescription": "Marble is World Labs' world model for building persistent 3D scenes from multimodal prompts. It is designed to maintain spatial consistency while turning text, images, video, or 360 captures into navigable worlds that can be exported in both 2D and 3D forms. The system emphasizes world persistence, scene coherence, and practical creator workflows rather than benchmark-driven control tasks, making it a notable entrant in the frontier of interactive world-building tools.", "notableFeatures": [ "Spatially consistent world generation", "Persistent 3D environments", "Supports text, image, video, and 360 inputs", "Exports to both 2D and 3D workflows" ], "useCases": [ "3D prototyping", "Game environment ideation", "Virtual production", "Spatial storytelling" ], "strengths": [ "Strong world persistence", "Broad multimodal input support", "Creator-oriented workflow", "Clear 3D positioning" ], "limitations": [ "Few public technical details", "Closed platform", "Limited evidence on control and planning tasks" ], "benchmarks": [], "paperUrl": "https://www.worldlabs.ai/", "websiteUrl": "https://www.worldlabs.ai/", "tags": [ "3d-generation", "multimodal", "persistent-worlds", "creator-tools", "foundation-model" ], "featured": true, "relatedModels": [ "genie-3", "nvidia-cosmos", "oasis" ], "relatedPapers": [], "relatedConcepts": [ "generative-world-models", "3d-generation" ], "citations": [ "World Labs, 2026. Marble world model platform." ], "lastUpdated": "2026-06-12" }, { "id": "1x-world-model", "name": "1X World Model", "slug": "1x-world-model", "company": "1X", "lab": "1X", "labSlug": "1x", "year": 2025, "category": "Foundation World Model", "subtype": "Humanoid Robot Video World Model", "worldModelType": "Physics-grounded action-conditioned video world model", "primaryDomain": "Humanoid Robotics", "status": "active", "architecture": "Physics-grounded action-conditioned video model for humanoid control", "modality": "Robot Video + Actions -> Future Observations", "trainingMethod": "Embodied video prediction grounded in robot interactions and failure-aware learning", "shortDescription": "1X's physics-grounded video world model for anticipating the outcomes of NEO's actions and supporting generalization to unseen household tasks.", "longDescription": "The 1X World Model is a video world model used inside 1X's humanoid robotics stack for NEO. According to 1X, it predicts the outcomes of the robot's actions before execution in the real world and now serves as NEO's cognitive core for generalizing to previously unseen tasks. Its emphasis is practical embodied deployment: using a world model as a decision substrate for real household robotics rather than as a benchmark-only research artifact.", "notableFeatures": [ "Used as NEO's cognitive core", "Predicts outcomes before real execution", "Grounded in real robot interaction", "Targets unseen household tasks" ], "useCases": [ "Humanoid task planning", "Action evaluation", "Household robotics", "Embodied generalization" ], "strengths": [ "Tied to real robot deployment", "Embodied action grounding", "Practical household focus", "Clear decision-support role" ], "limitations": [ "Limited public technical detail", "No peer-reviewed benchmark suite yet", "Closed product stack" ], "benchmarks": [], "paperUrl": "https://www.1x.tech/ai", "websiteUrl": "https://www.1x.tech/ai", "tags": [ "humanoid-robotics", "video-world-models", "physical-ai", "embodied-ai", "foundation-model" ], "featured": true, "relatedModels": [ "ami-world-model", "rt-2", "v-jepa-2" ], "relatedPapers": [], "relatedConcepts": [ "embodied-ai", "video-world-models" ], "citations": [ "1X, 2025-2026. 1X World Model updates for NEO." ], "lastUpdated": "2026-06-12" }, { "id": "playworld", "name": "PlayWorld", "slug": "playworld", "company": "Princeton University", "lab": "Princeton University", "labSlug": "princeton", "year": 2026, "category": "Generative World Model", "subtype": "Action-Conditioned Robot Video Model", "worldModelType": "Robot manipulation world simulator learned from autonomous play", "primaryDomain": "Robot Manipulation", "status": "active", "architecture": "Action-conditioned video world model trained on autonomous robot play data", "modality": "Multi-view Robot Video + Actions -> Future Observations", "trainingMethod": "Autonomous self-play data collection with action-conditioned video model training", "shortDescription": "A Princeton robot world model trained from autonomous self-play to simulate contact-rich manipulation and support policy evaluation and RL fine-tuning.", "longDescription": "PlayWorld is a robot world model from Princeton University that learns action-conditioned video simulation from autonomous robot play rather than only curated demonstrations. The project focuses on contact-rich manipulation, a regime where many video world models still hallucinate dynamics. By collecting large-scale unsupervised play data, PlayWorld improves physical consistency, supports fine-grained policy evaluation, and enables reinforcement learning inside the learned world model for real-world policy improvement.", "notableFeatures": [ "Learns from unsupervised robot self-play", "Targets contact-rich interaction fidelity", "Supports policy evaluation", "Enables RL fine-tuning in the learned world model" ], "useCases": [ "Robot manipulation simulation", "Policy evaluation", "Failure prediction", "World-model-based fine-tuning" ], "strengths": [ "Strong contact-rich focus", "Real-world policy improvement evidence", "Scalable autonomous data collection", "Clear robotics utility" ], "limitations": [ "Academic research stage", "Narrower domain than generalist foundation models", "Limited public deployment tooling" ], "benchmarks": [ { "name": "Policy Evaluation Improvement", "score": "Up to 40% improvement over human-collected data", "metric": "Relative Gain", "numericValue": 40, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2603.09030" }, { "name": "Real-World Success Rate", "score": "Up to 65% improvement", "metric": "Success Rate Gain", "numericValue": 65, "unit": "%", "sourceUrl": "https://arxiv.org/abs/2603.09030" } ], "paperUrl": "https://arxiv.org/abs/2603.09030", "websiteUrl": "https://robot-playworld.github.io/", "tags": [ "robotics", "self-play", "video-world-models", "manipulation", "policy-evaluation" ], "featured": true, "relatedModels": [ "td-mpc2", "unisim", "1x-world-model", "nvidia-cosmos" ], "relatedPapers": [], "relatedConcepts": [ "world-models-robotics", "generative-world-models" ], "citations": [ "Yin et al., 2026. PlayWorld: Learning Robot World Models from Autonomous Play. arXiv:2603.09030." ], "lastUpdated": "2026-06-12" } ] }