@article{brunton2020machine,
title={Machine learning for fluid mechanics},
author={Brunton, Steven L and Noack, Bernd R and Koumoutsakos, Petros},
journal={Annual review of fluid mechanics},
volume={52},
pages={477--508},
year={2020},
category={fluid, survey},
publisher={Annual Reviews},
link={https://www.annualreviews.org/content/journals/10.1146/annurev-fluid-010719-060214},
abstract={This video gives an overview of how Machine Learning is being used in Fluid Mechanics. In fact, fluid mechanics is one of the original 'big data' sciences, and many advances in ML came out of fluids.}
}
@article{xie2018tempogan,
title={tempoGAN: A temporally coherent, volumetric GAN for super-resolution fluid flow},
author={Xie, You and Franz, Erik and Chu, Mengyu and Thuerey, Nils},
journal={ACM Transactions on Graphics (TOG)},
volume={37},
number={4},
pages={1--15},
year={2018},
category={fluid},
tag={super_resolution,gan},
publisher={ACM New York, NY, USA},
link={https://ge.in.tum.de/publications/tempogan/},
abstract={We propose a temporally coherent generative model addressing the super-resolution problem for fluid flows. Our work represents a first approach to synthesize four-dimensional physics fields with neural networks. Based on a conditional generative adversarial network that is designed for the inference of three-dimensional volumetric data, our model generates consistent and detailed results by using a novel temporal discriminator, in addition to the commonly used spatial one. Our experiments show that the generator is able to infer more realistic high-resolution details by using additional physical quantities, such as low-resolution velocities or vorticities. Besides improvements in the training process and in the generated outputs, these inputs offer means for artistic control as well. We additionally employ a physics-aware data augmentation step, which is crucial to avoid overfitting and to reduce memory requirements. In this way, our network learns to generate advected quantities with highly detailed, realistic, and temporally coherent features. Our method works instantaneously, using only a single time-step of low-resolution fluid data. We demonstrate the abilities of our method using a variety of complex inputs and applications in two and three dimensions.}
}
@inproceedings{ummenhofer2019lagrangian,
title={Lagrangian fluid simulation with continuous convolutions},
author={Ummenhofer, Benjamin and Prantl, Lukas and Thuerey, Nils and Koltun, Vladlen},
booktitle={International Conference on Learning Representations},
year={2019},
category={fluid, neural_solver},
link={https://ge.in.tum.de/publications/2020-ummenhofer-iclr/},
abstract={We present an approach to Lagrangian fluid simulation with a new type of convolutional network. Our networks process sets of moving particles, which describe fluids in space and time. Unlike previous approaches, we do not build an explicit graph structure to connect the particles but use spatial convolutions as the main differentiable operation that relates particles to their neighbors. To this end we present a simple, novel, and effective extension of N-D convolutions to the continuous domain. We show that our network architecture can simulate different materials, generalizes to arbitrary collision geometries, and can be used for inverse problems. In addition, we demonstrate that our continuous convolutions outperform prior formulations in terms of accuracy and speed.}
}
@article{chu2021learning,
title={Learning meaningful controls for fluids},
author={Chu, Mengyu and Thuerey, Nils and Seidel, Hans-Peter and Theobalt, Christian and Zayer, Rhaleb},
journal={ACM Transactions on Graphics (TOG)},
volume={40},
number={4},
pages={1--13},
year={2021},
category={fluid,neural_representation,control},
tag={fluid_control,gan},
publisher={ACM New York, NY, USA},
link={https://rachelcmy.github.io/den2vel/},
abstract={While modern fluid simulation methods achieve high-quality simulation results, it is still a big challenge to interpret and control motion from visual quantities, such as the advected marker density. These visual quantities play an important role in user interactions: Being familiar and meaningful to humans, these quantities have a strong correlation with the underlying motion. We propose a novel data-driven conditional adversarial model that solves the challenging and theoretically ill-posed problem of deriving plausible velocity fields from a single frame of a density field. Besides density modifications, our generative model is the first to enable the control of the results using all of the following control modalities: obstacles, physical parameters, kinetic energy, and vorticity. Our method is based on a new conditional generative adversarial neural network that explicitly embeds physical quantities into the learned latent space, and a new cyclic adversarial network design for control disentanglement. We show the high quality and versatile controllability of our results for density-based inference, realistic obstacle interaction, and sensitive responses to modifications of physical parameters, kinetic energy, and vorticity. Code, models, and results can be found at https://github.com/RachelCmy/den2vel.}
}
@article{li2023mpmnet,
title={MPMNet: A data-driven MPM framework for dynamic fluid-solid interaction},
author={Li, Jin and Gao, Yang and Dai, Ju and Li, Shuai and Hao, Aimin and Qin, Hong},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2023},
category={multiphys,neural_solver,engine},
publisher={IEEE},
link={https://ieeexplore.ieee.org/document/10113697},
abstract={High-accuracy, high-efficiency physics-based fluid-solid interaction is essential for reality modeling and computer animation in online games or real-time Virtual Reality (VR) systems. However, the large-scale simulation of incompressible fluid and its interaction with the surrounding solid environment is either time-consuming or suffering from the reduced time/space resolution due to the complicated iterative nature pertinent to numerical computations of involved Partial Differential Equations (PDEs). In recent years, we have witnessed significant growth in exploring a different, alternative data-driven approach to addressing some of the existing technical challenges in conventional model-centric graphics and animation methods. This article showcases some of our exploratory efforts in this direction. One technical concern of our research is to address the central key challenge of how to best construct the numerical solver effectively and how to best integrate spatiotemporal/dimensional neural networks with the available MPM's pressure solvers. In particular, we devise the MPMNet, a hybrid data-driven framework supporting the popular and powerful MPM, to combine the comprehensive properties of MPM in numerically handling physical behaviors ranging from fluid to deformable solids and the high efficiency of data-driven models. At the architectural level, our MPMNet comprises three primary components: A data processing module to describe the physical properties by way of the input fields; A deep neural network group to learn the spatiotemporal features; And an iterative refinement process to continue to reduce possible numerical errors. The goal of these special technical developments is to aim at involved numerical acceleration while preserving physical accuracy, realizing efficient and accurate fluid-solid interactions in a data-driven fashion. The extensive experimental results verify that our MPMNet can tremendously speed up the computation compared with the popular numerical methods as the complexity of interaction scenes increases while better retaining the numerical accuracy.}
}
@article{deng2023fluid,
title={Fluid Simulation on Neural Flow Maps},
author={Deng, Yitong and Yu, Hong-Xing and Zhang, Diyang and Wu, Jiajun and Zhu, Bo},
journal={ACM Transactions on Graphics (TOG)},
volume={42},
number={6},
pages={1--21},
year={2023},
category={fluid,neural_representation},
tag={neural_field},
publisher={ACM New York, NY, USA},
link={https://yitongdeng-projects.github.io/neural_flow_maps_webpage/},
abstract={We introduce Neural Flow Maps, a novel simulation method bridging the emerging paradigm of implicit neural representations with fluid simulation based on the theory of flow maps, to achieve state-of-the-art simulation of in-viscid fluid phenomena. We devise a novel hybrid neural field representation, Spatially Sparse Neural Fields (SSNF), which fuses small neural networks with a pyramid of overlapping, multi-resolution, and spatially sparse grids, to compactly represent long-term spatiotemporal velocity fields at high accuracy. With this neural velocity buffer in hand, we compute long-term, bidirectional flow maps and their Jacobians in a mechanistically symmetric manner, to facilitate drastic accuracy improvement over existing solutions. These long-range, bidirectional flow maps enable high advection accuracy with low dissipation, which in turn facilitates high-fidelity incompressible flow simulations that manifest intricate vortical structures. We demonstrate the efficacy of our neural fluid simulation in a variety of challenging simulation scenarios, including leapfrogging vortices, colliding vortices, vortex reconnections, as well as vortex generation from moving obstacles and density differences. Our examples show increased performance over existing methods in terms of energy conservation, visual complexity, adherence to experimental observations, and preservation of detailed vortical structures.}
}
@inproceedings{fulton2019latent,
title={Latent-space dynamics for reduced deformable simulation},
author={Fulton, Lawson and Modi, Vismay and Duvenaud, David and Levin, David IW and Jacobson, Alec},
booktitle={Computer graphics forum},
volume={38},
number={2},
pages={379--391},
year={2019},
category={softbody,neural_representation},
organization={Wiley Online Library},
link={https://www.dgp.toronto.edu/projects/latent-space-dynamics/},
abstract={Abstract We propose the first reduced model simulation framework for deformable solid dynamics using autoencoder neural networks. We provide a data‐driven approach to generating nonlinear reduced spaces for deformation dynamics. In contrast to previous methods using machine learning which accelerate simulation by approximating the time‐stepping function, we solve the true equations of motion in the latent‐space using a variational formulation of implicit integration. Our approach produces drastically smaller reduced spaces than conventional linear model reduction, improving performance and robustness. Furthermore, our method works well with existing force‐approximation cubature methods.}
}
@inproceedings{sanchez2020learning,
title={Learning to simulate complex physics with graph networks},
author={Sanchez-Gonzalez, Alvaro and Godwin, Jonathan and Pfaff, Tobias and Ying, Rex and Leskovec, Jure and Battaglia, Peter},
booktitle={International conference on machine learning},
pages={8459--8468},
year={2020},
category={multiphys,neural_solver},
tag={gnn},
organization={PMLR},
link={https://sites.google.com/view/learning-to-simulate},
abstract={Here we present a machine learning framework and model implementation that can learn to simulate a wide variety of challenging physical domains, involving fluids, rigid solids, and deformable materials interacting with one another. Our framework---which we term "Graph Network-based Simulators" (GNS)---represents the state of a physical system with particles, expressed as nodes in a graph, and computes dynamics via learned message-passing. Our results show that our model can generalize from single-timestep predictions with thousands of particles during training, to different initial conditions, thousands of timesteps, and at least an order of magnitude more particles at test time. Our model was robust to hyperparameter choices across various evaluation metrics: the main determinants of long-term performance were the number of message-passing steps, and mitigating the accumulation of error by corrupting the training data with noise. Our GNS framework advances the state-of-the-art in learned physical simulation, and holds promise for solving a wide range of complex forward and inverse problems.}
}
@article{srinivasan2021learning,
title={Learning active quasistatic physics-based models from data},
author={Srinivasan, Sangeetha Grama and Wang, Qisi and Rojas, Junior and Kl{\'a}r, Gergely and Kavan, Ladislav and Sifakis, Eftychios},
journal={ACM Transactions on Graphics (TOG)},
volume={40},
number={4},
pages={1--14},
year={2021},
category={softbody,differentiable_simulation,embodied_ai,avatar},
tag={user_interaction},
publisher={ACM New York, NY, USA},
link={https://pages.cs.wisc.edu/~qisiw/SIG.html},
abstract={Humans and animals can control their bodies to generate a wide range of motions via low-dimensional action signals representing high-level goals. As such, human bodies and faces are prime examples of active objects, which can affect their shape via an internal actuation mechanism. This paper explores the following proposition: given a training set of example poses of an active deformable object, can we learn a low-dimensional control space that could reproduce the training set and generalize to new poses? In contrast to popular machine learning methods for dimensionality reduction such as auto-encoders, we model our active objects in a physics-based way. We utilize a differentiable, quasistatic, physics-based simulation layer and combine it with a decoder-type neural network. Our differentiable physics layer naturally fits into deep learning frameworks and allows the decoder network to learn actuations that reach the desired poses after physics-based simulation. In contrast to modeling approaches where users build anatomical models from first principles, medical literature or medical imaging, we do not presume knowledge of the underlying musculature, but learn the structure and control of the actuation mechanism directly from the input data. We present a training paradigm and several scalability-oriented enhancements that allow us to train effectively while accommodating high-resolution volumetric models, with as many as a quarter million simulation elements. The prime demonstration of the efficacy of our example-driven modeling framework targets facial animation, where we train on a collection of input expressions while generalizing to unseen poses, drive detailed facial animation from sparse motion capture input, and facilitate expression sculpting via direct manipulation.}
}
@article{yang2022implicit,
title={Implicit neural representation for physics-driven actuated soft bodies},
author={Yang, Lingchen and Kim, Byungsoo and Zoss, Gaspard and G{\"o}zc{\"u}, Baran and Gross, Markus and Solenthaler, Barbara},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={4},
pages={1--10},
year={2022},
category={softbody,differentiable_simulation,neural_representation,avatar},
tag={neural_field},
publisher={ACM New York, NY, USA},
link={https://people.inf.ethz.ch/zossg/publication/yang-2022/},
abstract={Active soft bodies can affect their shape through an internal actuation mechanism that induces a deformation. Similar to recent work, this paper utilizes a differentiable, quasi-static, and physics-based simulation layer to optimize for actuation signals parameterized by neural networks. Our key contribution is a general and implicit formulation to control active soft bodies by defining a function that enables a continuous mapping from a spatial point in the material space to the actuation value. This property allows us to capture the signal's dominant frequencies, making the method discretization agnostic and widely applicable. We extend our implicit model to mandible kinematics for the particular case of facial animation and show that we can reliably reproduce facial expressions captured with high-quality capture systems. We apply the method to volumetric soft bodies, human poses, and facial expressions, demonstrating artist-friendly properties, such as simple control over the latent space and resolution invariance at test time.}
}
@article{bertiche2022neural,
title={Neural cloth simulation},
author={Bertiche, Hugo and Madadi, Meysam and Escalera, Sergio},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={6},
pages={1--14},
year={2022},
category={cloth,neural_solver},
publisher={ACM New York, NY, USA},
link={https://hbertiche.github.io/NeuralClothSim/},
abstract={We present a general framework for the garment animation problem through unsupervised deep learning inspired in physically based simulation. Existing trends in the literature already explore this possibility. Nonetheless, these approaches do not handle cloth dynamics. Here, we propose the first methodology able to learn realistic cloth dynamics unsupervisedly, and henceforth, a general formulation for neural cloth simulation. The key to achieve this is to adapt an existing optimization scheme for motion from simulation based methodologies to deep learning. Then, analyzing the nature of the problem, we devise an architecture able to automatically disentangle static and dynamic cloth subspaces by design. We will show how this improves model performance. Additionally, this opens the possibility of a novel motion augmentation technique that greatly improves generalization. Finally, we show it also allows to control the level of motion in the predictions. This is a useful, never seen before, tool for artists. We provide of detailed analysis of the problem to establish the bases of neural cloth simulation and guide future research into the specifics of this domain.}
}
@article{xiang2022dressing,
title={Dressing avatars: Deep photorealistic appearance for physically simulated clothing},
author={Xiang, Donglai and Bagautdinov, Timur and Stuyck, Tuur and Prada, Fabian and Romero, Javier and Xu, Weipeng and Saito, Shunsuke and Guo, Jingfan and Smith, Breannan and Shiratori, Takaaki and others},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={6},
pages={1--15},
year={2022},
category={cloth,avatar},
publisher={ACM New York, NY, USA},
link={https://research.facebook.com/publications/dressing-avatars-deep-photorealistic-appearance-for-physically-simulated-clothing/},
abstract={Despite recent progress in developing animatable full-body avatars, realistic modeling of clothing - one of the core aspects of human self-expression - remains an open challenge. State-of-the-art physical simulation methods can generate realistically behaving clothing geometry at interactive rates. Modeling photorealistic appearance, however, usually requires physically-based rendering which is too expensive for interactive applications. On the other hand, data-driven deep appearance models are capable of efficiently producing realistic appearance, but struggle at synthesizing geometry of highly dynamic clothing and handling challenging body-clothing configurations. To this end, we introduce pose-driven avatars with explicit modeling of clothing that exhibit both photorealistic appearance learned from real-world data and realistic clothing dynamics. The key idea is to introduce a neural clothing appearance model that operates on top of explicit geometry: at training time we use high-fidelity tracking, whereas at animation time we rely on physically simulated geometry. Our core contribution is a physically-inspired appearance network, capable of generating photorealistic appearance with view-dependent and dynamic shadowing effects even for unseen body-clothing configurations. We conduct a thorough evaluation of our model and demonstrate diverse animation results on several subjects and different types of clothing. Unlike previous work on photorealistic full-body avatars, our approach can produce much richer dynamics and more realistic deformations even for many examples of loose clothing. We also demonstrate that our formulation naturally allows clothing to be used with avatars of different people while staying fully animatable, thus enabling, for the first time, photorealistic avatars with novel clothing.}
}
@article{lee2023clothcombo,
title={ClothCombo: Modeling Inter-Cloth Interaction for Draping Multi-Layered Clothes},
author={Lee, Dohae and Kang, Hyun and Lee, In-Kwon},
journal={ACM Transactions on Graphics (TOG)},
volume={42},
number={6},
pages={1--13},
year={2023},
category={cloth, neural_solver, contact},
tag={gnn},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/10.1145/3618376},
abstract={We present ClothCombo, a pipeline to drape arbitrary combinations of clothes on 3D human models with varying body shapes and poses. While existing learning-based approaches for draping clothes have shown promising results, multi-layered clothing remains challenging as it is non-trivial to model inter-cloth interaction. To this end, our method utilizes a GNN-based network to efficiently model the interaction between clothes in different layers, thus enabling multi-layered clothing. Specifically, we first create feature embedding for each cloth using a topology-agnostic network. Then, the draping network deforms all clothes to fit the target body shape and pose without considering inter-cloth interaction. Lastly, the untangling network predicts the per-vertex displacements in a way that resolves interpenetration between clothes. In experiments, the proposed model demonstrates strong performance in complex multi-layered scenarios. Being agnostic to cloth topology, our method can be readily used for layered virtual try-on of real clothes in diverse poses and combinations of clothes.}
}
@article{takahashi2021differentiable,
title={Differentiable Fluids with Solid Coupling for Learning and Control},
volume={35},
url={https://ojs.aaai.org/index.php/AAAI/article/view/16764},
DOI={10.1609/aaai.v35i7.16764},
number={7},
journal={Proceedings of the AAAI Conference on Artificial Intelligence},
author={Takahashi, Tetsuya and Liang, Junbang and Qiao, Yi-Ling and Lin, Ming C.},
year={2021},
category={multiphys,differentiable_simulation,neural_solver,control},
tag={reinforce_learning},
month={May},
pages={6138-6146},
link={https://ojs.aaai.org/index.php/AAAI/article/view/16764},
abstract={We introduce an efficient differentiable fluid simulator that can be integrated with deep neural networks as a part of layers for learning dynamics and solving control problems. It offers the capability to handle one-way coupling of fluids with rigid objects using a variational principle that naturally enforces necessary boundary conditions at the fluid-solid interface with sub-grid details. This simulator utilizes the adjoint method to efficiently compute the gradient for multiple time steps of fluid simulation with user defined objective functions. We demonstrate the effectiveness of our method for solving inverse and control problems on fluids with one-way coupled solids. Our method outperforms the previous gradient computations, state-of-the-art derivative-free optimization, and model-free reinforcement learning techniques by at least one order of magnitude.}
}
@inproceedings{zong2023neural,
title={Neural Stress Fields for Reduced-order Elastoplasticity and Fracture},
author={Zong, Zeshun and Li, Xuan and Li, Minchen and Chiaramonte, Maurizio M and Matusik, Wojciech and Grinspun, Eitan and Carlberg, Kevin and Jiang, Chenfanfu and Chen, Peter Yichen},
booktitle={SIGGRAPH Asia 2023 Conference Papers},
pages={1--11},
year={2023},
category={fluid,neural_representation,fracture},
tag={neural_field},
link={https://zeshunzong.github.io/reduced-order-mpm/},
abstract={We propose a hybrid neural network and physics framework for reduced-order modeling of elastoplasticity and fracture. State-of-the-art scientific computing models like the Material Point Method (MPM) faithfully simulate large-deformation elastoplasticity and fracture mechanics. However, their long runtime and large memory consumption render them unsuitable for applications constrained by computation time and memory usage, e.g., virtual reality. To overcome these barriers, we propose a reduced-order framework. Our key innovation is training a low-dimensional manifold for the Kirchhoff stress field via an implicit neural representation. This low-dimensional neural stress field (NSF) enables efficient evaluations of stress values and, correspondingly, internal forces at arbitrary spatial locations. In addition, we also train neural deformation and affine fields to build low-dimensional manifolds for the deformation and affine momentum fields. These neural stress, deformation, and affine fields share the same low-dimensional latent space, which uniquely embeds the high-dimensional simulation state. After training, we run new simulations by evolving in this single latent space, which drastically reduces the computation time and memory consumption. Our general continuum-mechanics-based reduced-order framework is applicable to any phenomena governed by the elastodynamics equation. To showcase the versatility of our framework, we simulate a wide range of material behaviors, including elastica, sand, metal, non-Newtonian fluids, fracture, contact, and collision. We demonstrate dimension reduction by up to 100,000 × and time savings by up to 10 ×.}
}
@article{huang2025deepfracture,
title={DeepFracture: A Generative Approach for Predicting Brittle Fractures with Neural Discrete Representation Learning},
author={Huang, Yuhang and Kanai, Takashi},
journal={Computer Graphics Forum},
pages={e70002},
year={2025},
category={softbody, neural_representation, fracture},
doi={10.1111/cgf.70002},
link={https://nikoloside.graphics/deepfracture/},
openalex_id={W4407236887},
abstract={In the field of brittle fracture animation, generating realistic destruction animations using physics-based simulation methods is computationally expensive. While techniques based on Voronoi diagrams or pre-fractured patterns are effective for real-time applications, they fail to incorporate collision conditions when determining fractured shapes during runtime. This paper introduces a novel learning-based approach for predicting fractured shapes based on collision dynamics at runtime. Our approach seamlessly integrates realistic brittle fracture animations with rigid body simulations, utilising boundary element method (BEM) brittle fracture simulations to generate training data. To integrate collision scenarios and fractured shapes into a deep learning framework, we introduce generative geometric segmentation, distinct from both instance and semantic segmentation, to represent 3D fragment shapes. We propose an eight-dimensional latent code to address the challenge of optimising multiple discrete fracture pattern targets that share similar continuous collision latent codes. This code will follow a discrete normal distribution corresponding to a specific fracture pattern within our latent impulse representation design. This adaptation enables the prediction of fractured shapes using neural discrete representation learning. Our experimental results show that our approach generates considerably more detailed brittle fractures than existing techniques, while the computational time is typically reduced compared to traditional simulation methods at comparable resolutions.}
}
@article{du2021diffpd,
title={Diffpd: Differentiable projective dynamics},
author={Du, Tao and Wu, Kui and Ma, Pingchuan and Wah, Sebastien and Spielberg, Andrew and Rus, Daniela and Matusik, Wojciech},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={2},
pages={1--21},
year={2021},
category={softbody,differentiable_simulation},
publisher={ACM New York, NY},
link={https://people.iiis.tsinghua.edu.cn/~taodu/},
abstract={We present a novel, fast differentiable simulator for soft-body learning and control applications. Existing differentiable soft-body simulators can be classified into two categories based on their time integration methods. Simulators using explicit time-stepping scheme require tiny time steps to avoid numerical instabilities in gradient computation, and simulators using implicit time integration typically compute gradients by employing the adjoint method to solve the expensive linearized dynamics. Inspired by Projective Dynamics (PD), we present DiffPD, an efficient differentiable soft-body simulator with implicit time integration. The key idea in DiffPD is to speed up backpropagation by exploiting the prefactorized Cholesky decomposition in PD to achieve a super-linear convergence rate. To handle contacts, DiffPD solves contact forces by analyzing a linear complementarity problem (LCP) and its gradients. With the assumption that contacts occur on a small number of nodes, we develop an efficient method for gradient computation by exploring the low-rank structure in the linearized dynamics. We evaluate the performance of DiffPD and observe a speedup of 4-19 times compared to the standard Newton's method in various applications including system identification, inverse design problems, trajectory optimization, and closed-loop control.}
}
@article{huang2024differentiable,
title={Differentiable solver for time-dependent deformation problems with contact},
author={Huang, Zizhou and Tozoni, Davi Colli and Gjoka, Arvi and Ferguson, Zachary and Schneider, Teseo and Panozzo, Daniele and Zorin, Denis},
journal={ACM Transactions on Graphics},
year={2024},
category={softbody,differentiable_simulation,contact},
tag={material_estimation,shape_design},
publisher={ACM New York, NY},
link={https://dl.acm.org/doi/10.1145/3657648},
abstract={We introduce a general differentiable solver for time-dependent deformation problems with contact and friction. Our approach uses a finite element discretization with a high-order time integrator coupled with the recently proposed incremental potential contact method for handling contact and friction forces to solve ODE- and PDE-constrained optimization problems on scenes with complex geometry. It supports static and dynamic problems and differentiation with respect to all physical parameters involved in the physical problem description, which include shape, material parameters, friction parameters, and initial conditions. Our analytically derived adjoint formulation is efficient, with a small overhead (typically less than 10% for nonlinear problems) over the forward simulation, and shares many similarities with the forward problem, allowing the reuse of large parts of existing forward simulator code. We implement our approach on top of the open-source PolyFEM library and demonstrate the applicability of our solver to shape design, initial condition optimization, and material estimation on both simulated results and physical validations.}
}
@article{wang2022differentiable,
title={Differentiable simulation of inertial musculotendons},
author={Wang, Ying and Verheul, Jasper and Yeo, Sang-Hoon and Kalantari, Nima Khademi and Sueda, Shinjiro},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={6},
pages={1--11},
year={2022},
category={softbody,differentiable_simulation},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/abs/10.1145/3550454.3555490},
abstract={We propose a simple and practical approach for incorporating the effects of muscle inertia, which has been ignored by previous musculoskeletal simulators in both graphics and biomechanics. We approximate the inertia of the muscle by assuming that muscle mass is distributed along the centerline of the muscle. We express the motion of the musculotendons in terms of the motion of the skeletal joints using a chain of Jacobians, so that at the top level, only the reduced degrees of freedom of the skeleton are used to completely drive both bones and musculotendons. Our approach can handle all commonly used musculotendon path types, including those with multiple path points and wrapping surfaces. For muscle paths involving wrapping surfaces, we use neural networks to model the Jacobians, trained using existing wrapping surface libraries, which allows us to effectively handle the Jacobian discontinuities that occur when musculotendon paths collide with wrapping surfaces. We demonstrate support for higher-order time integrators, complex joints, inverse dynamics, Hill-type muscle models, and differentiability. In the limit, as the muscle mass is reduced to zero, our approach gracefully degrades to traditional simulators without support for muscle inertia. Finally, it is possible to mix and match inertial and non-inertial musculotendons, depending on the application.}
}
@inproceedings{qiao2021efficient,
author = {Qiao, Yi-Ling and Liang, Junbang and Koltun, Vladlen and Lin, Ming C.},
title = {Efficient Differentiable Simulation of Articulated Bodies},
booktitle = {ICML},
year = {2021},
category={rigidbody,differentiable_simulation},
tag={reinforce_learning},
link={https://github.com/YilingQiao/diffarticulated},
abstract={We present a method for efficient differentiable simulation of articulated bodies. This enables integration of articulated body dynamics into deep learning frameworks, and gradient-based optimization of neural networks that operate on articulated bodies. We derive the gradients of the forward dynamics using spatial algebra and the adjoint method. Our approach is an order of magnitude faster than autodiff tools. By only saving the initial states throughout the simulation process, our method reduces memory requirements by two orders of magnitude. We demonstrate the utility of efficient differentiable dynamics for articulated bodies in a variety of applications. We show that reinforcement learning with articulated systems can be accelerated using gradients provided by our method. In applications to control and inverse problems, gradient-based optimization enabled by our work accelerates convergence by more than an order of magnitude.}
}
@inproceedings{schnell2022halfinverse,
title={Half-Inverse Gradients for Physical Deep Learning},
author={Patrick Schnell and Philipp Holl and Nils Thuerey},
booktitle={International Conference on Learning Representations},
year={2022},
category={fluid,differentiable_simulation,neural_solver},
url={https://openreview.net/forum?id=HTx7vrlLBEj},
link={https://arxiv.org/abs/2203.10131},
abstract={Recent works in deep learning have shown that integrating differentiable physics simulators into the training process can greatly improve the quality of results. Although this combination represents a more complex optimization task than supervised neural network training, the same gradient-based optimizers are typically employed to minimize the loss function. However, the integrated physics solvers have a profound effect on the gradient flow as manipulating scales in magnitude and direction is an inherent property of many physical processes. Consequently, the gradient flow is often highly unbalanced and creates an environment in which existing gradient-based optimizers perform poorly. In this work, we analyze the characteristics of both physical and neural network optimizations to derive a new method that does not suffer from this phenomenon. Our method is based on a half-inversion of the Jacobian and combines principles of both classical network and physics optimizers to solve the combined optimization task. Compared to state-of-the-art neural network optimizers, our method converges more quickly and yields better solutions, which we demonstrate on three complex learning problems involving nonlinear oscillators, the Schroedinger equation and the Poisson problem.}
}
@inproceedings{zesch2023neural,
author = {Zesch, Ryan S. and Modi, Vismay and Sueda, Shinjiro and Levin, David I.W.},
title = {Neural Collision Fields for Triangle Primitives},
year = {2023},
category={rigidbody,neural_representation,contact},
tag={neural_field},
isbn = {9798400703157},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3610548.3618225},
doi = {10.1145/3610548.3618225},
booktitle = {SIGGRAPH Asia 2023 Conference Papers},
articleno = {76},
numpages = {10},
keywords = {Collision Detection, Collision Resolution, Neural Network},
location = {, Sydney, NSW, Australia, },
series = {SA '23},
link={https://research.nvidia.com/labs/prl/publication/zesch2023ncf/},
abstract={We present neural collision fields as an alternative to contact point sampling in physics simulations. Our approach is built on top of a novel smoothed integral formulation for the contact surface patches between two triangle meshes. By reformulating collisions as an integral, we avoid issues of sampling common to many collision-handling algorithms. Because the resulting integral is difficult to evaluate numerically, we store its solution in an integrated neural collision field — a 6D neural field in the space of triangle pair vertex coordinates. Our network generalizes well to new triangle meshes without retraining. We demonstrate the effectiveness of our method by implementing it as a constraint in a position-based dynamics framework and show that our neural formulation successfully handles collisions in practical simulations involving both volumetric and thin-shell geometries.}
}
@MISC{coumans2021pybullet,
author = {Erwin Coumans and Yunfei Bai},
title = {PyBullet, a Python module for physics simulation for games, robotics and machine learning},
howpublished = {\url{http://pybullet.org}},
year = {2016--2021},
category={rigidbody,engine},
link={http://pybullet.org}
}
@article{howell2022dojo,
title={Dojo: A Differentiable Physics Engine for Robotics},
author={Howell, Taylor and Le Cleac'h, Simon and Bruedigam, Jan and Kolter, Zico and Schwager, Mac and Manchester, Zachary},
journal={arXiv preprint arXiv:2203.00806},
url={https://arxiv.org/abs/2203.00806},
year={2022},
category={rigidbody,differentiable_simulation,embodied_ai,engine},
link={https://sites.google.com/view/dojo-sim},
abstract={We present Dojo, a differentiable physics engine for robotics that prioritizes stable simulation, accurate contact physics, and differentiability with respect to states, actions, and system parameters. Dojo models hard contact and friction with a nonlinear complementarity problem with second-order cone constraints. We introduce a custom primal-dual interior-point method to solve the second order cone program for stable forward simulation over a broad range of sample rates. We obtain smooth gradient approximations with this solver through the implicit function theorem, giving gradients that are useful for downstream trajectory optimization, policy optimization, and system identification applications. Specifically, we propose to use the central path parameter threshold in the interior point solver as a user-tunable design parameter. A high value gives a smooth approximation to contact dynamics with smooth gradients for optimization and learning, while a low value gives precise simulation rollouts with hard contact. We demonstrate Dojo's differentiability in trajectory optimization, policy learning, and system identification examples. We also benchmark Dojo against MuJoCo, PyBullet, Drake, and Brax on a variety of robot models, and study the stability and simulation quality over a range of sample frequencies and accuracy tolerances. Finally, we evaluate the sim-to-real gap in hardware experiments with a Ufactory xArm 6 robot. Dojo is an open source project implemented in Julia with Python bindings, with code available at https://github.com/dojo-sim/Dojo.jl.}
}
@inproceedings{freeman2021brax,
title={Brax - A Differentiable Physics Engine for Large Scale Rigid Body Simulation},
author={C. Daniel Freeman and Erik Frey and Anton Raichuk and Sertan Girgin and Igor Mordatch and Olivier Bachem},
booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 1)},
year={2021},
category={rigidbody,differentiable_simulation,engine},
tag={reinforce_learning},
url={https://openreview.net/forum?id=VdvDlnnjzIN},
link={https://github.com/google/brax},
abstract={We present Brax, an open source library for rigid body simulation with a focus on performance and parallelism on accelerators, written in JAX. We present results on a suite of tasks inspired by the existing reinforcement learning literature, but remade in our engine. Additionally, we provide reimplementations of PPO, SAC, ES, and direct policy optimization in JAX that compile alongside our environments, allowing the learning algorithm and the environment processing to occur on the same device, and to scale seamlessly on accelerators. Finally, we include notebooks that facilitate training of performant policies on common OpenAI Gym MuJoCo-like tasks in minutes.}
}
@inproceedings{todorov2012mujoco,
title={MuJoCo: A physics engine for model-based control},
author={Todorov, Emanuel and Erez, Tom and Tassa, Yuval},
booktitle={2012 IEEE/RSJ International Conference on Intelligent Robots and Systems},
pages={5026--5033},
year={2012},
category={rigidbody,control,engine},
organization={IEEE},
doi={10.1109/IROS.2012.6386109},
link={https://mujoco.org/},
abstract={We describe a new physics engine tailored to model-based control. Multi-joint dynamics are represented in generalized coordinates and computed via recursive algorithms. Contact responses are computed via efficient new algorithms we have developed, based on the modern velocity-stepping approach which avoids the difficulties with spring-dampers. Models are specified using either a high-level C++ API or an intuitive XML file format. A built-in compiler transforms the user model into an optimized data structure used for runtime computation. The engine can compute both forward and inverse dynamics. The latter are well-defined even in the presence of contacts and equality constraints. The model can include tendon wrapping as well as actuator activation states (e.g. pneumatic cylinders or muscles). To facilitate optimal control applications and in particular sampling and finite differencing, the dynamics can be evaluated for different states and controls in parallel. Around 400,000 dynamics evaluations per second are possible on a 12-core machine, for a 3D homanoid with 18 dofs and 6 active contacts. We have already used the engine in a number of control applications. It will soon be made publicly available.}
}
@INPROCEEDINGS{werling2021fast,
AUTHOR = {Keenon Werling AND Dalton Omens AND Jeongseok Lee AND Ioannis Exarchos AND C. Karen Liu},
TITLE = {Fast and Feature-Complete Differentiable Physics Engine for Articulated Rigid Bodies with Contact Constraints},
BOOKTITLE = {Proceedings of Robotics: Science and Systems},
YEAR = {2021},
category={softbody,differentiable_simulation,engine,contact},
ADDRESS = {Virtual},
MONTH = {July},
DOI = {10.15607/RSS.2021.XVII.034},
link={https://arxiv.org/abs/2103.16021},
abstract={We present a fast and feature-complete differentiable physics engine, Nimble (nimblephysics.org), that supports Lagrangian dynamics and hard contact constraints for articulated rigid body simulation. Our differentiable physics engine offers a complete set of features that are typically only available in non-differentiable physics simulators commonly used by robotics applications. We solve contact constraints precisely using linear complementarity problems (LCPs). We present efficient and novel analytical gradients through the LCP formulation of inelastic contact that exploit the sparsity of the LCP solution. We support complex contact geometry, and gradients approximating continuous-time elastic collision. We also introduce a novel method to compute complementarity-aware gradients that help downstream optimization tasks avoid stalling in saddle points. We show that an implementation of this combination in a fork of an existing physics engine (DART) is capable of a 87x single-core speedup over finite-differencing in computing analytical Jacobians for a single timestep, while preserving all the expressiveness of original DART.}
}
@inproceedings{gang2024jade,
title={Jade: A Differentiable Physics Engine for Articulated Rigid Bodies with Intersection-Free Frictional Contact},
author={Yang, Gang and Luo, Siyuan and Feng, Yunhai and Sun, Zhixin and Tie, Chenrui and Shao, Lin},
booktitle={IEEE International Conference on Robotics and Automation (ICRA)},
year={2024},
category={rigidbody,differentiable_simulation,engine,contact},
organization={IEEE},
link={https://sites.google.com/view/diffsim/},
abstract={We present Jade, a differentiable physics engine for articulated rigid bodies. Jade models contacts as the Linear Complementarity Problem (LCP). Compared to existing differentiable simulations, Jade offers features including intersection-free collision simulation and stable LCP solutions for multiple frictional contacts. We use continuous collision detection to detect the time of impact and adopt the backtracking strategy to prevent intersection between bodies with complex geometry shapes. We derive the gradient calculation to ensure the whole simulation process is differentiable under the backtracking mechanism. We modify the popular Dantzig’s algorithm to get valid solutions under multiple frictional contacts. We conduct extensive experiments to demonstrate the effectiveness of our differentiable physics simulation over a variety of contact-rich tasks. Supplemental materials and videos are available on our project webpage at https://sites.google.com/view/diffsim}
}
@misc{russ2019drake,
author = "Russ Tedrake and the Drake Development Team",
title = "Drake: Model-based design and verification for robotics",
year = 2019,
category={rigidbody, engine, embodied_ai},
url = "https://drake.mit.edu",
link= "https://drake.mit.edu",
}
@inproceedings{heiden2021neuralsim,
author = {Heiden, Eric and Millard, David and Coumans, Erwin and Sheng, Yizhou and Sukhatme, Gaurav S},
year = {2021},
category={rigidbody,differentiable_simulation,neural_solver},
title = {Neural{S}im: Augmenting Differentiable Simulators with Neural Networks},
booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},
url = {https://github.com/google-research/tiny-differentiable-simulator},
link={https://github.com/erwincoumans/tiny-differentiable-simulator},
abstract={Differentiable simulators provide an avenue for closing the sim-to-real gap by enabling the use of efficient, gradient-based optimization algorithms to find the simulation parameters that best fit the observed sensor readings. Nonetheless, these analytical models can only predict the dynamical behavior of systems for which they have been designed. In this work, we study the augmentation of a novel differentiable rigid-body physics engine via neural networks that is able to learn nonlinear relationships between dynamic quantities and can thus model effects not accounted for in traditional simulators. Such augmentations require less data to train and generalize better compared to entirely data-driven models. Through extensive experiments, we demonstrate the ability of our hybrid simulator to learn complex dynamics involving frictional contacts from real data, as well as match known models of viscous friction, and present an approach for automatically discovering useful augmentations. We show that, besides benefiting dynamics modeling, inserting neural networks can accelerate model-based control architectures. We observe a ten-fold speedup when replacing the QP solver inside a model-predictive gait controller for quadruped robots with a neural network, allowing us to significantly improve control delays as we demonstrate in real-hardware experiments. We publish code, additional results and videos from our experiments on our project webpage at https://sites.google.com/usc.edu/neuralsim.}
}
@article{liu2021role,
author = "Liu, C. Karen and Negrut, Dan",
title = "The Role of Physics-Based Simulators in Robotics",
journal= "Annual Review of Control, Robotics, and Autonomous Systems",
year = "2021",
category={rigidbody, embodied_ai, survey},
volume = "4",
number = "Volume 4, 2021",
pages = "35-58",
doi = "https://doi.org/10.1146/annurev-control-072220-093055",
url = "https://www.annualreviews.org/content/journals/10.1146/annurev-control-072220-093055",
publisher = "Annual Reviews",
issn = "2573-5144",
type = "Journal Article",
link="https://doi.org/10.1146/annurev-control-072220-093055",
abstract={Physics-based simulation provides an accelerated and safe avenue for developing, verifying, and testing robotic control algorithms and prototype designs. In the quest to leverage machine learning for developing AI-enabled robots, physics-based simulation can generate a wealth of labeled training data in a short amount of time. Physics-based simulation also creates an ideal proving ground for developing intelligent robots that can both learn from their mistakes and be verifiable. This article provides an overview of the use of simulation in robotics, emphasizing how robots (with sensing and actuation components), the environment they operate in, and the humans they interact with are simulated in practice. It concludes with an overview of existing tools for simulation in robotics and a short discussion of aspects that limit the role that simulation plays today in intelligent robot design.}
}
@article{li2022diffcloth,
author = {Li, Yifei and Du, Tao and Wu, Kui and Xu, Jie and Matusik, Wojciech},
title = {DiffCloth: Differentiable Cloth Simulation with Dry Frictional Contact},
year = {2022},
category={cloth,differentiable_simulation,embodied_ai,contact},
tag={shape_design},
issue_date = {February 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {1},
issn = {0730-0301},
url = {https://doi.org/10.1145/3527660},
doi = {10.1145/3527660},
journal = {ACM Trans. Graph.},
month = {oct},
articleno = {2},
numpages = {20},
keywords = {differentiable simulation, Projective Dynamics, cloth simulation},
link={https://people.csail.mit.edu/liyifei/publication/diffcloth/},
abstract={Cloth simulation has wide applications in computer animation, garment design, and robot-assisted dressing. This work presents a differentiable cloth simulator whose additional gradient information facilitates cloth-related applications. Our differentiable simulator extends a state-of-the-art cloth simulator based on Projective Dynamics (PD) and with dry frictional contact [Ly et al. 2020 ]. We draw inspiration from previous work [Du et al. 2021 ] to propose a fast and novel method for deriving gradients in PD-based cloth simulation with dry frictional contact. Furthermore, we conduct a comprehensive analysis and evaluation of the usefulness of gradients in contact-rich cloth simulation. Finally, we demonstrate the efficacy of our simulator in a number of downstream applications, including system identification, trajectory optimization for assisted dressing, closed-loop control, inverse design, and real-to-sim transfer. We observe a substantial speedup obtained from using our gradient information in solving most of these applications.}
}
@article{ly2020projective,
author = {Ly, Micka\"{e}l and Jouve, Jean and Boissieux, Laurence and Bertails-Descoubes, Florence},
title = {Projective dynamics with dry frictional contact},
year = {2020},
category={cloth,contact},
issue_date = {August 2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {39},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3386569.3392396},
doi = {10.1145/3386569.3392396},
journal = {ACM Trans. Graph.},
month = {aug},
articleno = {57},
numpages = {8},
keywords = {cloth simulation, dry frictional contact, interactive simulation, projective dynamics},
link={https://astcort.github.io/},
abstract={Projective dynamics was introduced a few years ago as a fast method to yield an approximate yet stable solution to the dynamics of nodal systems subject to stiff internal forces. Previous attempts to include contact forces in that framework considered adding a quadratic penalty energy to the global system, which however broke the simple - constant matrix - structure of the global linear equation, while failing to treat contact in an implicit manner. In this paper we propose a simple yet effective method to integrate in a unified and semi-implicit way contact as well as dry frictional forces into the nested architecture of Projective dynamics. Assuming that contacts apply to nodes only, the key is to split the global matrix into a diagonal and a positive matrix, and use this splitting in the local step so as to make a good prediction of frictional contact forces at next iteration. Each frictional contact force is refined independently in the local step, while the original efficient structure of the global step is left unchanged. We apply our algorithm to cloth simulation and show that contact and dry friction can be captured at a reasonable precision within a few iterations only, hence one order of magnitude faster compared to global implicit contact solvers of the literature.}
}
@inproceedings{liang2019differentiable,
author = {Liang, Junbang and Lin, Ming and Koltun, Vladlen},
booktitle = {Advances in Neural Information Processing Systems},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {},
publisher = {Curran Associates, Inc.},
title = {Differentiable Cloth Simulation for Inverse Problems},
url = {https://proceedings.neurips.cc/paper_files/paper/2019/file/28f0b864598a1291557bed248a998d4e-Paper.pdf},
volume = {32},
year = {2019},
category={cloth, differentiable_simulation, control},
link={https://gamma.umd.edu/researchdirections/virtualtryon/differentiablecloth},
abstract={We propose a differentiable cloth simulator that can be embedded as a layer in deep neural networks. This approach provides an effective, robust framework for modeling cloth dynamics, self-collisions, and contacts. Due to the high dimensionality of the dynamical system in modeling cloth, traditional gradient computation for collision response can become impractical. To address this problem, we propose to compute the gradient directly using QR decomposition of a much smaller matrix. Experimental results indicate that our method can speed up backpropagation by two orders of magnitude. We demonstrate the presented approach on a number of inverse problems, including parameter estimation and motion control for cloth.}
}
@inproceedings{gong2024bayesian,
author = {Deshan Gong and Ningtao Mao and He Wang},
title = {Bayesian Differentiable Physics for Cloth Digitalization},
booktitle = {The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year = {2024},
category={cloth,reconstruction},
tag={real2sim,cloth_reconstruction,parameter_estimation},
link={https://github.com/realcrane/Bayesian-Differentiable-Physics-for-Cloth-Digitalization},
abstract={We propose a new method for cloth digitalization. Deviating from existing methods which learn from data captured under relatively casual settings, we propose to learn from data captured in strictly tested measuring protocols, and find plausible physical parameters of the cloths. However, such data is currently absent, so we first propose a new dataset with accurate cloth measurements. Further, the data size is considerably smaller than the ones in current deep learning, due to the nature of the data capture process. To learn from small data, we propose a new Bayesian differentiable cloth model to estimate the complex material heterogeneity of real cloths. It can provide highly accurate digitalization from very limited data samples. Through exhaustive evaluation and comparison, we show our method is accurate in cloth digitalization, efficient in learning from limited data samples, and general in capturing material variations. Code and data are available https://github.com/realcrane/Bayesian-Differentiable-Physics-for-Cloth-Digitalization}
}
@inproceedings{li2023diffavatar,
title={{DiffAvatar}: Simulation-Ready Garment Optimization with Differentiable Simulation},
author={Li, Yifei and Chen, Hsiao-yu and Larionov, Egor and Sarafianos, Nikolaos and Matusik, Wojciech and Stuyck, Tuur},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {June},
eprint={2311.12194},
year = {2024},
category={cloth,differentiable_simulation,optimization,avatar},
link={https://people.csail.mit.edu/liyifei/publication/diffavatar/},
abstract={The realism of digital avatars is crucial in enabling telepresence applications with self-expression and customization. While physical simulations can produce realistic motions for clothed humans, they require high-quality garment assets with associated physical parameters for cloth simulations. However, manually creating these assets and calibrating their parameters is labor-intensive and requires specialized expertise. Current methods focus on reconstructing geometry, but don't generate complete assets for physics-based applications. To address this gap, we propose DiffAvatar, a novel approach that performs body and garment co-optimization using differentiable simulation. By integrating physical simulation into the optimization loop and accounting for the complex nonlinear behavior of cloth and its intricate interaction with the body, our framework recovers body and garment geometry and extracts important material parameters in a physically plausible way. Our experiments demonstrate that our approach generates realistic clothing and body shape suitable for downstream applications. We provide additional insights and results on our webpage: people. csail. mit. edu/liyifei/publication/diffavatar.}
}
@article{bai2021predicting,
title={Predicting high-resolution turbulence details in space and time},
author={Bai, Kai and Wang, Chunhao and Desbrun, Mathieu and Liu, Xiaopei},
journal={ACM Transactions on Graphics (TOG)},
volume={40},
number={6},
pages={1--16},
year={2021},
category={fluid},
tag={super_resolution},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/abs/10.1145/3478513.3480492},
abstract={Predicting the fine and intricate details of a turbulent flow field in both space and time from a coarse input remains a major challenge despite the availability of modern machine learning tools. In this paper, we present a simple and effective dictionary-based approach to spatio-temporal upsampling of fluid simulation. We demonstrate that our neural network approach can reproduce the visual complexity of turbulent flows from spatially and temporally coarse velocity fields even when using a generic training set. Moreover, since our method generates finer spatial and/or temporal details through embarrassingly-parallel upsampling of small local patches, it can efficiently predict high-resolution turbulence details across a variety of grid resolutions. As a consequence, our method offers a whole range of applications varying from fluid flow upsampling to fluid data compression. We demonstrate the efficiency and generalizability of our method for synthesizing turbulent flows on a series of complex examples, highlighting dramatically better results in spatio-temporal upsampling and flow data compression than existing methods as assessed by both qualitative and quantitative comparisons.}
}
@article{roy2021neural,
title={Neural upflow: A scene flow learning approach to increase the apparent resolution of particle-based liquids},
author={Roy, Bruno and Poulin, Pierre and Paquette, Eric},
journal={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
volume={4},
number={3},
pages={1--26},
year={2021},
category={fluid},
tag={super_resolution},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/abs/10.1145/3480147},
abstract={We present a novel up-resing technique for generating high-resolution liquids based on scene flow estimation using deep neural networks. Our approach infers and synthesizes small- and large-scale details solely from a low-resolution particle-based liquid simulation. The proposed network leverages neighborhood contributions to encode inherent liquid properties throughout convolutions. We also propose a particle-based approach to interpolate between liquids generated from varying simulation discretizations using a state-of-the-art bidirectional optical flow solver method for fluids in addition to a novel key-event topological alignment constraint. In conjunction with the neighborhood contributions, our loss formulation allows the inference model throughout epochs to reward important differences in regard to significant gaps in simulation discretizations. Even when applied in an untested simulation setup, our approach is able to generate plausible high-resolution details. Using this interpolation approach and the predicted displacements, our approach combines the input liquid properties with the predicted motion to infer semi-Lagrangian advection. We furthermore showcase how the proposed interpolation approach can facilitate generating large simulation datasets with a subset of initial condition parameters.}
}
@article{bertiche2021pbns,
author = {Bertiche, Hugo and Madadi, Meysam and Escalera, Sergio},
title = {PBNS: physically based neural simulation for unsupervised garment pose space deformation},
year = {2021},
category={cloth,neural_solver},
issue_date = {December 2021},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {40},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3478513.3480479},
doi = {10.1145/3478513.3480479},
journal = {ACM Trans. Graph.},
month = {dec},
articleno = {198},
numpages = {14},
keywords = {animation, deep learning, garment, neural network, physics, pose space deformation, simulation},
link={https://hbertiche.github.io/PBNS/},
abstract={We present a methodology to automatically obtain Pose Space Deformation (PSD) basis for rigged garments through deep learning. Classical approaches rely on Physically Based Simulations (PBS) to animate clothes. These are general solutions that, given a sufficiently fine-grained discretization of space and time, can achieve highly realistic results. However, they are computationally expensive and any scene modification prompts the need of re-simulation. Linear Blend Skinning (LBS) with PSD offers a lightweight alternative to PBS, though, it needs huge volumes of data to learn proper PSD. We propose using deep learning, formulated as an implicit PBS, to un-supervisedly learn realistic cloth Pose Space Deformations in a constrained scenario: dressed humans. Furthermore, we show it is possible to train these models in an amount of time comparable to a PBS of a few sequences. To the best of our knowledge, we are the first to propose a neural simulator for cloth. While deep-based approaches in the domain are becoming a trend, these are data-hungry models. Moreover, authors often propose complex formulations to better learn wrinkles from PBS data. Supervised learning leads to physically inconsistent predictions that require collision solving to be used. Also, dependency on PBS data limits the scalability of these solutions, while their formulation hinders its applicability and compatibility. By proposing an unsupervised methodology to learn PSD for LBS models (3D animation standard), we overcome both of these drawbacks. Results obtained show cloth-consistency in the animated garments and meaningful pose-dependant folds and wrinkles. Our solution is extremely efficient, handles multiple layers of cloth, allows unsupervised outfit resizing and can be easily applied to any custom 3D avatar.}
}
@ARTICLE{xiao2020novel,
author={Xiao, Xiangyun and Zhou, Yanqing and Wang, Hui and Yang, Xubo},
journal={IEEE Transactions on Visualization and Computer Graphics},
title={A Novel CNN-Based Poisson Solver for Fluid Simulation},
year={2020},
category={fluid, neural_solver},
tag={cnn},
volume={26},
number={3},
pages={1454-1465},
keywords={Mathematical model;Poisson equations;Sparse matrices;Linear systems;Computational modeling;Acceleration;Artificial neural networks;Linear equation;poisson solver;machine learning;deep convolutional neural network},
doi={10.1109/TVCG.2018.2873375},
link={http://dalab.se.sjtu.edu.cn/www/home/?page_id=790},
abstract={Solving a large-scale Poisson system is computationally expensive for most of the Eulerian fluid simulation applications. We propose a novel machine learning-based approach to accelerate this process. At the heart of our approach is a deep convolutional neural network (CNN), with the capability of predicting the solution (pressure) of a Poisson system given the discretization structure and the intermediate velocities as input. Our system consists of four main components, namely, a deep neural network to solve the large linear equations, a geometric structure to describe the spatial hierarchies of the input vector, a Principal Component Analysis (PCA) process to reduce the dimension of input in training, and a novel loss function to control the incompressibility constraint. We have demonstrated the efficacy of our approach by simulating a variety of high-resolution smoke and liquid phenomena. In particular, we have shown that our approach accelerates the projection step in a conventional Eulerian fluid simulator by two orders of magnitude. In addition, we have also demonstrated the generality of our approach by producing a diversity of animations deviating from the original datasets.}
}
@inproceedings{li2023detail,
title={Detail-Aware Deep Clothing Animations Infused with Multi-Source Attributes},
author={Li, Tianxing and Shi, Rui and Kanai, Takashi},
booktitle={Computer Graphics Forum},
volume={42},
number={1},
pages={231--244},
year={2023},
category={cloth, neural_solver},
tag={gnn},
organization={Wiley Online Library},
link={https://onlinelibrary.wiley.com/doi/abs/10.1111/cgf.14651},
abstract={Abstract This paper presents a novel learning‐based clothing deformation method to generate rich and reasonable detailed deformations for garments worn by bodies of various shapes in various animations. In contrast to existing learning‐based methods, which require numerous trained models for different garment topologies or poses and are unable to easily realize rich details, we use a unified framework to produce high fidelity deformations efficiently and easily. Specifically, we first found that the fit between the garment and the body has an important impact on the degree of folds. We then designed an attribute parser to generate detail‐aware encodings and infused them into the graph neural network, therefore enhancing the discrimination of details under diverse attributes. Furthermore, to achieve better convergence and avoid overly smooth deformations, we proposed to reconstruct output to mitigate the complexity of the learning task. Experimental results show that our proposed deformation method achieves better performance over existing methods in terms of generalization ability and quality of details.}
}
@article{yang2016data,
author = {Yang, Cheng and Yang, Xubo and Xiao, Xiangyun},
title = {Data-driven projection method in fluid simulation},
journal = {Computer Animation and Virtual Worlds},
volume = {27},
number = {3-4},
pages = {415-424},
keywords = {artificial neural network, data-driven, grid-based fluid simulation, projection step},
doi = {https://doi.org/10.1002/cav.1695},
url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/cav.1695},
eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/cav.1695},
year = {2016},
category={fluid, neural_solver},
link={http://dalab.se.sjtu.edu.cn/www/home/?page_id=911},
abstract={Abstract Physically based fluid simulation requires much time in numerical calculation to solve Navier–Stokes equations. Especially in grid‐based fluid simulation, because of iterative computation, the projection step is much more time‐consuming than other steps. In this paper, we propose a novel data‐driven projection method using an artificial neural network to avoid iterative computation. Once the grid resolution is decided, our data‐driven method could obtain projection results in relatively constant time per grid cell, which is independent of scene complexity. Experimental results demonstrated that our data‐driven method drastically speeded up the computation in the projection step. With the growth of grid resolution, the speed‐up would increase strikingly. In addition, our method is still applicable in different fluid scenes with some alterations, when computational cost is more important than physical accuracy. Copyright © 2016 John Wiley & Sons, Ltd.}
}
@inproceedings{xiao2019cnn,
title={A CNN-based Flow Correction Method for Fast Preview},
author={Xiao, Xiangyun and Wang, Hui and Yang, Xubo},
booktitle={Computer Graphics Forum},
volume={38},
number={2},
pages={431--440},
year={2019},
category={fluid, neural_solver},
tag={cnn},
organization={Wiley Online Library},
link={http://dalab.se.sjtu.edu.cn/www/home/?page_id=763},
abstract={Abstract Eulerian‐based smoke simulations are sensitive to the initial parameters and grid resolutions. Due to the numerical dissipation on different levels of the grid and the nonlinearity of the governing equations, the differences in simulation resolutions will result in different results. This makes it challenging for artists to preview the animation results based on low‐resolution simulations. In this paper, we propose a learning‐based flow correction method for fast previewing based on low‐resolution smoke simulations. The main components of our approach lie in a deep convolutional neural network, a grid‐layer feature vector and a special loss function. We provide a novel matching model to represent the relationship between low‐resolution and high‐resolution smoke simulations and correct the overall shape of a low‐resolution simulation to closely follow the shape of a high‐resolution down‐sampled version. We introduce the grid‐layer concept to effectively represent the 3D fluid shape, which can also reduce the input and output dimensions. We design a special loss function for the fluid divergence‐free constraint in the neural network training process. We have demonstrated the efficacy and the generality of our approach by simulating a diversity of animations deviating from the original training set. In addition, we have integrated our approach into an existing fluid simulation framework to showcase its wide applications.}
}
@article{chen2021data,
title={Data-driven simulation in fluids animation: A survey},
author={Chen, Qian and Wang, Yue and Wang, Hui and Yang, Xubo},
journal={Virtual Reality \& Intelligent Hardware},
volume={3},
number={2},
pages={87--104},
year={2021},
category={fluid,survey},
publisher={Elsevier},
link={https://www.sciencedirect.com/science/article/pii/S2096579621000139},
abstract={The field of fluid simulation is developing rapidly, and data-driven methods provide many frameworks and techniques for fluid simulation. This paper presents a survey of data-driven methods used in fluid simulation in computer graphics in recent years. First, we provide a brief introduction of physicalbased fluid simulation methods based on their spatial discretization, including Lagrangian, Eulerian, and hybrid methods. The characteristics of these underlying structures and their inherent connection with datadriven methodologies are then analyzed. Subsequently, we review studies pertaining to a wide range of applications, including data-driven solvers, detail enhancement, animation synthesis, fluid control, and differentiable simulation. Finally, we discuss some related issues and potential directions in data-driven fluid simulation. We conclude that the fluid simulation combined with data-driven methods has some advantages, such as higher simulation efficiency, rich details and different pattern styles, compared with traditional methods under the same parameters. It can be seen that the data-driven fluid simulation is feasible and has broad prospects.}
}
@inproceedings{wiewel2020latent,
title={Latent space subdivision: stable and controllable time predictions for fluid flow},
author={Wiewel, Steffen and Kim, Byungsoo and Azevedo, Vinicius C and Solenthaler, Barbara and Thuerey, Nils},
booktitle={Computer graphics forum},
volume={39},
number={8},
pages={15--25},
year={2020},
category={fluid,neural_representation},
tag={cnn},
organization={Wiley Online Library},
link={https://ge.in.tum.de/publications/2020-lssubdiv-wiewel/},
abstract={Abstract We propose an end‐to‐end trained neural network architecture to robustly predict the complex dynamics of fluid flows with high temporal stability. We focus on single‐phase smoke simulations in 2D and 3D based on the incompressible Navier‐Stokes (NS) equations, which are relevant for a wide range of practical problems. To achieve stable predictions for long‐term flow sequences with linear execution times, a convolutional neural network (CNN) is trained for spatial compression in combination with a temporal prediction network that consists of stacked Long Short‐Term Memory (LSTM) layers. Our core contribution is a novel latent space subdivision (LSS) to separate the respective input quantities into individual parts of the encoded latent space domain. As a result, this allows to distinctively alter the encoded quantities without interfering with the remaining latent space values and hence maximizes external control. By selectively overwriting parts of the predicted latent space points, our proposed method is capable to robustly predict long‐term sequences of complex physics problems, like the flow of fluids. In addition, we highlight the benefits of a recurrent training on the latent space creation, which is performed by the spatial compression network. Furthermore, we thoroughly evaluate and discuss several different components of our method.}
}
@article{prantl2022guaranteed,
title={Guaranteed conservation of momentum for learning particle-based fluid dynamics},
author={Prantl, Lukas and Ummenhofer, Benjamin and Koltun, Vladlen and Thuerey, Nils},
journal={Advances in Neural Information Processing Systems},
volume={35},
pages={6901--6913},
year={2022},
category={fluid,neural_solver},
link={https://github.com/tum-pbs/DMCF},
abstract={We present a novel method for guaranteeing linear momentum in learned physics simulations. Unlike existing methods, we enforce conservation of momentum with a hard constraint, which we realize via antisymmetrical continuous convolutional layers. We combine these strict constraints with a hierarchical network architecture, a carefully constructed resampling scheme, and a training approach for temporal coherence. In combination, the proposed method allows us to increase the physical accuracy of the learned simulator substantially. In addition, the induced physical bias leads to significantly better generalization performance and makes our method more reliable in unseen test cases. We evaluate our method on a range of different, challenging fluid scenarios. Among others, we demonstrate that our approach generalizes to new scenarios with up to one million particles. Our results show that the proposed algorithm can learn complex dynamics while outperforming existing approaches in generalization and training performance. An implementation of our approach is available at https://github.com/tum-pbs/DMCF.}
}
@inproceedings{wang2023neuwigs,
title={Neuwigs: A neural dynamic model for volumetric hair capture and animation},
author={Wang, Ziyan and Nam, Giljoo and Stuyck, Tuur and Lombardi, Stephen and Cao, Chen and Saragih, Jason and Zollh{\"o}fer, Michael and Hodgins, Jessica and Lassner, Christoph},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={8641--8651},
year={2023},
category={softbody,neural_representation,reconstruction},
tag={real2sim,hair},
link={https://ziyanw1.github.io/neuwigs/},
abstract={The capture and animation of human hair are two of the major challenges in the creation of realistic avatars for the virtual reality. Both problems are highly challenging, because hair has complex geometry and appearance and exhibits challenging motion. In this paper, we present a two-stage approach that models hair independently of the head to address these challenges in a data-driven manner. The first stage, state compression, learns a low-dimensional latent space of 3D hair states including motion and appearance via a novel autoencoder-as-a-tracker strategy. To better disentangle the hair and head in appearance learning, we employ multi-view hair segmentation masks in combination with a differentiable volumetric renderer. The second stage optimizes a novel hair dynamics model that performs temporal hair transfer based on the discovered latent codes. To enforce higher stability while driving our dynamics model, we employ the 3D point-cloud autoencoder from the compression stage for denoising of the hair state. Our model outperforms the state of the art in novel view synthesis and is capable of creating novel hair animations without relying on hair observations as a driving signal. † † Project page at https://ziyanwl.github.io/neuwigs/.}
}
@inproceedings{santesteban2022snug,
title={Snug: Self-supervised neural dynamic garments},
author={Santesteban, Igor and Otaduy, Miguel A and Casas, Dan},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={8140--8150},
year={2022},
category={cloth, neural_solver},
link={http://mslab.es/projects/SNUG/},
abstract={We present a self-supervised method to learn dynamic 3D deformations of garments worn by parametric human bodies. State-of-the-art data-driven approaches to model 3D garment deformations are trained using supervised strategies that require large datasets, usually obtained by expensive physics-based simulation methods or professional multi-camera capture setups. In contrast, we propose a new training scheme that removes the need for ground-truth samples, enabling self-supervised training of dynamic 3D garment deformations. Our key contribution is to realize that physics-based deformation models, traditionally solved in a frame-by-frame basis by implicit integrators, can be recasted as an optimization problem. We leverage such optimization-based scheme to formulate a set of physics-based loss terms that can be used to train neural networks without precomputing ground-truth data. This allows us to learn models for interactive garments, including dynamic deformations and fine wrinkles, with a two orders of magnitude speed up in training time compared to state-of-the-art supervised methods.}
}
@inproceedings{santesteban2021self,
title={Self-supervised collision handling via generative 3d garment models for virtual try-on},
author={Santesteban, Igor and Thuerey, Nils and Otaduy, Miguel A and Casas, Dan},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={11763--11773},
year={2021},
category={cloth, contact, avatar},
link={http://mslab.es/projects/SelfSupervisedGarmentCollisions/},
abstract={We propose a new generative model for 3D garment deformations that enables us to learn, for the first time, a data-driven method for virtual try-on that effectively addresses garment-body collisions. In contrast to existing methods that require an undesirable postprocessing step to fix garment-body interpenetrations at test time, our approach directly outputs 3D garment configurations that do not collide with the underlying body. Key to our success is a new canonical space for garments that removes pose-and-shape deformations already captured by a new diffused human body model, which extrapolates body surface properties such as skinning weights and blendshapes to any 3D point. We leverage this representation to train a generative model with a novel self-supervised collision term that learns to reliably solve garment-body interpenetrations. We extensively evaluate and compare our results with recently proposed data-driven methods, and show that our method is the first to successfully address garment-body contact in unseen body shapes and motions, without compromising realism and detail.}
}
@inproceedings{pan2022predicting,
title={Predicting loose-fitting garment deformations using bone-driven motion networks},
author={Pan, Xiaoyu and Mai, Jiaming and Jiang, Xinwei and Tang, Dongxue and Li, Jingxiang and Shao, Tianjia and Zhou, Kun and Jin, Xiaogang and Manocha, Dinesh},
booktitle={ACM SIGGRAPH 2022 Conference Proceedings},
pages={1--10},
year={2022},
category={cloth, neural_solver},
link={http://www.cad.zju.edu.cn/home/jin/SigCloth2022/SigCloth2022.htm},
abstract={We present a learning algorithm that uses bone-driven motion networks to predict the deformation of loose-fitting garment meshes at interactive rates. Given a garment, we generate a simulation database and extract virtual bones from simulated mesh sequences using skin decomposition. At runtime, we separately compute low- and high-frequency deformations in a sequential manner. The low-frequency deformations are predicted by transferring body motions to virtual bones' motions, and the high-frequency deformations are estimated leveraging the global information of virtual bones' motions and local information extracted from low-frequency meshes. In addition, our method can estimate garment deformations caused by variations of the simulation parameters (e.g., fabric's bending stiffness) using an RBF kernel ensembling trained networks for different sets of simulation parameters. Through extensive comparisons, we show that our method outperforms state-of-the-art methods in terms of prediction accuracy of mesh deformations by about 20% in RMSE and 10% in Hausdorff distance and STED. The code and data are available at https://github.com/non-void/VirtualBones.}
}
@article{romero2021learning,
title={Learning contact corrections for handle-based subspace dynamics},
author={Romero, Cristian and Casas, Dan and P{\'e}rez, Jes{\'u}s and Otaduy, Miguel},
journal={ACM Transactions on Graphics (TOG)},
volume={40},
number={4},
pages={1--12},
year={2021},
category={softbody,neural_solver,contact},
publisher={ACM New York, NY, USA},
link={http://mslab.es/projects/LearningContactCorrections/},
abstract={This paper introduces a novel subspace method for the simulation of dynamic deformations. The method augments existing linear handle-based subspace formulations with nonlinear learning-based corrections parameterized by the same subspace. Together, they produce a compact nonlinear model that combines the fast dynamics and overall contact-based interaction of subspace methods, with the highly detailed deformations of learning-based methods. We propose a formulation of the model with nonlinear corrections applied on the local undeformed setting, and decoupling internal and external contact-driven corrections. We define a simple mapping of these corrections to the global setting, an efficient implementation for dynamic simulation, and a training pipeline to generate examples that efficiently cover the interaction space. Altogether, the method achieves unprecedented combination of speed and contact-driven deformation detail.}
}
@article{shen2021high,
author = {Shen, Siyuan and Yang, Yin and Shao, Tianjia and Wang, He and Jiang, Chenfanfu and Lan, Lei and Zhou, Kun},
title = {High-order differentiable autoencoder for nonlinear model reduction},
year = {2021},
category={softbody,differentiable_simulation,neural_representation},
issue_date = {August 2021},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {40},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3450626.3459754},
doi = {10.1145/3450626.3459754},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {68},
numpages = {15},
keywords = {model reduction, differentiation, deformable model, autoencoder, GPU},
link={https://dl.acm.org/doi/10.1145/3450626.3459754},
abstract={This paper provides a new avenue for exploiting deep neural networks to improve physics-based simulation. Specifically, we integrate the classic Lagrangian mechanics with a deep autoencoder to accelerate elastic simulation of deformable solids. Due to the inertia effect, the dynamic equilibrium cannot be established without evaluating the second-order derivatives of the deep autoencoder network. This is beyond the capability of off-the-shelf automatic differentiation packages and algorithms, which mainly focus on the gradient evaluation. Solving the nonlinear force equilibrium is even more challenging if the standard Newton's method is to be used. This is because we need to compute a third-order derivative of the network to obtain the variational Hessian. We attack those difficulties by exploiting complex-step finite difference, coupled with reverse automatic differentiation. This strategy allows us to enjoy the convenience and accuracy of complex-step finite difference and in the meantime, to deploy complex-value perturbations as collectively as possible to save excessive network passes. With a GPU-based implementation, we are able to wield deep autoencoders (e.g., 10+ layers) with a relatively high-dimension latent space in real-time. Along this pipeline, we also design a sampling network and a weighting network to enable weight-varying Cubature integration in order to incorporate nonlinearity in the model reduction. We believe this work will inspire and benefit future research efforts in nonlinearly reduced physical simulation problems.}
}
@article{romero2022contact,
title={Contact-centric deformation learning},
author={Romero, Cristian and Casas, Dan and Chiaramonte, Maurizio M and Otaduy, Miguel A},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={4},
pages={1--11},
year={2022},
category={softbody,neural_solver,contact},
publisher={ACM New York, NY, USA},
link={http://mslab.es/projects/ContactCentricLearning/},
abstract={We propose a novel method to machine-learn highly detailed, nonlinear contact deformations for real-time dynamic simulation. We depart from previous deformation-learning strategies, and model contact deformations in a contact-centric manner. This strategy shows excellent generalization with respect to the object's configuration space, and it allows for simple and accurate learning. We complement the contact-centric learning strategy with two additional key ingredients: learning a continuous vector field of contact deformations, instead of a discrete approximation; and sparsifying the mapping between the contact configuration and contact deformations. These two ingredients further contribute to the accuracy, efficiency, and generalization of the method. We integrate our learning-based contact deformation model with subspace dynamics, showing real-time dynamic simulations with fine contact deformation detail.}
}
@inproceedings{romero2023learning,
title={Learning Contact Deformations with General Collider Descriptors},
author={Romero, Cristian and Casas, Dan and Chiaramonte, Maurizio and Otaduy, Miguel A},
booktitle={SIGGRAPH Asia 2023 Conference Papers},
pages={1--10},
year={2023},
category={softbody,embodied_ai,contact},
link={https://dancasas.github.io/},
abstract={This paper presents a learning-based method for the simulation of rich contact deformations on reduced deformation models. Previous works learn deformation models for specific pairs of objects; we lift this limitation by designing a neural model that supports general rigid collider shapes. We do this by formulating a novel collider descriptor that characterizes local geometry in a region of interest. The paper shows that the learning-based deformation model can be trained on a library of colliders, but it accurately supports unseen collider shapes at runtime. We showcase our method on interactive dynamic simulations with animation of rich deformation detail, manipulation and exploration of untrained objects, and augmentation of contact information suitable for high-fidelity haptics.}
}
@inproceedings{sharp2023data,
title={Data-Free Learning of Reduced-Order Kinematics},
author={Sharp, Nicholas and Romero, Cristian and Jacobson, Alec and Vouga, Etienne and Kry, Paul and Levin, David IW and Solomon, Justin},
booktitle={ACM SIGGRAPH 2023 Conference Proceedings},
pages={1--9},
year={2023},
category={cloth, neural_representation, embodied_ai},
link={https://nmwsharp.com/research/neural-physics-subspaces/},
abstract={Physical systems ranging from elastic bodies to kinematic linkages are defined on high-dimensional configuration spaces, yet their typical low-energy configurations are concentrated on much lower-dimensional subspaces. This work addresses the challenge of identifying such subspaces automatically: given as input an energy function for a high-dimensional system, we produce a low-dimensional map whose image parameterizes a diverse yet low-energy submanifold of configurations. The only additional input needed is a single seed configuration for the system to initialize our procedure; no dataset of trajectories is required. We represent subspaces as neural networks that map a low-dimensional latent vector to the full configuration space, and propose a training scheme to fit network parameters to any system of interest. This formulation is effective across a very general range of physical systems; our experiments demonstrate not only nonlinear and very low-dimensional elastic body and cloth subspaces, but also more general systems like colliding rigid bodies and linkages. We briefly explore applications built on this formulation, including manipulation, latent interpolation, and sampling.}
}
@article{yang2022learning,
title={Learning physics constrained dynamics using autoencoders},
author={Yang, Tsung-Yen and Rosca, Justinian and Narasimhan, Karthik and Ramadge, Peter J},
journal={Advances in Neural Information Processing Systems},
volume={35},
pages={17157--17172},
year={2022},
category={rigidbody, neural_representation, neural_solver},
link={https://proceedings.neurips.cc/paper_files/paper/2022/hash/6d5e035724687454549b97d6c805dc84-Abstract-Conference.html}
}
@article{han2022learning,
title={Learning physical dynamics with subequivariant graph neural networks},
author={Han, Jiaqi and Huang, Wenbing and Ma, Hengbo and Li, Jiachen and Tenenbaum, Josh and Gan, Chuang},
journal={Advances in Neural Information Processing Systems},
volume={35},
pages={26256--26268},
year={2022},
category={rigidbody,neural_solver},
tag={gnn},
link={https://hanjq17.github.io/SGNN/},
abstract={Graph Neural Networks (GNNs) have become a prevailing tool for learning physical dynamics. However, they still encounter several challenges: 1) Physical laws abide by symmetry, which is a vital inductive bias accounting for model generalization and should be incorporated into the model design. Existing simulators either consider insufficient symmetry, or enforce excessive equivariance in practice when symmetry is partially broken by gravity. 2) Objects in the physical world possess diverse shapes, sizes, and properties, which should be appropriately processed by the model. To tackle these difficulties, we propose a novel backbone, Subequivariant Graph Neural Network, which 1) relaxes equivariance to subequivariance by considering external fields like gravity, where the universal approximation ability holds theoretically; 2) introduces a new subequivariant object-aware message passing for learning physical interactions between multiple objects of various shapes in the particle-based representation; 3) operates in a hierarchical fashion, allowing for modeling long-range and complex interactions. Our model achieves on average over 3% enhancement in contact prediction accuracy across 8 scenarios on Physion and 2X lower rollout MSE on RigidFall compared with state-of-the-art GNN simulators, while exhibiting strong generalization and data efficiency.}
}
@inproceedings{ma2023learning,
title={Learning neural constitutive laws from motion observations for generalizable pde dynamics},
author={Ma, Pingchuan and Chen, Peter Yichen and Deng, Bolei and Tenenbaum, Joshua B and Du, Tao and Gan, Chuang and Matusik, Wojciech},
booktitle={International Conference on Machine Learning},
pages={23279--23300},
year={2023},
category={multiphys,differentiable_simulation,neural_solver},
tag={neural_material},
organization={PMLR},
link={https://sites.google.com/view/nclaw},
abstract={We propose a hybrid neural network (NN) and PDE approach for learning generalizable PDE dynamics from motion observations. Many NN approaches learn an end-to-end model that implicitly models both the governing PDE and constitutive models (or material models). Without explicit PDE knowledge, these approaches cannot guarantee physical correctness and have limited generalizability. We argue that the governing PDEs are often well-known and should be explicitly enforced rather than learned. Instead, constitutive models are particularly suitable for learning due to their data-fitting nature. To this end, we introduce a new framework termed "Neural Constitutive Laws" (NCLaw), which utilizes a network architecture that strictly guarantees standard constitutive priors, including rotation equivariance and undeformed state equilibrium. We embed this network inside a differentiable simulation and train the model by minimizing a loss function based on the difference between the simulation and the motion observation. We validate NCLaw on various large-deformation dynamical systems, ranging from solids to fluids. After training on a single motion trajectory, our method generalizes to new geometries, initial/boundary conditions, temporal ranges, and even multi-physics systems. On these extremely out-of-distribution generalization tasks, NCLaw is orders-of-magnitude more accurate than previous NN approaches. Real-world experiments demonstrate our method's ability to learn constitutive laws from videos.}
}
@InProceedings{hennigh2021nvidia,
author="Hennigh, Oliver
and Narasimhan, Susheela
and Nabian, Mohammad Amin
and Subramaniam, Akshay
and Tangsali, Kaustubh
and Fang, Zhiwei
and Rietmann, Max
and Byeon, Wonmin
and Choudhry, Sanjay",
editor="Paszynski, Maciej
and Kranzlm{\"u}ller, Dieter
and Krzhizhanovskaya, Valeria V.
and Dongarra, Jack J.
and Sloot, Peter M.A.",
title="NVIDIA SimNet{\texttrademark}: An AI-Accelerated Multi-Physics Simulation Framework",
booktitle="Computational Science -- ICCS 2021",
year="2021",
category={multiphys,engine},
publisher="Springer International Publishing",
address="Cham",
pages="447--461",
isbn="978-3-030-77977-1",
link="https://arxiv.org/abs/2012.07938"
}
@article{xie2024fluid,
title={Fluid Inverse Volumetric Modeling and Applications from Surface Motion},
author={Xie, Xueguang and Gao, Yang and Hou, Fei and Cheng, Tianwei and Hao, Aimin and Qin, Hong},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2024},
category={fluid},
tag={parameter_estimation,cnn},
publisher={IEEE},
link={https://www.computer.org/csdl/journal/tg/5555/01/10452823/1UUuVAIPShy},
abstract={In this study, we devise a framework for volumetrically reconstructing fluid from observable, measurable free surface motion. Our innovative method amalgamates the benefits of deep learning and conventional simulation to preserve the guiding motion and temporal coherence of the reproduced fluid. We infer surface velocities by encoding and decoding spatiotemporal features of surface sequences, and a 3D CNN is used to generate the volumetric velocity field, which is then combined with 3D labels of obstacles and boundaries. Concurrently, we employ a network to estimate the fluid's physical properties. To progressively evolve the flow field over time, we input the reconstructed velocity field and estimated parameters into the physical simulator as the initial state. Our approach yields promising results for both synthetic fluid generated by different fluid solvers and captured real fluid. The developed framework naturally lends itself to a variety of graphics applications, such as 1) effective reproductions of fluid behaviors visually congruent with the observed surface motion, and 2) physics-guided re-editing of fluid scenes. Extensive experiments affirm that our novel method surpasses state-of-the-art approaches for 3D fluid inverse modeling and animation in graphics.}
}
@inproceedings{franz2021global,
title={Global transport for fluid reconstruction with learned self-supervision},
author={Franz, Erik and Solenthaler, Barbara and Thuerey, Nils},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={1632--1642},
year={2021},
category={fluid,reconstruction},
tag={fluid_reconstruction},
link={https://ge.in.tum.de/publications/},
abstract={We propose a novel method to reconstruct volumetric flows from sparse views via a global transport formulation. Instead of obtaining the space-time function of the observations, we reconstruct its motion based on a single initial state. In addition we introduce a learned self-supervision that constrains observations from unseen angles. These visual constraints are coupled via the transport constraints and a differentiable rendering step to arrive at a robust end-to-end reconstruction algorithm. This makes the reconstruction of highly realistic flow motions possible, even from only a single input view. We show with a variety of synthetic and real flows that the proposed global reconstruction of the transport process yields an improved reconstruction of the fluid motion.}
}
@article{xiao2024laplacian,
title={Laplacian Projection Based Global Physical Prior Smoke Reconstruction},
author={Xiao, Shibang and Tong, Chao and Zhang, Qifan and Cen, Yunchi and Li, Frederick WB and Liang, Xiaohui},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2024},
category={fluid,differentiable_simulation,reconstruction},
tag={fluid_reconstruction},
publisher={IEEE},
link={https://www.computer.org/csdl/journal/tg/5555/01/10414126/1TZIJbrBNo4},
abstract={We present a novel framework for reconstructing fluid dynamics in real-life scenarios. Our approach leverages sparse view images and incorporates physical priors across long series of frames, resulting in reconstructed fluids with enhanced physical consistency. Unlike previous methods, we utilize a differentiable fluid simulator (DFS) and a differentiable renderer (DR) to exploit global physical priors, reducing reconstruction errors without the need for manual regularization coefficients. We introduce divergence-free Laplacian eigenfunctions (div-free LE) as velocity bases, improving computational efficiency and memory usage. By employing gradient-related strategies, we achieve better convergence and superior results. Extensive experiments demonstrate the effectiveness of our method, showcasing improved reconstruction quality and computational efficiency compared to existing approaches. We validate our approach using both synthetic and real data, highlighting its practical potential.}
}
@article{yu2024inferring,
title={Inferring Hybrid Neural Fluid Fields from Videos},
author={Yu, Hong-Xing and Zheng, Yang and Gao, Yuan and Deng, Yitong and Zhu, Bo and Wu, Jiajun},
journal={Advances in Neural Information Processing Systems},
volume={36},
year={2023},
category={fluid,neural_solver,reconstruction},
tag={real2sim,fluid_reconstruction},
link={https://papers.nips.cc/paper_files/paper/2023/hash/00feea0d4eea58fdda7151f7e7f76c72-Abstract-Conference.html},
project={https://kovenyu.com/hyfluid/},
abstract={We study recovering fluid density and velocity from sparse multiview videos. Existing neural dynamic reconstruction methods predominantly rely on optical flows; therefore, they cannot accurately estimate the density and uncover the underlying velocity due to the inherent visual ambiguities of fluid velocity, as fluids are often shapeless and lack stable visual features. The challenge is further pronounced by the turbulent nature of fluid flows, which calls for properly designed fluid velocity representations. To address these challenges, we propose hybrid neural fluid fields (HyFluid), a neural approach to jointly infer fluid density and velocity fields. Specifically, to deal with visual ambiguities of fluid velocity, we introduce a set of physics-based losses that enforce inferring a physically plausible velocity field, which is divergence-free and drives the transport of density. To deal with the turbulent nature of fluid velocity, we design a hybrid neural velocity representation that includes a base neural velocity field that captures most irrotational energy and a vortex particle-based velocity that models residual turbulent velocity. We show that our method enables recovering vortical flow details. Our approach opens up possibilities for various learning and reconstruction applications centered around 3D incompressible flow, including fluid re-simulation and editing, future prediction, and neural dynamic scene composition. Project website: https://kovenyu.com/HyFluid/}
}
@article{xie2023physgaussian,
title={PhysGaussian: Physics-Integrated 3D Gaussians for Generative Dynamics},
author={Xie, Tianyi and Zong, Zeshun and Qiu, Yuxing and Li, Xuan and Feng, Yutao and Yang, Yin and Jiang, Chenfanfu},
journal={arXiv preprint arXiv:2311.12198},
year={2023},
category={fluid, neural_representation},
tag={3dgs},
link={https://xpandora.github.io/PhysGaussian/},
abstract={We introduce PhysGaussian, a new method that seamlessly integrates physically grounded Newtonian dynamics within 3D Gaussians to achieve high-quality novel motion synthesis. Employing a custom Material Point Method (MPM), our approach enriches 3D Gaussian kernels with physically meaningful kinematic deformation and mechanical stress attributes, all evolved in line with continuum mechanics principles. A defining characteristic of our method is the seamless integration between physical simulation and visual rendering: both components utilize the same 3D Gaussian kernels as their discrete representations. This negates the necessity for triangle/tetrahedron meshing, marching cubes, "cage meshes," or any other geometry embedding, highlighting the principle of "what you see is what you simulate (WS$^2$)." Our method demonstrates exceptional versatility across a wide variety of materials--including elastic entities, metals, non-Newtonian fluids, and granular materials--showcasing its strong capabilities in creating diverse visual content with novel viewpoints and movements. Our project page is at: https://xpandora.github.io/PhysGaussian/}
}
@misc{feng2024gaussian,
title={Gaussian Splashing: Dynamic Fluid Synthesis with Gaussian Splatting},
author={Yutao Feng and Xiang Feng and Yintong Shang and Ying Jiang and Chang Yu and Zeshun Zong and Tianjia Shao and Hongzhi Wu and Kun Zhou and Chenfanfu Jiang and Yin Yang},
year={2024},
category={fluid},
tag={nerf,3dgs},
eprint={2401.15318},
archivePrefix={arXiv},
primaryClass={cs.GR},
link={https://amysteriouscat.github.io/GaussianSplashing/},
abstract={3D Gaussian splatting (GS) has emerged as a transformative technique in radiance fields. Unlike mainstream implicit neural models, 3D GS uses millions of learnable 3D Gaussians for an explicit scene representation. Paired with a differentiable rendering algorithm, this approach achieves real-time rendering and unprecedented editability, making it a potential game-changer for 3D reconstruction and representation. In the present paper, we provide the first systematic overview of the recent developments and critical contributions in 3D GS. We begin with a detailed exploration of the underlying principles and the driving forces behind the emergence of 3D GS, laying the groundwork for understanding its significance. A focal point of our discussion is the practical applicability of 3D GS. By enabling unprecedented rendering speed, 3D GS opens up a plethora of applications, ranging from virtual reality to interactive media and beyond. This is complemented by a comparative analysis of leading 3D GS models, evaluated across various benchmark tasks to highlight their performance and practical utility. The survey concludes by identifying current challenges and suggesting potential avenues for future research. Through this survey, we aim to provide a valuable resource for both newcomers and seasoned researchers, fostering further exploration and advancement in explicit radiance field.}
}
@inproceedings{guan2022neurofluid,
title={Neurofluid: Fluid dynamics grounding with particle-driven neural radiance fields},
author={Guan, Shanyan and Deng, Huayu and Wang, Yunbo and Yang, Xiaokang},
booktitle={International Conference on Machine Learning},
pages={7919--7929},
year={2022},
category={fluid,neural_solver,reconstruction},
tag={fluid_reconstruction,parameter_estimation},
organization={PMLR},
link={https://syguan96.github.io/NeuroFluid/},
abstract={Deep learning has shown great potential for modeling the physical dynamics of complex particle systems such as fluids. Existing approaches, however, require the supervision of consecutive particle properties, including positions and velocities. In this paper, we consider a partially observable scenario known as fluid dynamics grounding, that is, inferring the state transitions and interactions within the fluid particle systems from sequential visual observations of the fluid surface. We propose a differentiable two-stage network named NeuroFluid. Our approach consists of (i) a particle-driven neural renderer, which involves fluid physical properties into the volume rendering function, and (ii) a particle transition model optimized to reduce the differences between the rendered and the observed images. NeuroFluid provides the first solution to unsupervised learning of particle-based fluid dynamics by training these two models jointly. It is shown to reasonably estimate the underlying physics of fluids with different initial shapes, viscosity, and densities.}
}
@article{halimi2022pattern,
title={Pattern-based cloth registration and sparse-view animation},
author={Halimi, Oshri and Stuyck, Tuur and Xiang, Donglai and Bagautdinov, Timur and Wen, He and Kimmel, Ron and Shiratori, Takaaki and Wu, Chenglei and Sheikh, Yaser and Prada, Fabian},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={6},
pages={1--17},
year={2022},
category={cloth,reconstruction,avatar},
tag={real2sim,cloth_reconstruction},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/abs/10.1145/3550454.3555448},
abstract={We propose a novel multi-view camera pipeline for the reconstruction and registration of dynamic clothing. Our proposed method relies on a specifically designed pattern that allows for precise video tracking in each camera view. We triangulate the tracked points and register the cloth surface in a fine-grained geometric resolution and low localization error. Compared to state-of-the-art methods, our registration exhibits stable correspondence, tracking the same points on the deforming cloth surface along the temporal sequence. As an application, we demonstrate how the use of our registration pipeline greatly improves state-of-the-art pose-based drivable cloth models. Furthermore, we propose a novel model, Garment Avatar , for driving cloth from a dense tracking signal which is obtained from two opposing camera views. The method produces realistic reconstructions which are faithful to the actual geometry of the deforming cloth. In this setting, the user wears a garment with our custom pattern which enables our driving model to reconstruct the geometry. Our code and data are available at https://github.com/HalimiOshri/Pattern-Based-Cloth-Registration-and-Sparse-View-Animation. The released data includes our pattern and registered mesh sequences containing four different subjects and 15k frames in total.}
}
@article{feng2022learning,
title={Learning-based bending stiffness parameter estimation by a drape tester},
author={Feng, Xudong and Huang, Wenchao and Xu, Weiwei and Wang, Huamin},
journal={ACM Transactions on Graphics (TOG)},
volume={41},
number={6},
pages={1--16},
year={2022},
category={cloth,reconstruction},
tag={real2sim,cloth_reconstruction,parameter_estimation},
publisher={ACM New York, NY, USA},
link={https://github.com/DrapeTester/ClothDrapeTester},
abstract={Real-world fabrics often possess complicated nonlinear, anisotropic bending stiffness properties. Measuring the physical parameters of such properties for physics-based simulation is difficult yet unnecessary, due to the persistent existence of numerical errors in simulation technology. In this work, we propose to adopt a simulation-in-the-loop strategy: instead of measuring the physical parameters, we estimate the simulation parameters to minimize the discrepancy between reality and simulation. This strategy offers good flexibility in test setups, but the associated optimization problem is computationally expensive to solve by numerical methods. Our solution is to train a regression-based neural network for inferring bending stiffness parameters, directly from drape features captured in the real world. Specifically, we choose the Cusick drape test method and treat multiple-view depth images as the feature vector. To effectively and efficiently train our network, we develop a highly expressive and physically validated bending stiffness model, and we use the traditional cantilever test to collect the parameters of this model for 618 real-world fabrics. Given the whole parameter data set, we then construct a parameter subspace, generate new samples within the sub-space, and finally simulate and augment synthetic data for training purposes. The experiment shows that our trained system can replace cantilever tests for quick, reliable and effective estimation of simulation-ready parameters. Thanks to the use of the system, our simulator can now faithfully simulate bending effects comparable to those in the real world.}
}
@article{qiao2022neuphysics,
title={Neuphysics: Editable neural geometry and physics from monocular videos},
author={Qiao, Yi-Ling and Gao, Alexander and Lin, Ming},
journal={Advances in Neural Information Processing Systems},
volume={35},
pages={12841--12854},
year={2022},
category={softbody,differentiable_simulation,neural_representation,reconstruction},
tag={real2sim,neural_field},
link={https://sites.google.com/view/neuphysics/home},
abstract={We present a method for learning 3D geometry and physics parameters of a dynamic scene from only a monocular RGB video input. To decouple the learning of underlying scene geometry from dynamic motion, we represent the scene as a time-invariant signed distance function (SDF) which serves as a reference frame, along with a time-conditioned deformation field. We further bridge this neural geometry representation with a differentiable physics simulator by designing a two-way conversion between the neural field and its corresponding hexahedral mesh, enabling us to estimate physics parameters from the source video by minimizing a cycle consistency loss. Our method also allows a user to interactively edit 3D objects from the source video by modifying the recovered hexahedral mesh, and propagating the operation back to the neural field representation. Experiments show that our method achieves superior mesh and video reconstruction of dynamic scenes compared to competing Neural Field approaches, and we provide extensive examples which demonstrate its ability to extract useful 3D representations from videos captured with consumer-grade cameras.}
}
@article{li2024garmentdreamer,
title={GarmentDreamer: 3DGS Guided Garment Synthesis with Diverse Geometry and Texture Details},
author={Li, Boqian and Li, Xuan and Jiang, Ying and Xie, Tianyi and Gao, Feng and Wang, Huamin and Yang, Yin and Jiang, Chenfanfu},
journal={arXiv preprint arXiv:2405.12420},
year={2024},
category={cloth},
tag={3dgs},
link={https://arxiv.org/abs/2405.12420},
abstract={Traditional 3D garment creation is labor-intensive, involving sketching, modeling, UV mapping, and texturing, which are time-consuming and costly. Recent advances in diffusion-based generative models have enabled new possibilities for 3D garment generation from text prompts, images, and videos. However, existing methods either suffer from inconsistencies among multi-view images or require additional processes to separate cloth from the underlying human model. In this paper, we propose GarmentDreamer, a novel method that leverages 3D Gaussian Splatting (GS) as guidance to generate wearable, simulation-ready 3D garment meshes from text prompts. In contrast to using multi-view images directly predicted by generative models as guidance, our 3DGS guidance ensures consistent optimization in both garment deformation and texture synthesis. Our method introduces a novel garment augmentation module, guided by normal and RGBA information, and employs implicit Neural Texture Fields (NeTF) combined with Score Distillation Sampling (SDS) to generate diverse geometric and texture details. We validate the effectiveness of our approach through comprehensive qualitative and quantitative experiments, showcasing the superior performance of GarmentDreamer over state-of-the-art alternatives. Our project page is available at: https://xuan-li.github.io/GarmentDreamerDemo/.}
}
@article{zhang2024physdreamer,
title={{PhysDreamer}: Physics-Based Interaction with 3D Objects via Video Generation},
author={Tianyuan Zhang and Hong-Xing Yu and Rundi Wu and
Brandon Y. Feng and Changxi Zheng and Noah Snavely and Jiajun Wu and William T. Freeman},
journal={arxiv},
year={2024},
category={softbody,reconstruction},
tag={real2sim,material_estimation},
link={https://physdreamer.github.io/},
abstract={Realistic object interactions are crucial for creating immersive virtual experiences, yet synthesizing realistic 3D object dynamics in response to novel interactions remains a significant challenge. Unlike unconditional or text-conditioned dynamics generation, action-conditioned dynamics requires perceiving the physical material properties of objects and grounding the 3D motion prediction on these properties, such as object stiffness. However, estimating physical material properties is an open problem due to the lack of material ground-truth data, as measuring these properties for real objects is highly difficult. We present PhysDreamer, a physics-based approach that endows static 3D objects with interactive dynamics by leveraging the object dynamics priors learned by video generation models. By distilling these priors, PhysDreamer enables the synthesis of realistic object responses to novel interactions, such as external forces or agent manipulations. We demonstrate our approach on diverse examples of elastic objects and evaluate the realism of the synthesized interactions through a user study. PhysDreamer takes a step towards more engaging and realistic virtual experiences by enabling static 3D objects to dynamically respond to interactive stimuli in a physically plausible manner. See our project page at https://physdreamer.github.io/.}
}
@inproceedings{jiang2025phystwin,
title={PhysTwin: Physics-Informed Reconstruction and Simulation of Deformable Objects from Videos},
author={Jiang, Hanxiao and Hsu, Hao-Yu and Zhang, Kaifeng and Yu, Hsin-Ni and Wang, Shenlong and Li, Yunzhu},
booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
year={2025},
category={cloth,reconstruction,embodied_ai},
tag={real2sim,cloth_reconstruction},
eprint={2503.17973},
archivePrefix={arXiv},
link={https://jianghanxiao.github.io/phystwin-web/},
abstract={Creating a physical digital twin of a real-world object has immense potential in robotics, content creation, and XR. In this paper, we present PhysTwin, a novel framework that uses sparse videos of dynamic objects under interaction to produce a photo- and physically realistic, realtime interactive virtual replica. Our approach centers on two key components: (1) a physics-informed representation that combines spring-mass models for realistic physical simulation, generative shape models for geometry, and Gaussian splats for rendering; and (2) a novel multi-stage, optimization-based inverse modeling framework that reconstructs complete geometry, infers dense physical properties, and replicates realistic appearance from videos. Our method integrates an inverse physics framework with visual perception cues, enabling high-fidelity reconstruction even from partial, occluded, and limited viewpoints. PhysTwin supports modeling various deformable objects, including ropes, stuffed animals, cloth, and delivery packages. Experiments show that PhysTwin outperforms competing methods in reconstruction, rendering, future prediction, and simulation under novel interactions. We further demonstrate its applications in interactive real-time simulation and model-based robotic motion planning.}
}
@inproceedings{feng2024pie,
title={Pie-nerf: Physics-based interactive elastodynamics with nerf},
author={Feng, Yutao and Shang, Yintong and Li, Xuan and Shao, Tianjia and Jiang, Chenfanfu and Yang, Yin},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={4450--4461},
year={2024},
category={softbody,reconstruction},
tag={real2sim,user_interaction,nerf},
link={https://fytalon.github.io/pienerf/},
abstract={We show that physics-based simulations can be seamlessly integrated with NeRF to generate high-quality elastodynamics of real-world objects. Unlike existing methods, we discretize nonlinear hyperelasticity in a meshless way, obviating the necessity for intermediate auxiliary shape proxies like a tetrahedral mesh or voxel grid. A quadratic generalized moving least square is employed to capture nonlinear dynamics and large deformation on the implicit model. Such meshless integration enables versatile simulations of complex and codimensional shapes. We adaptively place the least-square kernels according to the NeRF density field to significantly reduce the complexity of the nonlinear simulation. As a result, physically realistic animations can be conveniently synthesized using our method for a wide range of hyperelastic materials at an interactive rate. For more information, please visit our project page.}
,
}
@article{feng2024elastogen,
title={ElastoGen: 4D Generative Elastodynamics},
author={Feng, Yutao and Shang, Yintong and Feng, Xiang and Lan, Lei and Zhe, Shandian and Shao, Tianjia and Wu, Hongzhi and Zhou, Kun and Su, Hao and Jiang, Chenfanfu and others},
journal={arXiv preprint arXiv:2405.15056},
year={2024},
category={softbody, neural_solver},
link={https://arxiv.org/abs/2405.15056},
abstract={We present ElastoGen, a knowledge-driven AI model that generates physically accurate 4D elastodynamics. Unlike deep models that learn from video- or image-based observations, ElastoGen leverages the principles of physics and learns from established mathematical and optimization procedures. The core idea of ElastoGen is converting the differential equation, corresponding to the nonlinear force equilibrium, into a series of iterative local convolution-like operations, which naturally fit deep architectures. We carefully build our network module following this overarching design philosophy. ElastoGen is much more lightweight in terms of both training requirements and network scale than deep generative models. Because of its alignment with actual physical procedures, ElastoGen efficiently generates accurate dynamics for a wide range of hyperelastic materials and can be easily integrated with upstream and downstream deep modules to enable end-to-end 4D generation.}
}
@article{huang2023diffvl,
title={DiffVL: Scaling Up Soft Body Manipulation using Vision-Language Driven Differentiable Physics},
author={Huang, Zhiao and Chen, Feng and Pu, Yewen and Lin, Chunru and Su, Hao and Gan, Chuang},
journal={Advances in Neural Information Processing Systems},
volume={36},
pages={29875--29900},
year={2023},
category={softbody,differentiable_simulation,embodied_ai},
link={https://arxiv.org/abs/2312.06408},
abstract={Combining gradient-based trajectory optimization with differentiable physics simulation is an efficient technique for solving soft-body manipulation problems. Using a well-crafted optimization objective, the solver can quickly converge onto a valid trajectory. However, writing the appropriate objective functions requires expert knowledge, making it difficult to collect a large set of naturalistic problems from non-expert users. We introduce DiffVL, a method that enables non-expert users to communicate soft-body manipulation tasks -- a combination of vision and natural language, given in multiple stages -- that can be readily leveraged by a differential physics solver. We have developed GUI tools that enable non-expert users to specify 100 tasks inspired by real-life soft-body manipulations from online videos, which we'll make public. We leverage large language models to translate task descriptions into machine-interpretable optimization objectives. The optimization objectives can help differentiable physics solvers to solve these long-horizon multistage tasks that are challenging for previous baselines.}
}
@article{su2023generalized,
title={A generalized constitutive model for versatile mpm simulation and inverse learning with differentiable physics},
author={Su, Haozhe and Li, Xuan and Xue, Tao and Jiang, Chenfanfu and Aanjaneya, Mridul},
journal={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
volume={6},
number={3},
pages={1--20},
year={2023},
category={fluid,differentiable_simulation},
publisher={ACM New York, NY, USA},
link={https://xuan-li.github.io/publication/su2023generalized/},
abstract={We present a generalized constitutive model for versatile physics simulation of inviscid fluids, Newtonian viscosity, hyperelasticity, viscoplasticity, elastoplasticity, and other physical effects that arise due to a mixture of these behaviors. The key ideas behind our formulation are the design of a generalized Kirchhoff stress tensor that can describe hyperelasticity, Newtonian viscosity and inviscid fluids, and the use of pre-projection and post-correction rules for simulating material behaviors that involve plasticity, including elastoplasticity and viscoplasticity. We show how our generalized Kirchhoff stress tensor can be coupled together into a generalized constitutive model that allows the simulation of diverse material behaviors by only changing parameter values. We present several side-by-side comparisons with physics simulations for specific constitutive models to show that our generalized model produces visually similar results. More notably, our formulation allows for inverse learning of unknown material properties directly from data using differentiable physics simulations. We present several 3D simulations to highlight the robustness of our method, even with multiple different materials. To the best of our knowledge, our approach is the first to recover the knowledge of unknown material properties without making explicit assumptions about the data.}
}
@inproceedings{
li2023pacnerf,
title={{PAC}-Ne{RF}: Physics Augmented Continuum Neural Radiance Fields for Geometry-Agnostic System Identification},
author={Xuan Li and Yi-Ling Qiao and Peter Yichen Chen and Krishna Murthy Jatavallabhula and Ming Lin and Chenfanfu Jiang and Chuang Gan},
booktitle={International Conference on Learning Representations},
year={2023},
category={fluid,differentiable_simulation,reconstruction},
tag={real2sim,fluid_reconstruction,parameter_estimation,nerf},
url={https://openreview.net/forum?id=tVkrbkz42vc},
link={https://sites.google.com/view/PAC-NeRF},
abstract={Existing approaches to system identification (estimating the physical parameters of an object) from videos assume known object geometries. This precludes their applicability in a vast majority of scenes where object geometries are complex or unknown. In this work, we aim to identify parameters characterizing a physical system from a set of multi-view videos without any assumption on object geometry or topology. To this end, we propose "Physics Augmented Continuum Neural Radiance Fields" (PAC-NeRF), to estimate both the unknown geometry and physical parameters of highly dynamic objects from multi-view videos. We design PAC-NeRF to only ever produce physically plausible states by enforcing the neural radiance field to follow the conservation laws of continuum mechanics. For this, we design a hybrid Eulerian-Lagrangian representation of the neural radiance field, i.e., we use the Eulerian grid representation for NeRF density and color fields, while advecting the neural radiance fields via Lagrangian particles. This hybrid Eulerian-Lagrangian representation seamlessly blends efficient neural rendering with the material point method (MPM) for robust differentiable physics simulation. We validate the effectiveness of our proposed framework on geometry and physical parameter estimation over a vast range of materials, including elastic bodies, plasticine, sand, Newtonian and non-Newtonian fluids, and demonstrate significant performance gain on most tasks.}
}
@article{jiang2024vrgs,
title={VR-GS: A Physical Dynamics-Aware Interactive Gaussian Splatting System in Virtual Reality},
author={Jiang, Ying and Yu, Chang and Xie, Tianyi and Li, Xuan and Feng, Yutao and Wang, Huamin and Li, Minchen and Lau, Henry and Gao, Feng and Yang, Yin and Jiang, Chenfanfu},
journal={arXiv preprint arXiv:2401.16663},
year={2024},
category={softbody,embodied_ai},
tag={user_interaction,3dgs},
link={https://yingjiang96.github.io/VR-GS/},
abstract={As 3D content becomes increasingly prevalent, there’s a growing focus on the development of engagements with 3D virtual content. Unfortunately, traditional techniques for creating, editing, and interacting with this content are fraught with difficulties. They tend to be not only engineering-intensive but also require extensive expertise, which adds to the frustration and inefficiency in virtual object manipulation. Our proposed VR-GS system represents a leap forward in human-centered 3D content interaction, offering a seamless and intuitive user experience. By developing a physical dynamics-aware interactive Gaussian Splatting (GS) in a Virtual Reality (VR) setting, and constructing a highly efficient two-level embedding strategy alongside deformable body simulations, VR-GS ensures real-time execution with highly realistic dynamic responses. The components of our system are designed for high efficiency and effectiveness, starting from detailed scene reconstruction and object segmentation, advancing through multi-view image in-painting, and extending to interactive physics-based editing. The system also incorporates real-time deformation embedding and dynamic shadow casting, ensuring a comprehensive and engaging virtual experience.}
,
}
@article{qiu2024feature,
title={Feature Splatting: Language-Driven Physics-Based Scene Synthesis and Editing},
author={Qiu, Ri-Zhao and Yang, Ge and Zeng, Weijia and Wang, Xiaolong},
journal={arXiv preprint arXiv:2404.01223},
year={2024},
category={softbody},
tag={user_interaction,3dgs},
link={https://feature-splatting.github.io/},
abstract={Scene representations using 3D Gaussian primitives have produced excellent results in modeling the appearance of static and dynamic 3D scenes. Many graphics applications, however, demand the ability to manipulate both the appearance and the physical properties of objects. We introduce Feature Splatting, an approach that unifies physics-based dynamic scene synthesis with rich semantics from vision language foundation models that are grounded by natural language. Our first contribution is a way to distill high-quality, object-centric vision-language features into 3D Gaussians, that enables semi-automatic scene decomposition using text queries. Our second contribution is a way to synthesize physics-based dynamics from an otherwise static scene using a particle-based simulator, in which material properties are assigned automatically via text queries. We ablate key techniques used in this pipeline, to illustrate the challenge and opportunities in using feature-carrying 3D Gaussians as a unified format for appearance, geometry, material properties and semantics grounded on natural language. Project website: https://feature-splatting.github.io/}
}
@article{chu2022physics,
author = {Chu, Mengyu and Liu, Lingjie and Zheng, Quan and Franz, Erik and Seidel, Hans-Peter and Theobalt, Christian and Zayer, Rhaleb},
title = {Physics informed neural fields for smoke reconstruction with sparse data},
year = {2022},
category={fluid,neural_representation,reconstruction},
tag={real2sim,fluid_reconstruction,nerf},
issue_date = {July 2022},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {41},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3528223.3530169},
doi = {10.1145/3528223.3530169},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {119},
numpages = {14},
keywords = {NeRF, fluid reconstruction, physics-informed deep learning},
link={https://rachelcmy.github.io/pinf_smoke/},
abstract={High-fidelity reconstruction of dynamic fluids from sparse multiview RGB videos remains a formidable challenge, due to the complexity of the underlying physics as well as the severe occlusion and complex lighting in the captured data. Existing solutions either assume knowledge of obstacles and lighting, or only focus on simple fluid scenes without obstacles or complex lighting, and thus are unsuitable for real-world scenes with unknown lighting conditions or arbitrary obstacles. We present the first method to reconstruct dynamic fluid phenomena by leveraging the governing physics (ie, Navier -Stokes equations) in an end-to-end optimization from a mere set of sparse video frames without taking lighting conditions, geometry information, or boundary conditions as input. Our method provides a continuous spatio-temporal scene representation using neural networks as the ansatz of density and velocity solution functions for fluids as well as the radiance field for static objects. With a hybrid architecture that separates static and dynamic contents apart, fluid interactions with static obstacles are reconstructed for the first time without additional geometry input or human labeling. By augmenting time-varying neural radiance fields with physics-informed deep learning, our method benefits from the supervision of images and physical priors. Our progressively growing model with regularization further disentangles the density-color ambiguity in the radiance field, which allows for a more robust optimization from the given input of sparse views. A pretrained density-to-velocity fluid model is leveraged in addition as the data prior to avoid suboptimal velocity solutions which underestimate vorticity but trivially fulfill physical equations. Our method exhibits high-quality results with relaxed constraints and strong flexibility on a representative set of synthetic and real flow captures. Code and sample tests are at https://people.mpi-inf.mpg.de/~mchu/projects/PI-NeRF/.}
}
@inproceedings{qiao2023dynamic,
title={Dynamic mesh-aware radiance fields},
author={Qiao, Yi-Ling and Gao, Alexander and Xu, Yiran and Feng, Yue and Huang, Jia-Bin and Lin, Ming C},
booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
pages={385--396},
year={2023},
category={cloth,reconstruction},
tag={cloth_reconstruction,nerf},
link={https://mesh-aware-rf.github.io/},
abstract={Embedding polygonal mesh assets within photorealistic Neural Radience Fields (NeRF) volumes, such that they can be rendered and their dynamics simulated in a physically consistent manner with the NeRF, is under-explored from the system perspective of integrating NeRF into the traditional graphics pipeline. This paper designs a two-way coupling between mesh and NeRF during rendering and simulation. We first review the light transport equations for both mesh and NeRF, then distill them into an efficient algorithm for updating radiance and throughput along a cast ray with an arbitrary number of bounces. To resolve the discrepancy between the linear color space that the path tracer assumes and the sRGB color space that standard NeRF uses, we train NeRF with High Dynamic Range (HDR) images. We also present a strategy to estimate light sources and cast shadows on the NeRF. Finally, we consider how the hybrid surface-volumetric formulation can be efficiently integrated with a high-performance physics simulator that supports cloth, rigid and soft bodies. The full rendering and simulation system can be run on a GPU at interactive rates. We show that a hybrid system approach outperforms alternatives in visual realism for mesh insertion, because it allows realistic light transport from volumetric NeRF media onto surfaces, which affects the appearance of reflective/refractive surfaces and illumination of diffuse surfaces informed by the dynamic scene.}
}
@article{li2023difffr,
author = {Li, Zhehao and Xu, Qingyu and Ye, Xiaohan and Ren, Bo and Liu, Ligang},
title = {DiffFR: Differentiable SPH-Based Fluid-Rigid Coupling for Rigid Body Control},
year = {2023},
category={multiphys,differentiable_simulation,control},
tag={shape_design},
issue_date = {December 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3618318},
doi = {10.1145/3618318},
journal = {ACM Trans. Graph.},
month = {dec},
articleno = {179},
numpages = {17},
keywords = {differentiable simulation, fluid-rigid coupling, physics-based simulation},
link={https://zhehaoli1999.github.io/DiffFR/},
abstract={Differentiable physics simulation has shown its efficacy in inverse design problems. Given the pervasiveness of the diverse interactions between fluids and solids in life, a differentiable simulator for the inverse design of the motion of rigid objects in two-way fluid-rigid coupling is also demanded. There are two main challenges to develop a differentiable two-way fluid-solid coupling simulator for rigid body control tasks: the ubiquitous, discontinuous contacts in fluid-solid interactions, and the high computational cost of gradient formulation due to the large number of degrees of freedom (DoF) of fluid dynamics. In this work, we propose a novel differentiable SPH-based two-way fluid-rigid coupling simulator to address these challenges. Our purpose is to provide a differentiable simulator for SPH which incorporates a unified representation for both fluids and solids using particles. However, naively differentiating the forward simulation of the particle system encounters gradient explosion issues. We investigate the instability in differentiating the SPH-based fluid-rigid coupling simulator and present a feasible gradient computation scheme to address its differentiability. In addition, we also propose an efficient method to compute the gradient of fluid-rigid coupling without incurring the high computational cost of differentiating the entire high-DoF fluid system. We show the efficacy, scalability, and extensibility of our method in various challenging rigid body control tasks with diverse fluid-rigid interactions and multi-rigid contacts, achieving up to an order of magnitude speedup in optimization compared to baseline methods in experiments.}
}
@inproceedings{rao2020rl,
title={Rl-cyclegan: Reinforcement learning aware simulation-to-real},
author={Rao, Kanishka and Harris, Chris and Irpan, Alex and Levine, Sergey and Ibarz, Julian and Khansari, Mohi},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={11157--11166},
year={2020},
category={rigidbody,embodied_ai},
tag={reinforce_learning},
link={https://arxiv.org/abs/2006.09001},
abstract={Deep neural network based reinforcement learning (RL) can learn appropriate visual representations for complex tasks like vision-based robotic grasping without the need for manually engineering or prior learning a perception system. However, data for RL is collected via running an agent in the desired environment, and for applications like robotics, running a robot in the real world may be extremely costly and time consuming. Simulated training offers an appealing alternative, but ensuring that policies trained in simulation can transfer effectively into the real world requires additional machinery. Simulations may not match reality, and typically bridging the simulation-to-reality gap requires domain knowledge and task-specific engineering. We can automate this process by employing generative models to translate simulated images into realistic ones. However, this sort of translation is typically task-agnostic, in that the translated images may not preserve all features that are relevant to the task. In this paper, we introduce the RL-scene consistency loss for image translation, which ensures that the translation operation is invariant with respect to the Q-values associated with the image. This allows us to learn a task-aware translation. Incorporating this loss into unsupervised domain translation, we obtain the RL-CycleGAN, a new approach for simulation-to-real-world transfer for reinforcement learning. In evaluations of RL-CycleGAN on two vision-based robotics grasping tasks, we show that RL-CycleGAN offers a substantial improvement over a number of prior methods for sim-to-real transfer, attaining excellent real-world performance with only a modest number of real-world observations.}
}
@inproceedings{runia2020cloth,
title={Cloth in the wind: A case study of physical measurement through simulation},
author={Runia, Tom FH and Gavrilyuk, Kirill and Snoek, Cees GM and Smeulders, Arnold WM},
booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
pages={10498--10507},
year={2020},
category={cloth, reconstruction},
tag={real2sim, parameter_estimation},
link={https://arxiv.org/abs/2003.05065},
abstract={For many of the physical phenomena around us, we have developed sophisticated models explaining their behavior. Nevertheless, measuring physical properties from visual observations is challenging due to the high number of causally underlying physical parameters -- including material properties and external forces. In this paper, we propose to measure latent physical properties for cloth in the wind without ever having seen a real example before. Our solution is an iterative refinement procedure with simulation at its core. The algorithm gradually updates the physical model parameters by running a simulation of the observed phenomenon and comparing the current simulation to a real-world observation. The correspondence is measured using an embedding function that maps physically similar examples to nearby points. We consider a case study of cloth in the wind, with curling flags as our leading example - a seemingly simple phenomena but physically highly involved. Based on the physics of cloth and its visual manifestation, we propose an instantiation of the embedding function. For this mapping, modeled as a deep network, we introduce a spectral layer that decomposes a video volume into its temporal spectral power and corresponding frequencies. Our experiments demonstrate that the proposed method compares favorably to prior work on the task of measuring cloth material properties and external wind force from a real-world video.}
}
@inproceedings{ehsani2020use,
title={Use the force, luke! learning to predict physical forces by simulating effects},
author={Ehsani, Kiana and Tulsiani, Shubham and Gupta, Saurabh and Farhadi, Ali and Gupta, Abhinav},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={224--233},
year={2020},
category={rigidbody, reconstruction},
tag={real2sim},
link={https://ehsanik.github.io/forcecvpr2020/},
abstract={When we humans look at a video of human-object interaction, we can not only infer what is happening but we can even extract actionable information and imitate those interactions. On the other hand, current recognition or geometric approaches lack the physicality of action representation. In this paper, we take a step towards more physical understanding of actions. We address the problem of inferring contact points and the physical forces from videos of humans interacting with objects. One of the main challenges in tackling this problem is obtaining ground-truth labels for forces. We sidestep this problem by instead using a physics simulator for supervision. Specifically, we use a simulator to predict effects, and enforce that estimated forces must lead to same effect as depicted in the video. Our quantitative and qualitative results show that (a) we can predict meaningful forces from videos whose effects lead to accurate imitation of the motions observed, (b) by jointly optimizing for contact point and force prediction, we can improve the performance on both tasks in comparison to independent training, and (c) we can learn a representation from this model that generalizes to novel objects using few shot examples.}
}
@inproceedings{thapa2020dynamic,
title={Dynamic fluid surface reconstruction using deep neural network},
author={Thapa, Simron and Li, Nianyi and Ye, Jinwei},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={21--30},
year={2020},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction},
link={https://ivlab.cse.lsu.edu/FSRN_CVPR20.html},
abstract={Recovering the dynamic fluid surface is a long-standing challenging problem in computer vision. Most existing image-based methods require multiple views or a dedicated imaging system. Here we present a learning-based single-image approach for 3D fluid surface reconstruction. Specifically, we design a deep neural network that estimates the depth and normal maps of a fluid surface by analyzing the refractive distortion of a reference background image. Due to the dynamic nature of fluid surfaces, our network uses recurrent layers that carry temporal information from previous frames to achieve spatio-temporally consistent reconstruction given a video input. Due to the lack of fluid data, we synthesize a large fluid dataset using physics-based fluid modeling and rendering techniques for network training and validation. Through experiments on simulated and real captured fluid images, we demonstrate that our proposed deep neural network trained on our fluid dataset can recover dynamic 3D fluid surfaces with high accuracy.}
}
@inproceedings{zang2020tomofluid,
title={Tomofluid: Reconstructing dynamic fluid from sparse view videos},
author={Zang, Guangming and Idoughi, Ramzi and Wang, Congli and Bennett, Anthony and Du, Jianguo and Skeen, Scott and Roberts, William L and Wonka, Peter and Heidrich, Wolfgang},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={1870--1879},
year={2020},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction},
link={https://openaccess.thecvf.com/content_CVPR_2020/html/Zang_TomoFluid_Reconstructing_Dynamic_Fluid_From_Sparse_View_Videos_CVPR_2020_paper.html},
abstract={Visible light tomography is a promising and increasingly popular technique for fluid imaging. However, the use of a sparse number of viewpoints in the capturing setups makes the reconstruction of fluid flows very challenging. In this paper, we present a state-of-the-art 4D tomographic reconstruction framework that integrates several regularizers into a multi-scale matrix free optimization algorithm. In addition to existing regularizers, we propose two new regularizers for improved results: a regularizer based on view interpolation of projected images and a regularizer to encourage reprojection consistency. We demonstrate our method with extensive experiments on both simulated and real data.}
}
@inproceedings{rasheed2020learning,
title={Learning to measure the static friction coefficient in cloth contact},
author={Rasheed, Abdullah Haroon and Romero, Victor and Bertails-Descoubes, Florence and Wuhrer, Stefanie and Franco, Jean-S{\'e}bastien and Lazarus, Arnaud},
booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
pages={9912--9921},
year={2020},
category={cloth, reconstruction, contact},
tag={real2sim, parameter_estimation},
link={https://openaccess.thecvf.com/content_CVPR_2020/html/Rasheed_Learning_to_Measure_the_Static_Friction_Coefficient_in_Cloth_Contact_CVPR_2020_paper.html},
abstract={Measuring friction coefficients between cloth and an external body is a longstanding issue in mechanical engineering, never yet addressed with a pure vision-based system. The latter offers the prospect of simpler, less invasive friction measurement protocols compared to traditional ones, and can vastly benefit from recent deep learning advances. Such a novel measurement strategy however proves challenging, as no large labelled dataset for cloth contact exists, and creating one would require thousands of physics workbench measurements with broad coverage of cloth-material pairs. Using synthetic data instead is only possible assuming the availability of a soft-body mechanical simulator with true-to-life friction physics accuracy, yet to be verified. We propose a first vision-based measurement network for friction between cloth and a substrate, using a simple and repeatable video acquisition protocol. We train our network on purely synthetic data generated by a state-of-the-art frictional contact simulator, which we carefully calibrate and validate against real experiments under controlled conditions. We show promising results on a large set of contact pairs between real cloth samples and various kinds of substrates, with 93.6% of all measurements predicted within 0.1 range of standard physics bench measurements.}
}
@inproceedings{labbe2021single,
title={Single-view robot pose and joint angle estimation via render \& compare},
author={Labb{\'e}, Yann and Carpentier, Justin and Aubry, Mathieu and Sivic, Josef},
booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
pages={1654--1663},
year={2021},
category={rigidbody,reconstruction,embodied_ai},
tag={real2sim},
link={https://www.di.ens.fr/willow/research/robopose/},
abstract={We introduce RoboPose, a method to estimate the joint angles and the 6D camera-to-robot pose of a known articulated robot from a single RGB image. This is an important problem to grant mobile and itinerant autonomous systems the ability to interact with other robots using only visual information in non-instrumented environments, especially in the context of collaborative robotics. It is also challenging because robots have many degrees of freedom and an infinite space of possible configurations that often result in self-occlusions and depth ambiguities when imaged by a single camera. The contributions of this work are three-fold. First, we introduce a new render & compare approach for estimating the 6D pose and joint angles of an articulated robot that can be trained from synthetic data, generalizes to new unseen robot configurations at test time, and can be applied to a variety of robots. Second, we experimentally demonstrate the importance of the robot parametrization for the iterative pose updates and design a parametrization strategy that is independent of the robot structure. Finally, we show experimental results on existing benchmark datasets for four different robots and demonstrate that our method significantly outperforms the state of the art. Code and pre-trained models are available on the project webpage [1].}
}
@article{yan2020interactive,
title={Interactive liquid splash modeling by user sketches},
author={Yan, Guowei and Chen, Zhili and Yang, Jimei and Wang, Huamin},
journal={ACM Transactions on Graphics (TOG)},
volume={39},
number={6},
pages={1--13},
year={2020},
category={fluid},
tag={user_interaction,gan},
publisher={ACM New York, NY, USA},
link={https://dl.acm.org/doi/abs/10.1145/3414685.3417832},
abstract={Splashing is one of the most fascinating liquid phenomena in the real world and it is favored by artists to create stunning visual effects, both statically and dynamically. Unfortunately, the generation of complex and specialized liquid splashes is a challenging task and often requires considerable time and effort. In this paper, we present a novel system that synthesizes realistic liquid splashes from simple user sketch input. Our system adopts a conditional generative adversarial network (cGAN) trained with physics-based simulation data to produce raw liquid splash models from input sketches, and then applies model refinement processes to further improve their small-scale details. The system considers not only the trajectory of every user stroke, but also its speed, which makes the splash model simulation-ready with its underlying 3D flow. Compared with simulation-based modeling techniques through trials and errors, our system offers flexibility, convenience and intuition in liquid splash design and editing. We evaluate the usability and the efficiency of our system in an immersive virtual reality environment. Thanks to this system, an amateur user can now generate a variety of realistic liquid splashes in just a few minutes.}
}
@article{vrhandnet,
title={Vr-handnet: A visually and physically plausible hand manipulation system in virtual reality},
author={Han, DongHeun and Lee, RoUn and Kim, KyeongMin and Kang, HyeongYeop},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2023},
category={rigidbody,embodied_ai},
tag={user_interaction,reinforce_learning},
publisher={IEEE},
link={https://ieeexplore.ieee.org/abstract/document/10066837},
abstract={This study aims to allow users to perform dexterous hand manipulation of objects in virtual environments with hand-held VR controllers. To this end, the VR controller is mapped to the virtual hand and the hand motions are dynamically synthesized when the virtual hand approaches an object. At each frame, given the information about the virtual hand, VR controller input, and hand-object spatial relations, the deep neural network determines the desired joint orientations of the virtual hand model in the next frame. The desired orientations are then converted into a set of torques acting on hand joints and applied to a physics simulation to determine the hand pose at the next frame. The deep neural network, named VR-HandNet, is trained with a reinforcement learning-based approach. Therefore, it can produce physically plausible hand motion since the trial-and-error training process can learn how the interaction between hand and object is performed under the environment that is simulated by a physics engine. Furthermore, we adopted an imitation learning paradigm to increase visual plausibility by mimicking the reference motion datasets. Through the ablation studies, we validated the proposed method is effectively constructed and successfully serves our design goal. A live demo is demonstrated in the supplementary video.}
}
@article{lyu2020real,
title={Real-time hair simulation with neural interpolation},
author={Lyu, Qing and Chai, Menglei and Chen, Xiang and Zhou, Kun},
journal={IEEE Transactions on Visualization and Computer Graphics},
volume={28},
number={4},
pages={1894--1905},
year={2020},
category={softbody,neural_solver},
tag={cnn,hair},
publisher={IEEE},
link={https://www.mlchai.com/publication/lyu2020real/},
abstract={Traditionally, reduced hair simulation methods are either restricted to heuristic approximations or bound to specific hairstyles. We introduce the first CNN-integrated framework for simulating various hairstyles. The approach produces visually realistic hairs with an interactive speed. To address the technical challenges, our hair simulation pipeline is designed as a two-stage process. First, we present a fully-convolutional neural interpolator as the backbone generator to compute dynamic weights for guide hair interpolation. Then, we adopt a second generator to produce fine-scale displacements to enhance the hair details. We train the neural interpolator with a dedicated loss function and the displacement generator with an adversarial discriminator. Experimental results demonstrate that our method is effective, efficient, and superior to the state-of-the-art on a wide variety of hairstyles. We further propose a performance-driven digital avatar system and an interactive hairstyle editing tool to illustrate the practical applications.}
}
@article{ma2018fluid,
author = {Ma, Pingchuan and Tian, Yunsheng and Pan, Zherong and Ren, Bo and Manocha, Dinesh},
title = {Fluid directed rigid body control using deep reinforcement learning},
year = {2018},
category={fluid,control},
tag={fluid_control,reinforce_learning},
issue_date = {August 2018},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {37},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3197517.3201334},
doi = {10.1145/3197517.3201334},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {96},
numpages = {11},
keywords = {fluid/rigid coupling, optimal control, reinforcement learning},
link={https://gamma.cs.unc.edu/DRL_FluidRigid/},
abstract={We present a learning-based method to control a coupled 2D system involving both fluid and rigid bodies. Our approach is used to modify the fluid/rigid simulator's behavior by applying control forces only at the simulation domain boundaries. The rest of the domain, corresponding to the interior, is governed by the Navier-Stokes equation for fluids and Newton-Euler's equation for the rigid bodies. We represent our controller using a general neural-net, which is trained using deep reinforcement learning. Our formulation decomposes a control task into two stages: a precomputation training stage and an online generation stage. We utilize various fluid properties, e.g., the liquid's velocity field or the smoke's density field, to enhance the controller's performance. We set up our evaluation benchmark by letting controller drive fluid jets move on the domain boundary and allowing them to shoot fluids towards a rigid body to accomplish a set of challenging 2D tasks such as keeping a rigid body balanced, playing a two-player ping-pong game, and driving a rigid body to sequentially hit specified points on the wall. In practice, our approach can generate physically plausible animations.}
}
@article{aurand2022efficient,
author = {Aurand, Joshua and Ortiz, Raphael and Nauer, Silvia and Azevedo, Vinicius C.},
title = {Efficient Neural Style Transfer for Volumetric Simulations},
year = {2022},
category={fluid},
tag={style_transfer},
issue_date = {December 2022},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {41},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3550454.3555517},
doi = {10.1145/3550454.3555517},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {257},
numpages = {10},
keywords = {style transfer, smoke simulation, machine learning},
link={https://studios.disneyresearch.com/2022/11/30/efficient-neural-style-transfer-for-volumetric-simulations/},
abstract={Artistically controlling fluids has always been a challenging task. Recently, volumetric Neural Style Transfer (NST) techniques have been used to artistically manipulate smoke simulation data with 2D images. In this work, we revisit previous volumetric NST techniques for smoke, proposing a suite of upgrades that enable stylizations that are significantly faster, simpler, more controllable and less prone to artifacts. Moreover, the energy minimization solved by previous methods is camera dependent. To avoid that, a computationally expensive iterative optimization performed for multiple views sampled around the original simulation is needed, which can take up to several minutes per frame. We propose a simple feed-forward neural network architecture that is able to infer view-independent stylizations that are three orders of the magnitude faster than its optimization-based counterpart.}
}
@article{kim2020lagrangian,
author = {Kim, Byungsoo and Azevedo, Vinicius C. and Gross, Markus and Solenthaler, Barbara},
title = {Lagrangian neural style transfer for fluids},
year = {2020},
category={fluid},
tag={style_transfer},
issue_date = {August 2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {39},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3386569.3392473},
doi = {10.1145/3386569.3392473},
journal = {ACM Trans. Graph.},
month = {aug},
articleno = {52},
numpages = {10},
keywords = {physically-based animation, neural style transfer, fluid simulation, deep learning},
link={https://github.com/byungsook/neural-flow-style},
abstract={Artistically controlling the shape, motion and appearance of fluid simulations pose major challenges in visual effects production. In this paper, we present a neural style transfer approach from images to 3D fluids formulated in a Lagrangian viewpoint. Using particles for style transfer has unique benefits compared to grid-based techniques. Attributes are stored on the particles and hence are trivially transported by the particle motion. This intrinsically ensures temporal consistency of the optimized stylized structure and notably improves the resulting quality. Simultaneously, the expensive, recursive alignment of stylization velocity fields of grid approaches is unnecessary, reducing the computation time to less than an hour and rendering neural flow stylization practical in production settings. Moreover, the Lagrangian representation improves artistic control as it allows for multi-fluid stylization and consistent color transfer from images, and the generality of the method enables stylization of smoke and liquids likewise.}
}
@article{kim2019transport,
author = {Kim, Byungsoo and Azevedo, Vinicius C. and Gross, Markus and Solenthaler, Barbara},
title = {Transport-based neural style transfer for smoke simulations},
year = {2019},
category={fluid},
tag={style_transfer},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356560},
doi = {10.1145/3355089.3356560},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {188},
numpages = {11},
keywords = {physically-based animation, neural style transfer, fluid simulation},
link={https://dl.acm.org/doi/10.1145/3355089.3356560},
abstract={Artistically controlling fluids has always been a challenging task. Optimization techniques rely on approximating simulation states towards target velocity or density field configurations, which are often handcrafted by artists to indirectly control smoke dynamics. Patch synthesis techniques transfer image textures or simulation features to a target flow field. However, these are either limited to adding structural patterns or augmenting coarse flows with turbulent structures, and hence cannot capture the full spectrum of different styles and semantically complex structures. In this paper, we propose the first Transport-based Neural Style Transfer (TNST) algorithm for volumetric smoke data. Our method is able to transfer features from natural images to smoke simulations, enabling general content-aware manipulations ranging from simple patterns to intricate motifs. The proposed algorithm is physically inspired, since it computes the density transport from a source input smoke to a desired target configuration. Our transport-based approach allows direct control over the divergence of the stylization velocity field by optimizing incompressible and irrotational potentials that transport smoke towards stylization. Temporal consistency is ensured by transporting and aligning subsequent stylized velocities, and 3D reconstructions are computed by seamlessly merging stylizations from different camera viewpoints.}
}
@article{eckert2019scalar,
author = {Eckert, Marie-Lena and Um, Kiwon and Thuerey, Nils},
title = {ScalarFlow: a large-scale volumetric data set of real-world scalar transport flows for computer animation and machine learning},
year = {2019},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356545},
doi = {10.1145/3355089.3356545},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {239},
numpages = {16},
keywords = {user studies, optimization, machine learning, fluid reconstruction, data set},
link={https://ge.in.tum.de/publications/2019-scalarflow-eckert/},
abstract={In this paper, we present ScalarFlow , a first large-scale data set of reconstructions of real-world smoke plumes. In addition, we propose a framework for accurate physics-based reconstructions from a small number of video streams. Central components of our framework are a novel estimation of unseen inflow regions and an efficient optimization scheme constrained by a simulation to capture real-world fluids. Our data set includes a large number of complex natural buoyancy-driven flows. The flows transition to turbulence and contain observable scalar transport processes. As such, the ScalarFlow data set is tailored towards computer graphics, vision, and learning applications. The published data set contains volumetric reconstructions of velocity and density as well as the corresponding input image sequences with calibration data, code, and instructions how to reproduce the commodity hardware capture setup. We further demonstrate one of the many potential applications: a first perceptual evaluation study, which reveals that the complexity of the reconstructed flows would require large simulation resolutions for regular solvers in order to recreate at least parts of the natural complexity contained in the captured data.}
}
@article{guo2021volumetric,
author = {Guo, Jie and Li, Mengtian and Zong, Zijing and Liu, Yuntao and He, Jingwu and Guo, Yanwen and Yan, Ling-Qi},
title = {Volumetric appearance stylization with stylizing kernel prediction network},
year = {2021},
category={fluid},
tag={style_transfer},
issue_date = {August 2021},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {40},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3450626.3459799},
doi = {10.1145/3450626.3459799},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {162},
numpages = {15},
keywords = {volume rendering, temporal coherence, stylization, single-scattering albedo, deep learning},
link={https://dl.acm.org/doi/abs/10.1145/3450626.3459799},
abstract={This paper aims to efficiently construct the volume of heterogeneous single-scattering albedo for a given medium that would lead to desired color appearance. We achieve this goal by formulating it as a volumetric style transfer problem in which an input 3D density volume is stylized using color features extracted from a reference 2D image. Unlike existing algorithms that require cumbersome iterative optimizations, our method leverages a feed-forward deep neural network with multiple well-designed modules. At the core of our network is a stylizing kernel predictor (SKP) that extracts multi-scale feature maps from a 2D style image and predicts a handful of stylizing kernels as a highly non-linear combination of the feature maps. Each group of stylizing kernels represents a specific style. A volume autoencoder (VolAE) is designed and jointly learned with the SKP to transform a density volume to an albedo volume based on these stylizing kernels. Since the autoencoder does not encode any style information, it can generate different albedo volumes with a wide range of appearance once training is completed. Additionally, a hybrid multi-scale loss function is used to learn plausible color features and guarantee temporal coherence for time-evolving volumes. Through comprehensive experiments, we validate the effectiveness of our method and show its superiority by comparing against state-of-the-arts. We show that with our method a novice user can easily create a diverse set of realistic translucent effects for 3D models (either static or dynamic), neglecting any cumbersome process of parameter tuning.}
}
@article{bai2020dynamic,
author = {Bai, Kai and Li, Wei and Desbrun, Mathieu and Liu, Xiaopei},
title = {Dynamic Upsampling of Smoke through Dictionary-based Learning},
year = {2020},
category={fluid},
tag={super_resolution},
issue_date = {February 2021},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {40},
number = {1},
issn = {0730-0301},
url = {https://doi.org/10.1145/3412360},
doi = {10.1145/3412360},
journal = {ACM Trans. Graph.},
month = {sep},
articleno = {4},
numpages = {19},
keywords = {smoke animation, neural networks, dictionary learning, Fluid simulation},
link={https://faculty.sist.shanghaitech.edu.cn/faculty/liuxp/projects/ss_dbnn/index.htm},
abstract={Simulating turbulent smoke flows with fine details is computationally intensive. For iterative editing or simply faster generation, efficiently upsampling a low-resolution numerical simulation is an attractive alternative. We propose a novel learning approach to the dynamic upsampling of smoke flows based on a training set of flows at coarse and fine resolutions. Our multiscale neural network turns an input coarse animation into a sparse linear combination of small velocity patches present in a precomputed over-complete dictionary. These sparse coefficients are then used to generate a high-resolution smoke animation sequence by blending the fine counterparts of the coarse patches. Our network is initially trained from a sequence of example simulations to both construct the dictionary of corresponding coarse and fine patches and allow for the fast evaluation of a sparse patch encoding of any coarse input. The resulting network provides an accurate upsampling when the coarse input simulation is well approximated by patches present in the training set (e.g., for re-simulation), or simply visually plausible upsampling when input and training sets differ significantly. We show a variety of examples to ascertain the strengths and limitations of our approach and offer comparisons to existing approaches to demonstrate its quality and effectiveness.}
}
@article{takahashi2019video,
author = {Takahashi, Tetsuya and Lin, Ming C.},
title = {Video-guided real-to-virtual parameter transfer for viscous fluids},
year = {2019},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction,material_estimation},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356551},
doi = {10.1145/3355089.3356551},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {237},
numpages = {12},
keywords = {data-driven simulation, fluid simulation, optimization, viscosity},
link={https://gamma.cs.unc.edu/ParameterTransfer/},
abstract={In physically-based simulation, it is essential to choose appropriate material parameters to generate desirable simulation results. In many cases, however, choosing appropriate material parameters is very challenging, and often tedious trial-and-error parameter tuning steps are inevitable. In this paper, we propose a real-to-virtual parameter transfer framework that identifies material parameters of viscous fluids with example video data captured from real-world phenomena. Our method first extracts positional data of fluids and then uses the extracted data as a reference to identify the viscosity parameters, combining forward viscous fluid simulations and parameter optimization in an iterative process. We evaluate our method with a range of synthetic and real-world example data, and demonstrate that our method can identify the hidden physical variables and viscosity parameters. This set of recovered physical variables and parameters can then be effectively used in novel scenarios to generate viscous fluid behaviors visually consistent with the example videos.}
}
@article{hahn2019real,
author = {Hahn, David and Banzet, Pol and Bern, James M. and Coros, Stelian},
title = {Real2Sim: visco-elastic parameter estimation from dynamic motion},
year = {2019},
category={softbody,differentiable_simulation,reconstruction},
tag={real2sim,material_estimation},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356548},
doi = {10.1145/3355089.3356548},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {236},
numpages = {13},
keywords = {robotics, physically-based simulation, optimization, motion capture, deformable models},
link={https://dl.acm.org/doi/10.1145/3355089.3356548},
abstract={This paper presents a method for optimizing visco-elastic material parameters of a finite element simulation to best approximate the dynamic motion of real-world soft objects. We compute the gradient with respect to the material parameters of a least-squares error objective function using either direct sensitivity analysis or an adjoint state method. We then optimize the material parameters such that the simulated motion matches real-world observations as closely as possible. In this way, we can directly build a useful simulation model that captures the visco-elastic behaviour of the specimen of interest. We demonstrate the effectiveness of our method on various examples such as numerical coarsening, custom-designed objective functions, and of course real-world flexible elastic objects made of foam or 3D printed lattice structures, including a demo application in soft robotics.}
}
@article{wang2019learning,
author = {Wang, Tuanfeng Y. and Shao, Tianjia and Fu, Kai and Mitra, Niloy J.},
title = {Learning an intrinsic garment space for interactive authoring of garment animation},
year = {2019},
category={cloth,differentiable_simulation},
tag={user_interaction},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356512},
doi = {10.1145/3355089.3356512},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {220},
numpages = {12},
keywords = {interactive garment animation authoring, intrinsic garment space, latent representation},
link={https://geometry.cs.ucl.ac.uk/projects/2019/garment_authoring/},
abstract={Authoring dynamic garment shapes for character animation on body motion is one of the fundamental steps in the CG industry. Established workflows are either time and labor consuming (i.e., manual editing on dense frames with controllers), or lack keyframe-level control (i.e., physically-based simulation). Not surprisingly, garment authoring remains a bottleneck in many production pipelines. Instead, we present a deep-learning-based approach for semi-automatic authoring of garment animation, wherein the user provides the desired garment shape in a selection of keyframes, while our system infers a latent representation for its motion-independent intrinsic parameters (e.g., gravity, cloth materials, etc.). Given new character motions, the latent representation allows to automatically generate a plausible garment animation at interactive rates. Having factored out character motion, the learned intrinsic garment space enables smooth transition between keyframes on a new motion sequence. Technically, we learn an intrinsic garment space with an motion-driven autoencoder network, where the encoder maps the garment shapes to the intrinsic space under the condition of body motions, while the decoder acts as a differentiable simulator to generate garment shapes according to changes in character body motion and intrinsic parameters. We evaluate our approach qualitatively and quantitatively on common garment types. Experiments demonstrate our system can significantly improve current garment authoring workflows via an interactive user interface. Compared with the standard CG pipeline, our system significantly reduces the ratio of required keyframes from 20% to 1 -- 2%.}
}
@article{min2019softcon,
author = {Min, Sehee and Won, Jungdam and Lee, Seunghwan and Park, Jungnam and Lee, Jehee},
title = {SoftCon: simulation and control of soft-bodied animals with biomimetic actuators},
year = {2019},
category={softbody,control},
tag={reinforce_learning},
issue_date = {December 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3355089.3356497},
doi = {10.1145/3355089.3356497},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {208},
numpages = {12},
keywords = {character animation, deformable character, finite element method, optimal control, physics-based control, reinforcement learning, soft-bodied animal},
link={https://mrl.snu.ac.kr/publications/ProjectSoftCon/SoftCon.html?utm_campaign=Artificial%2BIntelligence%2BWeekly&utm_medium=rss&utm_source=Artificial_Intelligence_Weekly_153},
abstract={We present a novel and general framework for the design and control of underwater soft-bodied animals. The whole body of an animal consisting of soft tissues is modeled by tetrahedral and triangular FEM meshes. The contraction of muscles embedded in the soft tissues actuates the body and limbs to move. We present a novel muscle excitation model that mimics the anatomy of muscular hydrostats and their muscle excitation patterns. Our deep reinforcement learning algorithm equipped with the muscle excitation model successfully learned the control policy of soft-bodied animals, which can be physically simulated in real-time, controlled interactively, and resilient to external perturbations. We demonstrate the effectiveness of our approach with various simulated animals including octopuses, lampreys, starfishes, stingrays and cuttlefishes. They learn diverse behaviors such as swimming, grasping, and escaping from a bottle. We also implemented a simple user interface system that allows the user to easily create their creatures.}
}
@article{geilinger2020add,
author = {Geilinger, Moritz and Hahn, David and Zehnder, Jonas and B\"{a}cher, Moritz and Thomaszewski, Bernhard and Coros, Stelian},
title = {ADD: analytically differentiable dynamics for multi-body systems with frictional contact},
year = {2020},
category={softbody,differentiable_simulation,control,embodied_ai,contact},
issue_date = {December 2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {39},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3414685.3417766},
doi = {10.1145/3414685.3417766},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {190},
numpages = {15},
keywords = {robotics, rigid-body dynamics, optimization, differentiable simulation, deformable models, contact mechanics},
link={https://arxiv.org/abs/2007.00987},
abstract={We present a differentiable dynamics solver that is able to handle frictional contact for rigid and deformable objects within a unified framework. Through a principled mollification of normal and tangential contact forces, our method circumvents the main difficulties inherent to the non-smooth nature of frictional contact. We combine this new contact model with fully-implicit time integration to obtain a robust and efficient dynamics solver that is analytically differentiable. In conjunction with adjoint sensitivity analysis, our formulation enables gradient-based optimization with adaptive trade-offs between simulation accuracy and smoothness of objective function landscapes. We thoroughly analyse our approach on a set of simulation examples involving rigid bodies, visco-elastic materials, and coupled multi-body systems. We furthermore showcase applications of our differentiable simulator to parameter estimation for deformable objects, motion planning for robotic manipulation, trajectory optimization for compliant walking robots, as well as efficient self-supervised learning of control policies.}
}
@article{zhang2020learning,
author = {Zhang, Yunbo and Yu, Wenhao and Liu, C. Karen and Kemp, Charlie and Turk, Greg},
title = {Learning to manipulate amorphous materials},
year = {2020},
category={fluid,control,embodied_ai},
tag={fluid_control,reinforce_learning},
issue_date = {December 2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {39},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3414685.3417868},
doi = {10.1145/3414685.3417868},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {189},
numpages = {11},
keywords = {reinforcement learning, physics simulation, deformable materials, character control},
link={https://tml.stanford.edu/publications/2020/learning-manipulate-amorphous-materials},
abstract={We present a method of training character manipulation of amorphous materials such as those often used in cooking. Common examples of amorphous materials include granular materials (salt, uncooked rice), fluids (honey), and visco-plastic materials (sticky rice, softened butter). A typical task is to spread a given material out across a flat surface using a tool such as a scraper or knife. We use reinforcement learning to train our controllers to manipulate materials in various ways. The training is performed in a physics simulator that uses position-based dynamics of particles to simulate the materials to be manipulated. The neural network control policy is given observations of the material (e.g. a low-resolution density map), and the policy outputs actions such as rotating and translating the knife. We demonstrate policies that have been successfully trained to carry out the following tasks: spreading, gathering, and flipping. We produce a final animation by using inverse kinematics to guide a character's arm and hand to match the motion of the manipulation tool such as a knife or a frying pan.}
}
@article{ren2022versatile,
author = {Ren, Bo and Ye, Xiaohan and Pan, Zherong and Zhang, Taiyuan},
title = {Versatile Control of Fluid-directed Solid Objects Using Multi-task Reinforcement Learning},
year = {2022},
category={fluid,control},
tag={fluid_control,reinforce_learning},
issue_date = {April 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {2},
issn = {0730-0301},
url = {https://doi.org/10.1145/3554731},
doi = {10.1145/3554731},
journal = {ACM Trans. Graph.},
month = {oct},
articleno = {15},
numpages = {14},
keywords = {Fluid/solid coupling, optimal control, reinforcement learning},
link={https://dl.acm.org/doi/10.1145/3554731},
abstract={We propose a learning-based controller for high-dimensional dynamic systems with coupled fluid and solid objects. The dynamic behaviors of such systems can vary across different simulators and the control tasks subject to changing requirements from users. Our controller features high versatility and can adapt to changing dynamic behaviors and multiple tasks without re-training, which is achieved by combining two training strategies. We use meta-reinforcement learning to inform the controller of changing simulation parameters. We further design a novel task representation, which allows the controller to adapt to continually changing tasks via hindsight experience replay. We highlight the robustness and generality of our controller on a row of dynamic-rich tasks, including scooping up solid balls from a water pool, in-air ball acrobatics using fluid spouts, and zero-shot transferring to unseen simulators and constitutive models. In all the scenarios, our controller consistently outperforms the plain multi-task reinforcement-learning baseline.}
}
@article{numerow2024voronoi,
title={Differentiable Voronoi Diagrams for Simulation of Cell-Based Mechanical Systems},
author={Numerow, Logan and Li, Yue and Coros, Stelian and Thomaszewski, Bernhard},
journal={ACM Transactions on Graphics (TOG)},
volume={43},
number={4},
year={2024},
category={softbody,differentiable_simulation},
publisher={ACM New York, NY, USA},
link={https://github.com/lnumerow-ethz/VoronoiCellSim},
abstract={Navigating topological transitions in cellular mechanical systems is a significant challenge for existing simulation methods. While abstract models lack predictive capabilities at the cellular level, explicit network representations struggle with topology changes, and per-cell representations are computationally too demanding for large-scale simulations. To address these challenges, we propose a novel cell-centered approach based on differentiable Voronoi diagrams. Representing each cell with a Voronoi site, our method defines shape and topology of the interface network implicitly. In this way, we substantially reduce the number of problem variables, eliminate the need for explicit contact handling, and ensure continuous geometry changes during topological transitions. Closed-form derivatives of network positions facilitate simulation with Newton-type methods for a wide range of per-cell energies. Finally, we extend our differentiable Voronoi diagrams to enable coupling with arbitrary rigid and deformable boundaries. We apply our approach to a diverse set of examples, highlighting splitting and merging of cells as well as neighborhood changes. We illustrate applications to inverse problems by matching soap foam simulations to real-world images. Comparative analysis with explicit cell models reveals that our method achieves qualitatively comparable results at significantly faster computation times.}
}
@inproceedings{chang2023licrom,
author = {Chang, Yue and Chen, Peter Yichen and Wang, Zhecheng and Chiaramonte, Maurizio M. and Carlberg, Kevin and Grinspun, Eitan},
title = {LiCROM: Linear-Subspace Continuous Reduced Order Modeling with Neural Fields},
year = {2023},
category={softbody,neural_representation},
tag={neural_field},
isbn = {9798400703157},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3610548.3618158},
doi = {10.1145/3610548.3618158},
booktitle = {SIGGRAPH Asia 2023 Conference Papers},
articleno = {111},
numpages = {12},
keywords = {Implicit neural representation, Neural Field, Physical simulation, Reduced-order modeling},
location = {, Sydney, NSW, Australia, },
series = {SA '23},
link={https://arxiv.org/abs/2310.15907},
abstract={Linear reduced-order modeling (ROM) simplifies complex simulations by approximating the behavior of a system using a simplified kinematic representation. Typically, ROM is trained on input simulations created with a specific spatial discretization, and then serves to accelerate simulations with the same discretization. This discretization-dependence is restrictive.}
}
@article{li2023neural,
author = {Li, Yue and Coros, Stelian and Thomaszewski, Bernhard},
title = {Neural Metamaterial Networks for Nonlinear Material Design},
year = {2023},
category={softbody, differentiable_simulation, neural_representation, optimization},
issue_date = {December 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3618325},
doi = {10.1145/3618325},
journal = {ACM Trans. Graph.},
month = {dec},
articleno = {186},
numpages = {13},
keywords = {FEM, homogenization, inverse design, isohedral tilings, meta materials, neural networks},
link={https://github.com/liyuesolo/NeuralMetamaterialNetwork},
abstract={Nonlinear metamaterials with tailored mechanical properties have applications in engineering, medicine, robotics, and beyond. While modeling their macromechanical behavior is challenging in itself, finding structure parameters that lead to ideal approximation of high-level performance goals is a challenging task. In this work, we propose Neural Metamaterial Networks (NMN)---smooth neural representations that encode the nonlinear mechanics of entire metamaterial families. Given structure parameters as input, NMN return continuously differentiable strain energy density functions, thus guaranteeing conservative forces by construction. Though trained on simulation data, NMN do not inherit the discontinuities resulting from topo-logical changes in finite element meshes. They instead provide a smooth map from parameter to performance space that is fully differentiable and thus well-suited for gradient-based optimization. On this basis, we formulate inverse material design as a nonlinear programming problem that leverages neural networks for both objective functions and constraints. We use this approach to automatically design materials with desired strain-stress curves, prescribed directional stiffness and Poisson ratio profiles. We furthermore conduct ablation studies on network nonlinearities and show the advantages of our approach compared to native-scale optimization.}
}
@article{grandia2023doc,
author = {Grandia, Ruben and Farshidian, Farbod and Knoop, Espen and Schumacher, Christian and Hutter, Marco and B\"{a}cher, Moritz},
title = {DOC: Differentiable Optimal Control for Retargeting Motions onto Legged Robots},
year = {2023},
category={rigidbody,control,embodied_ai},
tag={robot_control},
issue_date = {August 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3592454},
doi = {10.1145/3592454},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {96},
numpages = {14},
keywords = {differentiable optimal control, differential dynamic programming, motion retargeting, model-predictive control},
link={https://la.disneyresearch.com/publication/doc-differentiable-optimal-control-for-retargeting-motions-onto-legged-robots/},
abstract={Legged robots are designed to perform highly dynamic motions. However, it remains challenging for users to retarget expressive motions onto these complex systems. In this paper, we present a Differentiable Optimal Control (DOC) framework that facilitates the transfer of rich motions from either animals or animations onto these robots. Interfacing with either motion capture or animation data, we formulate retargeting objectives whose parameters make them agnostic to differences in proportions and numbers of degrees of freedom between input and robot. Optimizing these parameters over the manifold spanned by optimal state and control trajectories, we minimize the retargeting error. We demonstrate the utility and efficacy of our modeling by applying DOC to a Model-Predictive Control (MPC) formulation, showing retargeting results for a family of robots of varying proportions and mass distribution. With a hardware deployment, we further show that the retargeted motions are physically feasible, while MPC ensures that the robots retain their capability to react to unexpected disturbances.}
}
@article{zhang2021dynamic,
author = {Zhang, Meng and Wang, Tuanfeng Y. and Ceylan, Duygu and Mitra, Niloy J.},
title = {Dynamic neural garments},
year = {2021},
category={cloth, neural_solver},
issue_date = {December 2021},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {40},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/3478513.3480497},
doi = {10.1145/3478513.3480497},
journal = {ACM Trans. Graph.},
month = {dec},
articleno = {235},
numpages = {15},
keywords = {neural simulation, neural rendering, dynamic garments, avatars, animation},
link={https://geometry.cs.ucl.ac.uk/projects/2021/DynamicNeuralGarments/},
abstract={A vital task of the wider digital human effort is the creation of realistic garments on digital avatars, both in the form of characteristic fold patterns and wrinkles in static frames as well as richness of garment dynamics under avatars' motion. Existing workflow of modeling, simulation, and rendering closely replicates the physics behind real garments, but is tedious and requires repeating most of the workflow under changes to characters' motion, camera angle, or garment resizing. Although data-driven solutions exist, they either focus on static scenarios or only handle dynamics of tight garments. We present a solution that, at test time, takes in body joint motion to directly produce realistic dynamic garment image sequences. Specifically, given the target joint motion sequence of an avatar, we propose dynamic neural garments to synthesize plausible dynamic garment appearance from a desired viewpoint. Technically, our solution generates a coarse garment proxy sequence, learns deep dynamic features attached to this template, and neurally renders the features to produce appearance changes such as folds, wrinkles, and silhouettes. We demonstrate generalization behavior to both unseen motion and unseen camera views. Further, our network can be fine-tuned to adopt to new body shape and/or background images. We demonstrate our method on a wide range of real and synthetic garments. We also provide comparisons against existing neural rendering and image sequence translation approaches, and report clear quantitative and qualitative improvements. Project page: http://geometry.cs.ucl.ac.uk/projects/2021/DynamicNeuralGarments/}
}
@article{shacklett2023extensible,
author = {Shacklett, Brennan and Rosenzweig, Luc Guy and Xie, Zhiqiang and Sarkar, Bidipta and Szot, Andrew and Wijmans, Erik and Koltun, Vladlen and Batra, Dhruv and Fatahalian, Kayvon},
title = {An Extensible, Data-Oriented Architecture for High-Performance, Many-World Simulation},
year = {2023},
category={rigidbody, engine},
issue_date = {August 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3592427},
doi = {10.1145/3592427},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {90},
numpages = {13},
keywords = {game AI, reinforcement learning},
link={https://madrona-engine.github.io/},
abstract={Training AI agents to perform complex tasks in simulated worlds requires millions to billions of steps of experience. To achieve high performance, today's fastest simulators for training AI agents adopt the idea of batch simulation: using a single simulation engine to simultaneously step many environments in parallel. We introduce a framework for productively authoring novel training environments (including custom logic for environment generation, environment time stepping, and generating agent observations and rewards) that execute as high-performance, GPU-accelerated batched simulators. Our key observation is that the entity-component-system (ECS) design pattern, popular for expressing CPU-side game logic today, is also well-suited for providing the structure needed for high-performance batched simulators. We contribute the first fully-GPU accelerated ECS implementation that natively supports batch environment simulation. We demonstrate how ECS abstractions impose structure on a training environment's logic and state that allows the system to efficiently manage state, amortize work, and identify GPU-friendly coherent parallel computations within and across different environments. We implement several learning environments in this framework, and demonstrate GPU speedups of two to three orders of magnitude over open source CPU baselines and 5-33× over strong baselines running on a 32-thread CPU. An implementation of the OpenAI hide and seek 3D environment written in our framework, which performs rigid body physics and ray tracing in each simulator step, achieves over 1.9 million environment steps per second on a single GPU.}
}
@article{xu2019learning,
author = {Xu, Jie and Du, Tao and Foshey, Michael and Li, Beichen and Zhu, Bo and Schulz, Adriana and Matusik, Wojciech},
title = {Learning to fly: computational controller design for hybrid UAVs with reinforcement learning},
year = {2019},
category={rigidbody,control},
tag={reinforce_learning},
issue_date = {August 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {38},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3306346.3322940},
doi = {10.1145/3306346.3322940},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {42},
numpages = {12},
keywords = {hybrid UAVs, neural network controllers},
link={https://people.csail.mit.edu/jiex/papers/LearningToFly/index.html},
abstract={Hybrid unmanned aerial vehicles (UAV) combine advantages of multicopters and fixed-wing planes: vertical take-off, landing, and low energy use. However, hybrid UAVs are rarely used because controller design is challenging due to its complex, mixed dynamics. In this paper, we propose a method to automate this design process by training a mode-free, model-agnostic neural network controller for hybrid UAVs. We present a neural network controller design with a novel error convolution input trained by reinforcement learning. Our controller exhibits two key features: First, it does not distinguish among flying modes, and the same controller structure can be used for copters with various dynamics. Second, our controller works for real models without any additional parameter tuning process, closing the gap between virtual simulation and real fabrication. We demonstrate the efficacy of the proposed controller both in simulation and in our custom-built hybrid UAVs (Figure 1, 8). The experiments show that the controller is robust to exploit the complex dynamics when both rotors and wings are active in flight tests.}
}
@inproceedings{shao2022transformer,
title={Transformer with implicit edges for particle-based physics simulation},
author={Shao, Yidi and Loy, Chen Change and Dai, Bo},
booktitle={European Conference on Computer Vision},
pages={549--564},
year={2022},
category={fluid,neural_solver},
tag={gnn,transformer},
organization={Springer},
link={https://www.mmlab-ntu.com/project/tie/index.html},
abstract={Particle-based systems provide a flexible and unified way to simulate physics systems with complex dynamics. Most existing data-driven simulators for particle-based systems adopt graph neural networks (GNNs) as their network backbones, as particles and their interactions can be naturally represented by graph nodes and graph edges. However, while particle-based systems usually contain hundreds even thousands of particles, the explicit modeling of particle interactions as graph edges inevitably leads to a significant computational overhead, due to the increased number of particle interactions. Consequently, in this paper we propose a novel Transformer-based method, dubbed as Transformer with Implicit Edges (TIE), to capture the rich semantics of particle interactions in an edge-free manner. The core idea of TIE is to decentralize the computation involving pair-wise particle interactions into per-particle updates. This is achieved by adjusting the self-attention module to resemble the update formula of graph edges in GNN. To improve the generalization ability of TIE, we further amend TIE with learnable material-specific abstract particles to disentangle global material-wise semantics from local particle-wise semantics. We evaluate our model on diverse domains of varying complexity and materials. Compared with existing GNN-based methods, without bells and whistles, TIE achieves superior performance and generalization across all these domains. Codes and models are available at https://github.com/ftbabi/TIE_ECCV2022.git.}
}
@inproceedings{wang2024neural,
title={Neural Physical Simulation with Multi-Resolution Hash Grid Encoding},
author={Wang, Haoxiang and Yu, Tao and Yang, Tianwei and Qiao, Hui and Dai, Qionghai},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
volume={38},
number={6},
pages={5410--5418},
year={2024},
category={fluid, neural_representation, neural_solver},
link={https://ojs.aaai.org/index.php/AAAI/article/view/28349},
abstract={We explore the generalization of the implicit representation in the physical simulation task. Traditional time-dependent partial differential equations (PDEs) solvers for physical simulation often adopt the grid or mesh for spatial discretization, which is memory-consuming for high resolution and lack of adaptivity. Many implicit representations like local extreme machine or Siren are proposed but they are still too compact to suffer from limited accuracy in handling local details and a long time of convergence. We contribute a neural simulation framework based on multi-resolution hash grid representation to introduce hierarchical consideration of global and local information, simultaneously. Furthermore, we propose two key strategies: 1) a numerical gradient method for computing high-order derivatives with boundary conditions; 2) a range analysis sample method for fast neural geometry boundary sampling with dynamic topologies. Our method shows much higher accuracy and strong flexibility for various simulation problems: e.g., large elastic deformations, complex fluid dynamics, and multi-scale phenomena which remain challenging for existing neural physical solvers.}
}
@inproceedings{liu2023fast,
title={Fast fluid simulation via dynamic multi-scale gridding},
author={Liu, Jinxian and Chen, Ye and Ni, Bingbing and Ren, Wei and Yu, Zhenbo and Huang, Xiaoyang},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
volume={37},
number={2},
pages={1675--1682},
year={2023},
category={fluid, neural_solver},
link={https://ojs.aaai.org/index.php/AAAI/article/view/25255},
abstract={Recent works on learning-based frameworks for Lagrangian (i.e., particle-based) fluid simulation, though bypassing iterative pressure projection via efficient convolution operators, are still time-consuming due to excessive amount of particles. To address this challenge, we propose a dynamic multi-scale gridding method to reduce the magnitude of elements that have to be processed, by observing repeated particle motion patterns within certain consistent regions. Specifically, we hierarchically generate multi-scale micelles in Euclidean space by grouping particles that share similar motion patterns/characteristics based on super-light motion and scale estimation modules. With little internal motion variation, each micelle is modeled as a single rigid body with convolution only applied to a single representative particle. In addition, a distance-based interpolation is conducted to propagate relative motion message among micelles. With our efficient design, the network produces high visual fidelity fluid simulations with the inference time to be only 4.24 ms/frame (with 6K fluid particles), hence enables real-time human-computer interaction and animation. Experimental results on multiple datasets show that our work achieves great simulation acceleration with negligible prediction error increase.}
}
@inproceedings{pan2024learning,
title={Learning Reduced Fluid Dynamics},
author={Pan, Zherong and Gao, Xifeng and Wu, Kui},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
volume={38},
number={13},
pages={14517--14526},
year={2024},
category={fluid,differentiable_simulation,neural_solver},
link={https://ojs.aaai.org/index.php/AAAI/article/view/29367},
abstract={Predicting the state evolution of ultra high-dimensional, time-reversible fluid dynamic systems is a crucial but computationally expensive task. Existing physics-informed neural networks either incur high inference cost or cannot preserve the time-reversible nature of the underlying dynamics system. We propose a model-based approach to identify low-dimensional, time reversible, nonlinear fluid dynamic systems. Our method utilizes the symplectic structure of reduced Eulerian fluid and use stochastic Riemann optimization to obtain a low-dimensional bases that minimize the expected trajectory-wise dimension-reduction error over a given distribution of initial conditions. We show that such minimization is well-defined since the reduced trajectories are differentiable with respect to the subspace bases over the entire Grassmannian manifold, under proper choices of timestep sizes and numerical integrators. Finally, we propose a loss function measuring the trajectory-wise discrepancy between the original and reduced models. By tensor precomputation, we show that gradient information of such loss function can be evaluated efficiently over a long trajectory without time-integrating the high-dimensional dynamic system. Through evaluations on a row of simulation benchmarks, we show that our method reduces the discrepancy by 50-90 percent over conventional reduced models and we outperform PINNs by exactly preserving the time reversibility.}
}
@article{qiao2021differentiable,
title={Differentiable simulation of soft multi-body systems},
author={Qiao, Yiling and Liang, Junbang and Koltun, Vladlen and Lin, Ming},
journal={Advances in Neural Information Processing Systems},
volume={34},
pages={17123--17135},
year={2021},
category={softbody,differentiable_simulation},
link={https://github.com/YilingQiao/diff_fem},
abstract={We present a method for differentiable simulation of soft articulated bodies. Our work enables the integration of differentiable physical dynamics into gradient-based pipelines. We develop a top-down matrix assembly algorithm within Projective Dynamics and derive a generalized dry friction model for soft continuum using a new matrix splitting strategy. We derive a differentiable control framework for soft articulated bodies driven by muscles, joint torques, or pneumatic tubes. The experiments demonstrate that our designs make soft body simulation more stable and realistic compared to other frameworks. Our method accelerates the solution of system identification problems by more than an order of magnitude, and enables efficient gradient-based learning of motion control with soft robots.}
}
@article{morton2018deep,
title={Deep dynamical modeling and control of unsteady fluid flows},
author={Morton, Jeremy and Jameson, Antony and Kochenderfer, Mykel J and Witherden, Freddie},
journal={Advances in Neural Information Processing Systems},
volume={31},
year={2018},
category={fluid,neural_solver,control},
tag={fluid_control},
link={https://github.com/sisl/deep_flow_control},
abstract={The design of flow control systems remains a challenge due to the nonlinear nature of the equations that govern fluid flow. However, recent advances in computational fluid dynamics (CFD) have enabled the simulation of complex fluid flows with high accuracy, opening the possibility of using learning-based approaches to facilitate controller design. We present a method for learning the forced and unforced dynamics of airflow over a cylinder directly from CFD data. The proposed approach, grounded in Koopman theory, is shown to produce stable dynamical models that can predict the time evolution of the cylinder system over extended time horizons. Finally, by performing model predictive control with the learned dynamical models, we are able to find a straightforward, interpretable control law for suppressing vortex shedding in the wake of the cylinder.}
}
@inproceedings{qiao2020scalable,
author = {Qiao, Yi-Ling and Liang, Junbang and Koltun, Vladlen and Lin, Ming C.},
title = {Scalable Differentiable Physics for Learning and Control},
booktitle = {ICML},
year = {2020},
category={rigidbody,differentiable_simulation,control},
link = {https://github.com/YilingQiao/diffsim},
link={},
abstract={Differentiable physics is a powerful approach to learning and control problems that involve physical objects and environments. While notable progress has been made, the capabilities of differentiable physics solvers remain limited. We develop a scalable framework for differentiable physics that can support a large number of objects and their interactions. To accommodate objects with arbitrary geometry and topology, we adopt meshes as our representation and leverage the sparsity of contacts for scalable differentiable collision handling. Collisions are resolved in localized regions to minimize the number of optimization variables even when the number of simulated objects is high. We further accelerate implicit differentiation of optimization with nonlinear constraints. Experiments demonstrate that the presented framework requires up to two orders of magnitude less memory and computation in comparison to recent particle-based methods. We further validate the approach on inverse problems and control scenarios, where it outperforms derivative-free and model-free baselines by at least an order of magnitude.}
}
@inproceedings{
winchenbach2024symmetric,
title={Symmetric Basis Convolutions for Learning Lagrangian Fluid Mechanics},
author={Rene Winchenbach and Nils Thuerey},
booktitle={International Conference on Learning Representations},
year={2024},
category={fluid, neural_solver},
url={https://openreview.net/forum?id=HKgRwNhI9R},
link={https://github.com/tum-pbs/SFBC},
abstract={Learning physical simulations has been an essential and central aspect of many recent research efforts in machine learning, particularly for Navier-Stokes-based fluid mechanics. Classic numerical solvers have traditionally been computationally expensive and challenging to use in inverse problems, whereas Neural solvers aim to address both concerns through machine learning. We propose a general formulation for continuous convolutions using separable basis functions as a superset of existing methods and evaluate a large set of basis functions in the context of (a) a compressible 1D SPH simulation, (b) a weakly compressible 2D SPH simulation, and (c) an incompressible 2D SPH Simulation. We demonstrate that even and odd symmetries included in the basis functions are key aspects of stability and accuracy. Our broad evaluation shows that Fourier-based continuous convolutions outperform all other architectures regarding accuracy and generalization. Finally, using these Fourier-based networks, we show that prior inductive biases, such as window functions, are no longer necessary. An implementation of our approach, as well as complete datasets and solver implementations, is available at https://github.com/tum-pbs/SFBC.}
}
@article{holzschuh2023solving,
title={Solving Inverse Physics Problems with Score Matching},
author={Holzschuh, Benjamin and Vegetti, Simona and Thuerey, Nils},
journal={Advances in Neural Information Processing Systems},
volume={36},
year={2023},
category={fluid, neural_solver, optimization},
link={https://github.com/tum-pbs/SMDP},
abstract={We propose to solve inverse problems involving the temporal evolution of physics systems by leveraging recent advances from diffusion models. Our method moves the system's current state backward in time step by step by combining an approximate inverse physics simulator and a learned correction function. A central insight of our work is that training the learned correction with a single-step loss is equivalent to a score matching objective, while recursively predicting longer parts of the trajectory during training relates to maximum likelihood training of a corresponding probability flow. We highlight the advantages of our algorithm compared to standard denoising score matching and implicit score matching, as well as fully learned baselines for a wide range of inverse physics problems. The resulting inverse solver has excellent accuracy and temporal stability and, in contrast to other learned inverse solvers, allows for sampling the posterior of the solutions.}
}
@inproceedings{oh2021two,
title={Two-step Temporal Interpolation Network Using Forward Advection for Efficient Smoke Simulation},
author={Oh, Young Jin and Lee, In-Kwon},
booktitle={Computer Graphics Forum},
volume={40},
number={2},
pages={355--365},
year={2021},
category={fluid},
tag={temporal_interpolation},
organization={Wiley Online Library},
link={https://onlinelibrary.wiley.com/doi/10.1111/cgf.142638},
abstract={Abstract In this paper, we propose a two‐step temporal interpolation network using forward advection to generate smoke simulation efficiently. By converting a low frame rate smoke simulation computed with a large time step into a high frame rate smoke simulation through inference of temporal interpolation networks, the proposed method can efficiently generate smoke simulation with a high frame rate and low computational costs. The first step of the proposed method is optical flow‐based temporal interpolation using deep neural networks (DNNs) for two given smoke animation frames. In the next step, we compute temporary smoke frames with forward advection, a physical computation with a low computational cost. We then interpolate between the results of the forward advection and those of the first step to generate more accurate and enhanced interpolated results. We performed quantitative analyses of the results generated by the proposed method and previous temporal interpolation methods. Furthermore, we experimentally compared the performance of the proposed method with previous methods using DNNs for smoke simulation. We found that the results generated by the proposed method are more accurate and closer to the ground truth smoke simulation than those generated by the previous temporal interpolation methods. We also confirmed that the proposed method generates smoke simulation results more efficiently with lower computational costs than previous smoke simulation methods using DNNs.}
}
@inproceedings{li2023d,
title={D-Cloth: Skinning-based Cloth Dynamic Prediction with a Three-stage Network},
author={Li, Yu Di and Tang, Min and Chen, Xiao Rui and Yang, Yun and Tong, Ruo Feng and An, Bai Lin and Yang, Shuang Cai and Li, Yao and Kou, Qi Long},
booktitle={Computer Graphics Forum},
volume={42},
number={7},
pages={e14937},
year={2023},
category={cloth, neural_solver},
organization={Wiley Online Library},
link={https://min-tang.github.io/home/DCloth/},
abstract={Abstract We propose a three‐stage network that utilizes a skinning‐based model to accurately predict dynamic cloth deformation. Our approach decomposes cloth deformation into three distinct components: static, coarse dynamic, and wrinkle dynamic components. To capture these components, we train our three‐stage network accordingly. In the first stage, the static component is predicted by constructing a static skinning model that incorporates learned joint increments and skinning weight increments. Then, in the second stage, the coarse dynamic component is added to the static skinning model by incorporating serialized skeleton information. Finally, in the third stage, the mesh sequence stage refines the prediction by incorporating the wrinkle dynamic component using serialized mesh information. We have implemented our network and used it in a Unity game scene, enabling real‐time prediction of cloth dynamics. Our implementation achieves impressive prediction speeds of approximately 3.65ms using an NVIDIA GeForce RTX 3090 GPU and 9.66ms on an Intel i7‐7700 CPU. Compared to SOTA methods, our network excels in accurately capturing fine dynamic cloth deformations.}
}
@INPROCEEDINGS{hu2019chain,
author={Hu, Yuanming and Liu, Jiancheng and Spielberg, Andrew and Tenenbaum, Joshua B. and Freeman, William T. and Wu, Jiajun and Rus, Daniela and Matusik, Wojciech},
booktitle={International Conference on Robotics and Automation (ICRA)},
title={ChainQueen: A Real-Time Differentiable Physical Simulator for Soft Robotics},
year={2019},
category={softbody,differentiable_simulation,control,embodied_ai},
volume={},
number={},
pages={6265-6271},
keywords={Graphics processing units;Computational modeling;Soft robotics;Three-dimensional displays;Planning;Inverse problems},
doi={10.1109/ICRA.2019.8794333},
link={https://github.com/yuanming-hu/ChainQueen},
abstract={Physical simulators have been widely used in robot planning and control. Among them, differentiable simulators are particularly favored, as they can be incorporated into gradient-based optimization algorithms that are efficient in solving inverse problems such as optimal control and motion planning. Therefore, rigid body simulators and recently their differentiable variants are studied extensively. Simulating deformable objects is, however, more challenging compared to rigid body dynamics. The underlying physical laws of deformable objects are more complex, and the resulting systems have orders of magnitude more degrees of freedom and there-fore they are significantly more computationally expensive to simulate. Computing gradients with respect to physical design or controller parameters is typically even more computationally challenging. In this paper, we propose a real-time, differentiable hybrid Lagrangian-Eulerian physical simulator for deformable objects, ChainQueen, based on the Moving Least Squares Material Point Method (MLS-MPM). MLS-MPM can simulate deformable objects with collisions and can be seamlessly incorporated into soft robotic systems. We demonstrate that our simulator achieves high precision in both forward simulation and backward gradient computation. We have successfully employed it in a diverse set of inference, control and co-design tasks for soft robotics.}
}
@inproceedings{deng2023vortex,
title={Learning Vortex Dynamics for Fluid Inference and Prediction},
author={Yitong Deng and Hong-Xing Yu and Jiajun Wu and Bo Zhu},
booktitle={Proceedings of the International Conference on Learning Representations},
year={2023},
category={fluid,differentiable_simulation,neural_solver,neural_representation,reconstruction},
tag={real2sim,fluid_reconstruction},
link={https://github.com/yitongdeng-projects/learning_vortex_dynamics_code},
abstract={We propose a novel differentiable vortex particle (DVP) method to infer and predict fluid dynamics from a single video. Lying at its core is a particle-based latent space to encapsulate the hidden, Lagrangian vortical evolution underpinning the observable, Eulerian flow phenomena. Our differentiable vortex particles are coupled with a learnable, vortex-to-velocity dynamics mapping to effectively capture the complex flow features in a physically-constrained, low-dimensional space. This representation facilitates the learning of a fluid simulator tailored to the input video that can deliver robust, long-term future predictions. The value of our method is twofold: first, our learned simulator enables the inference of hidden physics quantities (e.g., velocity field) purely from visual observation; secondly, it also supports future prediction, constructing the input video's sequel along with its future dynamics evolution. We compare our method with a range of existing methods on both synthetic and real-world videos, demonstrating improved reconstruction quality, visual plausibility, and physical integrity.}
}
@article{xiong2023neural,
title = {Neural vortex method: From finite Lagrangian particles to infinite dimensional Eulerian dynamics},
journal = {Computers \& Fluids},
volume = {258},
pages = {105811},
year = {2023},
category={fluid, neural_solver},
issn = {0045-7930},
doi = {https://doi.org/10.1016/j.compfluid.2023.105811},
url = {https://www.sciencedirect.com/science/article/pii/S0045793023000361},
author = {Shiying Xiong and Xingzhe He and Yunjin Tong and Yitong Deng and Bo Zhu},
keywords = {Vortex method, Neural network, Lagrangian dynamics, Eulerian dynamics},
link={https://arxiv.org/abs/2006.04178}
}
@inproceedings{xian2023fluidlab,
title={FluidLab: A Differentiable Environment for Benchmarking Complex Fluid Manipulation},
author={Xian, Zhou and Zhu, Bo and Xu, Zhenjia and Tung, Hsiao-Yu and Torralba, Antonio and Fragkiadaki, Katerina and Gan, Chuang},
booktitle={International Conference on Learning Representations},
year={2023},
category={fluid,differentiable_simulation,embodied_ai,engine},
tag={reinforce_learning},
link={https://fluidlab2023.github.io/},
abstract={Humans manipulate various kinds of fluids in their everyday life: creating latte art, scooping floating objects from water, rolling an ice cream cone, etc. Using robots to augment or replace human labors in these daily settings remain as a challenging task due to the multifaceted complexities of fluids. Previous research in robotic fluid manipulation mostly consider fluids governed by an ideal, Newtonian model in simple task settings (e.g., pouring). However, the vast majority of real-world fluid systems manifest their complexities in terms of the fluid's complex material behaviors and multi-component interactions, both of which were well beyond the scope of the current literature. To evaluate robot learning algorithms on understanding and interacting with such complex fluid systems, a comprehensive virtual platform with versatile simulation capabilities and well-established tasks is needed. In this work, we introduce FluidLab, a simulation environment with a diverse set of manipulation tasks involving complex fluid dynamics. These tasks address interactions between solid and fluid as well as among multiple fluids. At the heart of our platform is a fully differentiable physics simulator, FluidEngine, providing GPU-accelerated simulations and gradient calculations for various material types and their couplings. We identify several challenges for fluid manipulation learning by evaluating a set of reinforcement learning and trajectory optimization methods on our platform. To address these challenges, we propose several domain-specific optimization schemes coupled with differentiable physics, which are empirically shown to be effective in tackling optimization problems featured by fluid system's non-convex and non-smooth properties. Furthermore, we demonstrate reasonable sim-to-real transfer by deploying optimized trajectories in real-world settings.}
}
@article{ladicky2015data,
author = {Ladick\'{y}, L'ubor and Jeong, SoHyeon and Solenthaler, Barbara and Pollefeys, Marc and Gross, Markus},
title = {Data-driven fluid simulations using regression forests},
year = {2015},
category={fluid,reconstruction},
tag={fluid_reconstruction},
issue_date = {November 2015},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {34},
number = {6},
issn = {0730-0301},
url = {https://doi.org/10.1145/2816795.2818129},
doi = {10.1145/2816795.2818129},
journal = {ACM Trans. Graph.},
month = {nov},
articleno = {199},
numpages = {9},
keywords = {regression forest, fluid simulation, data-driven},
link={https://dl.acm.org/doi/10.1145/2816795.2818129},
abstract={Traditional fluid simulations require large computational resources even for an average sized scene with the main bottleneck being a very small time step size, required to guarantee the stability of the solution. Despite a large progress in parallel computing and efficient algorithms for pressure computation in the recent years, realtime fluid simulations have been possible only under very restricted conditions. In this paper we propose a novel machine learning based approach, that formulates physics-based fluid simulation as a regression problem, estimating the acceleration of every particle for each frame. We designed a feature vector, directly modelling individual forces and constraints from the Navier-Stokes equations, giving the method strong generalization properties to reliably predict positions and velocities of particles in a large time step setting on yet unseen test videos. We used a regression forest to approximate the behaviour of particles observed in the large training set of simulations obtained using a traditional solver. Our GPU implementation led to a speed-up of one to three orders of magnitude compared to the state-of-the-art position-based fluid solver and runs in real-time for systems with up to 2 million particles.}
}
@inproceedings{franz2023learning,
title={Learning to Estimate Single-View Volumetric Flow Motions without 3D Supervision},
author={Erik Franz and Barbara Solenthaler and Nils Thuerey},
booktitle={International Conference on Learning Representations},
year={2023},
category={fluid,reconstruction},
tag={fluid_reconstruction},
url={https://openreview.net/forum?id=2vmGv5wPDBZ},
link={https://ge.in.tum.de/publications/2023-franz-neuralglobtrans/},
abstract={We address the challenging problem of jointly inferring the 3D flow and volumetric densities moving in a fluid from a monocular input video with a deep neural network. Despite the complexity of this task, we show that it is possible to train the corresponding networks without requiring any 3D ground truth for training. In the absence of ground truth data we can train our model with observations from real-world capture setups instead of relying on synthetic reconstructions. We make this unsupervised training approach possible by first generating an initial prototype volume which is then moved and transported over time without the need for volumetric supervision. Our approach relies purely on image-based losses, an adversarial discriminator network, and regularization. Our method can estimate long-term sequences in a stable manner, while achieving closely matching targets for inputs such as rising smoke plumes.}
}
@article{xie2024dynamic,
title={Dynamic ocean inverse modeling based on differentiable rendering},
author={Xie, Xueguang and Gao, Yang and Hou, Fei and Hao, Aimin and Qin, Hong},
journal={Computational Visual Media},
volume={10},
number={2},
pages={279--294},
year={2024},
category={fluid,differentiable_simulation,reconstruction},
tag={real2sim,fluid_reconstruction},
publisher={Springer},
link={https://link.springer.com/article/10.1007/s41095-023-0338-4},
abstract={Learning and inferring underlying motion patterns of captured 2D scenes and then re-creating dynamic evolution consistent with the real-world natural phenomena have high appeal for graphics and animation. To bridge the technical gap between virtual and real environments, we focus on the inverse modeling and reconstruction of visually consistent and property-verifiable oceans, taking advantage of deep learning and differentiable physics to learn geometry and constitute waves in a self-supervised manner. First, we infer hierarchical geometry using two networks, which are optimized via the differentiable renderer. We extract wave components from the sequence of inferred geometry through a network equipped with a differentiable ocean model. Then, ocean dynamics can be evolved using the reconstructed wave components. Through extensive experiments, we verify that our new method yields satisfactory results for both geometry reconstruction and wave estimation. Moreover, the new framework has the inverse modeling potential to facilitate a host of graphics applications, such as the rapid production of physically accurate scene animation and editing guided by real ocean scenes.}
}
@article{Stuyck2023DiffXPBD,
author = {Stuyck, Tuur and Chen, Hsiao-yu},
title = {DiffXPBD: Differentiable Position-Based Simulation of Compliant Constraint Dynamics},
year = {2023},
category={cloth,differentiable_simulation},
tag={material_estimation},
issue_date = {August 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {6},
number = {3},
url = {https://doi.org/10.1145/3606923},
doi = {10.1145/3606923},
journal = {Proc. ACM Comput. Graph. Interact. Tech.},
month = {aug},
articleno = {51},
numpages = {14},
keywords = {differentiable simulation, parameter estimation},
link={https://doi.org/10.1145/3606923},
abstract={We present DiffXPBD, a novel and efficient analytical formulation for the differentiable position-based simulation of compliant constrained dynamics (XPBD). Our proposed method allows computation of gradients of numerous parameters with respect to a goal function simultaneously leveraging a performant simulation model. The method is efficient, thus enabling differentiable simulations of high resolution geometries and degrees of freedom (DoFs). Collisions are naturally included in the framework. Our differentiable model allows a user to easily add additional optimization variables. Every control variable gradient requires the computation of only a few partial derivatives which can be computed using automatic differentiation code. We demonstrate the efficacy of the method with examples such as elastic cloth and volumetric material parameter estimation, initial value optimization, optimizing for underlying body shape and pose by only observing the clothing, and optimizing a time-varying external force sequence to match sparse keyframe shapes at specific times. Our approach demonstrates excellent efficiency and we demonstrate this on high resolution meshes with optimizations involving over 26 million degrees of freedom. Making an existing solver differentiable requires only a few modifications and the model is compatible with both modern CPU and GPU multi-core hardware.}
}
@inproceedings{tompson2017accelerating,
title={Accelerating eulerian fluid simulation with convolutional networks},
author={Tompson, Jonathan and Schlachter, Kristofer and Sprechmann, Pablo and Perlin, Ken},
booktitle={International conference on machine learning},
pages={3424--3433},
year={2017},
category={fluid,neural_solver},
organization={PMLR},
link={https://github.com/google/FluidNet},
abstract={Efficient simulation of the Navier-Stokes equations for fluid flow is a long standing problem in applied mathematics, for which state-of-the-art methods require large compute resources. In this work, we propose a data-driven approach that leverages the approximation power of deep-learning with the precision of standard solvers to obtain fast and highly realistic simulations. Our method solves the incompressible Euler equations using the standard operator splitting method, in which a large sparse linear system with many free parameters must be solved. We use a Convolutional Network with a highly tailored architecture, trained using a novel unsupervised learning framework to solve the linear system. We present real-time 2D and 3D simulations that outperform recently proposed data-driven methods; the obtained results are realistic and show good generalization properties.}
}
@article{chu2017data,
author = {Chu, Mengyu and Thuerey, Nils},
title = {Data-driven synthesis of smoke flows with CNN-based feature descriptors},
year = {2017},
category={fluid},
tag={cnn, super_resolution},
issue_date = {August 2017},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {36},
number = {4},
issn = {0730-0301},
url = {https://doi.org/10.1145/3072959.3073643},
doi = {10.1145/3072959.3073643},
journal = {ACM Trans. Graph.},
month = {jul},
articleno = {69},
numpages = {14},
keywords = {convolutional neural networks, fluid simulation, low-dimensional feature descriptors},
link={https://ge.in.tum.de/publications/2017-sig-chu/},
abstract={We present a novel data-driven algorithm to synthesize high resolution flow simulations with reusable repositories of space-time flow data. In our work, we employ a descriptor learning approach to encode the similarity between fluid regions with differences in resolution and numerical viscosity. We use convolutional neural networks to generate the descriptors from fluid data such as smoke density and flow velocity. At the same time, we present a deformation limiting patch advection method which allows us to robustly track deformable fluid regions. With the help of this patch advection, we generate stable space-time data sets from detailed fluids for our repositories. We can then use our learned descriptors to quickly localize a suitable data set when running a new simulation. This makes our approach very efficient, and resolution independent. We will demonstrate with several examples that our method yields volumes with very high effective resolutions, and non-dissipative small scale details that naturally integrate into the motions of the underlying flow.}
}
@article{tang2023physics,
author = {Tang, Jingwei and Kim, Byungsoo and Azevedo, Vinicius C. and Solenthaler, Barbara},
title = {Physics-Informed Neural Corrector for Deformation-based Fluid Control},
journal = {Computer Graphics Forum},
volume = {42},
number = {2},
pages = {161-173},
keywords = {CCS Concepts, Computing methodologies:Physical Simulation, Neural networks},
doi = {https://doi.org/10.1111/cgf.14751},
url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/cgf.14751},
eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/cgf.14751},
year = {2023},
category={fluid,differentiable_simulation,control},
tag={fluid_control},
link={https://studios.disneyresearch.com/2023/05/07/physics-informed-neural-corrector-for-deformation-based-fluid-control/},
abstract={Abstract Controlling fluid simulations is notoriously difficult due to its high computational cost and the fact that user control inputs can cause unphysical motion. We present an interactive method for deformation‐based fluid control. Our method aims at balancing the direct deformations of fluid fields and the preservation of physical characteristics. We train convolutional neural networks with physics‐inspired loss functions together with a differentiable fluid simulator, and provide an efficient workflow for flow manipulations at test time. We demonstrate diverse test cases to analyze our carefully designed objectives and show that they lead to physical and eventually visually appealing modifications on edited fluid data.}
}
@article{kim2022deep,
author = {Kim, Byungsoo and Huang, Xingchang and Wuelfroth, Laura and Tang, Jingwei and Cordonnier, Guillaume and Gross, Markus and Solenthaler, Barbara},
title = {Deep Reconstruction of 3D Smoke Densities from Artist Sketches},
journal = {Computer Graphics Forum},
volume = {41},
number = {2},
pages = {97-110},
keywords = {CCS Concepts, Computing methodologies:Shape modeling, Neural networks},
doi = {https://doi.org/10.1111/cgf.14461},
url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/cgf.14461},
eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/cgf.14461},
year = {2022},
category={fluid,reconstruction},
tag={fluid_reconstruction,user_interaction,cnn},
link={https://onlinelibrary.wiley.com/doi/abs/10.1111/cgf.14461},
abstract={Abstract Creative processes of artists often start with hand‐drawn sketches illustrating an object. Pre‐visualizing these keyframes is especially challenging when applied to volumetric materials such as smoke. The authored 3D density volumes must capture realistic flow details and turbulent structures, which is highly non‐trivial and remains a manual and time‐consuming process. We therefore present a method to compute a 3D smoke density field directly from 2D artist sketches, bridging the gap between early‐stage prototyping of smoke keyframes and pre‐visualization. From the sketch inputs, we compute an initial volume estimate and optimize the density iteratively with an updater CNN. Our differentiable sketcher is embedded into the end‐to‐end training, which results in robust reconstructions. Our training data set and sketch augmentation strategy are designed such that it enables general applicability. We evaluate the method on synthetic inputs and sketches from artists depicting both realistic smoke volumes and highly non‐physical smoke shapes. The high computational performance and robustness of our method at test time allows interactive authoring sessions of volumetric density fields for rapid prototyping of ideas by novice users.}
}
@inproceedings{holl2020learningto,
title={Learning to Control PDEs with Differentiable Physics},
author={Philipp Holl and Vladlen Koltun and Nils Thuerey},
booktitle={International Conference on Learning Representations},
year={2020},
category={fluid,differentiable_simulation,control},
tag={fluid_control},
doi={10.48550/arxiv.2001.07457},
link={http://arxiv.org/abs/2001.07457},
project={https://github.com/p-holl/PDE-Control},
openalex_id={W2995001833},
abstract={Predicting outcomes and planning interactions with the physical world are long-standing goals for machine learning. A variety of such tasks involves continuous physical systems, which can be described by partial differential equations (PDEs) with many degrees of freedom. Existing methods that aim to control the dynamics of such systems are typically limited to relatively short time frames or a small number of interaction parameters. We present a novel hierarchical predictor-corrector scheme which enables neural networks to learn to understand and control complex nonlinear physical systems over long time frames. We propose to split the problem into two distinct tasks: planning and control. To this end, we introduce a predictor network that plans optimal trajectories and a control network that infers the corresponding control parameters. Both stages are trained end-to-end using a differentiable PDE solver. We demonstrate that our method successfully develops an understanding of complex physical systems and learns to control them for tasks involving PDEs such as the incompressible Navier-Stokes equations.},
project_kind={code_repo}
}
@inproceedings{pfaff2020learningmesh,
title={Learning Mesh-Based Simulation with Graph Networks},
author={Tobias Pfaff and Meire Fortunato and Álvaro Sánchez‐González and Peter Battaglia},
booktitle={International Conference on Learning Representations},
year={2020},
category={cloth,neural_solver},
tag={gnn},
doi={10.48550/arxiv.2010.03409},
link={http://arxiv.org/abs/2010.03409},
project={https://sites.google.com/view/meshgraphnets},
openalex_id={W3092228416},
abstract={Mesh-based simulations are central to modeling complex physical systems in many disciplines across science and engineering. Mesh representations support powerful numerical integration methods and their resolution can be adapted to strike favorable trade-offs between accuracy and efficiency. However, high-dimensional scientific simulations are very expensive to run, and solvers and parameters must often be tuned individually to each system studied. Here we introduce MeshGraphNets, a framework for learning mesh-based simulations using graph neural networks. Our model can be trained to pass messages on a mesh graph and to adapt the mesh discretization during forward simulation. Our results show it can accurately predict the dynamics of a wide range of physical systems, including aerodynamics, structural mechanics, and cloth. The model's adaptivity supports learning resolution-independent dynamics and can scale to more complex state spaces at test time. Our method is also highly efficient, running 1-2 orders of magnitude faster than the simulation on which it is trained. Our approach broadens the range of problems on which neural network simulators can operate and promises to improve the efficiency of complex, scientific modeling tasks.},
project_kind={webpage}
}
@inproceedings{huang2021modelpredictive,
title={Model-Predictive Control of Blood Suction for Surgical Hemostasis using Differentiable Fluid Simulations},
author={Jing-bin Huang and Fei Liu and Florian Richter and Michael C. Yip},
booktitle={2021 IEEE International Conference on Robotics and Automation (ICRA)},
year={2021},
category={fluid,differentiable_simulation,control,embodied_ai},
tag={fluid_control},
doi={10.1109/icra48506.2021.9561624},
link={https://doi.org/10.1109/icra48506.2021.9561624},
project={https://ucsdarclab.com/autopublication/model-predictive-control-of-blood-suction-for-surgical-hemostasis-using-differentiable-fluid-simulations},
openalex_id={W3207462099},
abstract={Recent developments in surgical robotics have led to new advancements in the automation of surgical sub-tasks such as suturing, soft tissue manipulation, tissue tensioning and cutting. However, integration of dynamics to optimize these control policies for the variety of scenes encountered in surgery remains unsolved. Towards this effort, we investigate the integration of differentiable fluid dynamics to optimizing a suction tool’s trajectory to clear the surgical field from blood as fast as possible. The fully differentiable fluid dynamics is integrated with a novel suction model for effective model predictive control of the tool. The differentiability of the fluid model is crucial because we utilize the gradients of the fluid states with respect to the suction tool position to optimize the trajectory. Through a series of experiments, we demonstrate how, by incorporating fluid models, the trajectories generated by our method can perform as good as or better than handcrafted human-intuitive suction policies. We also show that our method is adaptable and can work in different cavity conditions while using a single handcrafted strategy fails.},
project_kind={webpage}
}
@inproceedings{heiden2021disectdifferentiable,
title={DiSECt: A Differentiable Simulation Engine for Autonomous Robotic Cutting},
author={Eric Heiden and Miles Macklin and Yashraj Narang and Dieter Fox and Animesh Garg and Fábio Ramos},
booktitle={Robotics: Science and Systems XVII},
year={2021},
category={rigidbody,differentiable_simulation,neural_solver,reconstruction,embodied_ai,engine},
doi={10.15607/rss.2021.xvii.067},
link={https://doi.org/10.15607/rss.2021.xvii.067},
project={https://eric-heiden.com/publication/2021-disect-rss},
openalex_id={W3164136237},
abstract={Robotic cutting of soft materials is critical for applications such as food processing, household automation, and surgical manipulation. As in other areas of robotics, simulators can facilitate controller verification, policy learning, and dataset generation. Moreover, differentiable simulators can enable gradient-based optimization, which is invaluable for calibrating simulation parameters and optimizing controllers. In this work, we present DiSECt: the first differentiable simulator for cutting soft materials. The simulator augments the finite element method (FEM) with a continuous contact model based on signed distance fields (SDF), as well as a continuous damage model that inserts springs on opposite sides of the cutting plane and allows them to weaken until zero stiffness, enabling crack formation. Through various experiments, we evaluate the performance of the simulator. We first show that the simulator can be calibrated to match resultant forces and deformation fields from a state-of-the-art commercial solver and real-world cutting datasets, with generality across cutting velocities and object instances. We then show that Bayesian inference can be performed efficiently by leveraging the differentiability of the simulator, estimating posteriors over hundreds of parameters in a fraction of the time of derivative-free methods. Finally, we illustrate that control parameters in the simulation can be optimized to minimize cutting forces via lateral slicing motions. We publish videos and additional results on our project website at https://diff-cutting-sim.github.io.},
project_kind={webpage}
}
@inproceedings{yang2021learningto,
title={Learning to Propagate Interaction Effects for Modeling Deformable Linear Objects Dynamics},
author={Yuxuan Yang and Johannes A. Stork and Todor Stoyanov},
booktitle={2021 IEEE International Conference on Robotics and Automation (ICRA)},
year={2021},
category={softbody,neural_solver,control,embodied_ai},
doi={10.1109/icra48506.2021.9561636},
link={https://doi.org/10.1109/icra48506.2021.9561636},
project={https://amm.aass.oru.se/icra2021-learning-dlo},
openalex_id={W3206634839},
abstract={Modeling dynamics of deformable linear objects (DLOs), such as cables, hoses, sutures, and catheters, is an important and challenging problem for many robotic manipulation applications. In this paper, we propose the first method to model and learn full 3D dynamics of DLOs from data. Our approach is capable of capturing the complex twisting and bending dynamics of DLOs and allows local effects to propagate globally. To this end, we adapt the interaction network (IN) dynamics learning method for capturing the interaction between neighboring segments in a DLO and augment it with a recurrent model for propagating interaction effects along the length of a DLO. For learning twisting and bending dynamics in 3D, we also introduce a new suitable representation of DLO segments and their relationships. Unlike the original IN method, our model learns to propagate the effects of local interaction between neighboring segments to each segment in the chain within a single time step, without the need for iterated propagation steps. Evaluation of our model with synthetic and newly collected real-world data shows better accuracy and generalization in short-term and long- term predictions than the current state of the art. We further integrate our learned model in a model predictive control scheme and use it to successfully control the shape of a DLO. Our implementation is available at https://gitsvn-nt.oru.se/ammlab–public/in–bilstm.},
project_kind={webpage}
}
@inproceedings{heiden2021neuralsimaugmenting,
title={NeuralSim: Augmenting Differentiable Simulators with Neural Networks},
author={Eric Heiden and David Millard and Erwin Coumans and Yizhou Sheng and Gaurav S. Sukhatme},
booktitle={2021 IEEE International Conference on Robotics and Automation (ICRA)},
year={2021},
category={rigidbody,differentiable_simulation,neural_solver},
doi={10.1109/icra48506.2021.9560935},
link={https://doi.org/10.1109/icra48506.2021.9560935},
project={https://uscresl.org/publication/neuralsim-augmenting-differentiable-simulators-with-neural-networks},
openalex_id={W3098074920},
abstract={Differentiable simulators provide an avenue for closing the sim-to-real gap by enabling the use of efficient, gradient-based optimization algorithms to find the simulation parameters that best fit the observed sensor readings. Nonetheless, these analytical models can only predict the dynamical behavior of systems for which they have been designed. In this work, we study the augmentation of a novel differentiable rigid-body physics engine via neural networks that is able to learn nonlinear relationships between dynamic quantities and can thus model effects not accounted for in traditional simulators. Such augmentations require less data to train and generalize better compared to entirely data-driven models. Through extensive experiments, we demonstrate the ability of our hybrid simulator to learn complex dynamics involving frictional contacts from real data, as well as match known models of viscous friction, and present an approach for automatically discovering useful augmentations. We show that, besides benefiting dynamics modeling, inserting neural networks can accelerate model-based control architectures. We observe a ten-fold speedup when replacing the QP solver inside a model-predictive gait controller for quadruped robots with a neural network, allowing us to significantly improve control delays as we demonstrate in real-hardware experiments. We publish code, additional results and videos from our experiments on our project webpage at https://sites.google.com/usc.edu/neuralsim.},
project_kind={webpage}
}
@inproceedings{zheng2021deepemulator,
title={A Deep Emulator for Secondary Motion of 3D Characters},
author={Mianlun Zheng and Yi Zhou and Duygu Ceylan and Jernej Barbič},
booktitle={2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2021},
category={softbody, neural_solver, avatar},
doi={10.1109/cvpr46437.2021.00587},
link={https://doi.org/10.1109/cvpr46437.2021.00587},
project={https://github.com/ZhengMianlun/deep_emulator},
openalex_id={W3178713097},
abstract={Fast and light-weight methods for animating 3D characters are desirable in various applications such as computer games. We present a learning-based approach to enhance skinning-based animations of 3D characters with vivid secondary motion effects. We design a neural network that encodes each local patch of a character simulation mesh where the edges implicitly encode the internal forces between the neighboring vertices. The network emulates the ordinary differential equations of the character dynamics, predicting new vertex positions from the current accelerations, velocities and positions. Being a local method, our network is independent of the mesh topology and generalizes to arbitrarily shaped 3D character meshes at test time. We further represent per-vertex constraints and material properties such as stiffness, enabling us to easily adjust the dynamics in different parts of the mesh. We evaluate our method on various character meshes and complex motion sequences. Our method can be over 30 times more efficient than ground-truth physically based simulation, and outperforms alternative solutions that provide fast approximations.},
project_kind={code_repo}
}
@article{shao2021accuratelysolving,
title={Accurately Solving Rod Dynamics with Graph Learning},
author={Han Shao and Tassilo Kugelstadt and Torsten Hädrich and Wojtek Pałubicki and Jan Bender and Sören Pirk and Dominik L. Michels},
journal={Neural Information Processing Systems},
year={2021},
category={softbody,neural_solver},
tag={gnn},
link={http://hdl.handle.net/10754/679142},
project={https://computationalsciences.org/publications/shao-2021-physical-systems-graph-learning.html},
openalex_id={W3132745411},
abstract={Iterative solvers are widely used to accurately simulate physical systems. These solvers require initial guesses to generate a sequence of improving approximate solutions. In this contribution, we introduce a novel method to accelerate iterative solvers for rod dynamics with graph networks (GNs) by predicting the initial guesses to reduce the number of iterations. Unlike existing methods that aim to learn physical systems in an end-to-end manner, our approach guarantees long-term stability and therefore leads to more accurate solutions. Furthermore, our method improves the run time performance of traditional iterative solvers for rod dynamics. To explore our method we make use of position-based dynamics (PBD) as a common solver for physical systems and evaluate it by simulating the dynamics of elastic rods. Our approach is able to generalize across different initial conditions, discretizations, and realistic material properties. We demonstrate that it also performs well when taking discontinuous effects into account such as collisions between individual rods. Finally, to illustrate the scalability of our approach, we simulate complex 3D tree models composed of over a thousand individual branch segments swaying in wind fields.},
project_kind={webpage}
}
@article{ma2021diffaqua,
title={DiffAqua},
author={Pingchuan Ma and Tao Du and John Z. Zhang and Kui Wu and Andrew Spielberg and Robert K. Katzschmann and Wojciech Matusik},
journal={ACM Transactions on Graphics},
year={2021},
category={softbody, differentiable_simulation, optimization, embodied_ai},
tag={soft_robot, underwater_robot, robot_control},
doi={10.1145/3450626.3459832},
link={https://doi.org/10.1145/3450626.3459832},
project={https://diffaqua.csail.mit.edu/},
openalex_id={W3149466088},
abstract={The computational design of soft underwater swimmers is challenging because of the high degrees of freedom in soft-body modeling. In this paper, we present a differentiable pipeline for co-designing a soft swimmer's geometry and controller. Our pipeline unlocks gradient-based algorithms for discovering novel swimmer designs more efficiently than traditional gradient-free solutions. We propose Wasserstein barycenters as a basis for the geometric design of soft underwater swimmers since it is differentiable and can naturally interpolate between bio-inspired base shapes via optimal transport. By combining this design space with differentiable simulation and control, we can efficiently optimize a soft underwater swimmer's performance with fewer simulations than baseline methods. We demonstrate the efficacy of our method on various design problems such as fast, stable, and energy-efficient swimming and demonstrate applicability to multi-objective design.},
project_kind={project_page}
}
@inproceedings{ma2022learninglatent,
title={Learning Latent Graph Dynamics for Visual Manipulation of Deformable Objects},
author={Xiao Ma and David Hsu and Wee Sun Lee},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={cloth,neural_solver,reconstruction,embodied_ai},
tag={real2sim,cloth_reconstruction,gnn},
doi={10.1109/icra46639.2022.9811597},
link={https://doi.org/10.1109/icra46639.2022.9811597},
openalex_id={W4285102216},
abstract={Manipulating deformable objects, such as ropes and clothing, is a long-standing challenge in robotics, because of their large degrees of freedom, complex non-linear dynamics, and self-occlusion in visual perception. The key difficulty is a suitable representation, rich enough to capture the object shape, dynamics for manipulation and yet simple enough to be estimated reliably from visual observations. This work aims to learn latent Graph dynamics for DefOrmable Object Manipulation (G-DOOM). G-DOOM approximates a deformable object as a sparse set of interacting keypoints, which are extracted automatically from images via unsupervised learning. It learns a graph neural network that captures abstractly the geometry and the interaction dynamics of the keypoints. To handle object self-occlusion, G-DOOM uses a recurrent neural network to track the keypoints over time and condition their interactions on the history. We then train the resulting recurrent graph dynamics model through contrastive learning in a high-fidelity simulator. For manipulation planning, G-DOOM reasons explicitly about the learned dynamics model through model-predictive control applied at each keypoint. Preliminary experiments of G-DOOM on a set of challenging rope and cloth manipulation tasks indicate strong performance, compared with state-of-the-art methods. Although trained in a simulator, G-DOOM transfers directly to a real robot for both rope and cloth manipulation 1 1 Demo video available online at https://youtu.be/oCfbNMx2sQI.}
}
@article{li2022fluidictopology,
title={Fluidic Topology Optimization with an Anisotropic Mixture Model},
author={Yifei Li and Tao Du and Sangeetha Srinivasan and Kui Wu and Bo Zhu and Eftychios Sifakis and Wojciech Matusik},
journal={ACM Transactions on Graphics},
year={2022},
category={fluid,optimization},
tag={shape_design},
doi={10.1145/3550454.3555429},
link={https://doi.org/10.1145/3550454.3555429},
project={https://people.csail.mit.edu/liyifei/publication/anisotropicstokes},
openalex_id={W4296973799},
abstract={Fluidic devices are crucial components in many industrial applications involving fluid mechanics. Computational design of a high-performance fluidic system faces multifaceted challenges regarding its geometric representation and physical accuracy. We present a novel topology optimization method to design fluidic devices in a Stokes flow context. Our approach is featured by its capability in accommodating a broad spectrum of boundary conditions at the solid-fluid interface. Our key contribution is an anisotropic and differentiable constitutive model that unifies the representation of different phases and boundary conditions in a Stokes model, enabling a topology optimization method that can synthesize novel structures with accurate boundary conditions from a background grid discretization. We demonstrate the efficacy of our approach by conducting several fluidic system design tasks with over four million design parameters.},
project_kind={webpage}
}
@inproceedings{lou2022learningobject,
title={Learning Object Relations with Graph Neural Networks for Target-Driven Grasping in Dense Clutter},
author={Xibai Lou and Yang Yang and Changhyun Choi},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={rigidbody, control, embodied_ai},
tag={gnn, robot_control},
doi={10.1109/icra46639.2022.9811601},
link={https://doi.org/10.1109/icra46639.2022.9811601},
openalex_id={W4221153162},
abstract={Robots in the real world frequently come across identical objects in dense clutter. When evaluating grasp poses in these scenarios, a target-driven grasping system requires knowledge of spatial relations between scene objects (e.g., proximity, adjacency, and occlusions). To efficiently complete this task, we propose a target-driven grasping system that simultaneously considers object relations and predicts 6-DoF grasp poses. A densely cluttered scene is first formulated as a grasp graph with nodes representing object geometries in the grasp coordinate frame and edges indicating spatial relations between the objects. We design a Grasp Graph Neural Network (G2N2) that evaluates the grasp graph and finds the most feasible 6-DoF grasp pose for a target object. Additionally, we develop a shape completion-assisted grasp pose sampling method that improves sample quality and consequently grasping efficiency. We compare our method against several baselines in both simulated and real settings. In real-world experiments with novel objects, our approach achieves a 77.78% grasping accuracy in densely cluttered scenarios, surpassing the best-performing baseline by more than 15%. Supplementary material is available at https://sites.google.com/umn.edu/graph-grasping.}
}
@inproceedings{zhu2022learningto,
title={Learning to Synthesize Volumetric Meshes from Vision-based Tactile Imprints},
author={Xinghao Zhu and Siddarth Jain and Masayoshi Tomizuka and Jeroen van Baar},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={softbody,reconstruction,embodied_ai},
tag={gnn},
doi={10.1109/icra46639.2022.9812092},
link={https://doi.org/10.1109/icra46639.2022.9812092},
openalex_id={W4221153262},
abstract={Vision-based tactile sensors typically utilize a deformable elastomer and a camera mounted above to provide high-resolution image observations of contacts. Obtaining accurate volumetric meshes for the deformed elastomer can provide direct contact information and benefit robotic grasping and manipulation. This paper focuses on learning to synthesize the volumetric mesh of the elastomer based on the image imprints acquired from vision-based tactile sensors. Synthetic image-mesh pairs and real-world images are gathered from 3D finite element methods (FEM) and physical sensors, respectively. A graph neural network (GNN) is introduced to learn the image-to-mesh mappings with supervised learning. A self-supervised adaptation method and image augmentation techniques are proposed to transfer networks from simulation to reality, from primitive contacts to unseen contacts, and from one sensor to another. Using these learned and adapted networks, our proposed method can accurately reconstruct the deformation of the real-world tactile sensor elastomer in various domains, as indicated by the quantitative and qualitative results.}
}
@inproceedings{heiden2022probabilisticinference,
title={Probabilistic Inference of Simulation Parameters via Parallel Differentiable Simulation},
author={Eric Heiden and Christopher E. Denniston and David Millard and Fábio Ramos and Gaurav S. Sukhatme},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={rigidbody,differentiable_simulation,reconstruction,embodied_ai},
doi={10.1109/icra46639.2022.9812293},
link={https://doi.org/10.1109/icra46639.2022.9812293},
project={https://uscresl.github.io/prob-diff-sim},
openalex_id={W3197308999},
abstract={Reproducing real world dynamics in simulation is critical for the development of new control and perception methods. This task typically involves the estimation of simu-lation parameter distributions from observed rollouts through an inverse inference problem characterized by multi-modality and skewed distributions. We address this challenging problem through a novel Bayesian inference approach that approximates a posterior distribution over simulation parameters given real sensor measurements. By extending the commonly used Gaus-sian likelihood model for trajectories via the multiple-shooting formulation, our gradient-based particle inference algorithm, Stein Variational Gradient Descent, is able to identify highly nonlinear, underactuated systems. We leverage GPU code gen-eration and differentiable simulation to evaluate the likelihood and its gradient for many particles in parallel. Our algorithm infers nonparametric distributions over simulation parame-ters more accurately than comparable baselines and handles constraints over parameters efficiently through gradient-based optimization. We evaluate estimation performance on several physical experiments. On an underactuated mechanism where a 7-DOF robot arm excites an object with an unknown mass configuration, we demonstrate how the inference technique can identify symmetries between the parameters and provide highly accurate predictions. Website: https://uscresl.github.io/prob-diff-sim},
project_kind={project_page}
}
@inproceedings{shi2022robocraftlearning,
title={RoboCraft: Learning to See, Simulate, and Shape Elasto-Plastic Objects with Graph Networks},
author={Haochen Shi and Huazhe Xu and Zhiao Huang and Yunzhu Li and Jiajun Wu},
booktitle={Robotics: Science and Systems XVIII},
year={2022},
category={softbody,neural_solver,reconstruction,embodied_ai},
tag={gnn},
doi={10.15607/rss.2022.xviii.008},
link={https://doi.org/10.15607/rss.2022.xviii.008},
openalex_id={W4229451778},
abstract={Modeling and manipulating elasto-plastic objects are essential capabilities for robots to perform complex industrial and household interaction tasks (e.g., stuffing dumplings, rolling sushi, and making pottery). However, due to the high degree of freedom of elasto-plastic objects, significant challenges exist in virtually every aspect of the robotic manipulation pipeline, e.g., representing the states, modeling the dynamics, and synthesizing the control signals. We propose to tackle these challenges by employing a particle-based representation for elasto-plastic objects in a model-based planning framework. Our system, RoboCraft, only assumes access to raw RGBD visual observations. It transforms the sensing data into particles and learns a particle-based dynamics model using graph neural networks (GNNs) to capture the structure of the underlying system. The learned model can then be coupled with model-predictive control (MPC) algorithms to plan the robot's behavior. We show through experiments that with just 10 minutes of real-world robotic interaction data, our robot can learn a dynamics model that can be used to synthesize control signals to deform elasto-plastic objects into various target shapes, including shapes that the robot has never encountered before. We perform systematic evaluations in both simulation and the real world to demonstrate the robot's manipulation capabilities and ability to generalize to a more complex action space, different tool shapes, and a mixture of motion modes. We also conduct comparisons between RoboCraft and untrained human subjects controlling the gripper to manipulate deformable objects in both simulation and the real world. Our learned modelbased planning framework is comparable to and sometimes better than human subjects on the tested tasks. 1}
}
@inproceedings{lv2022sagcisystem,
title={SAGCI-System: Towards Sample-Efficient, Generalizable, Compositional, and Incremental Robot Learning},
author={Jun Lv and Qiaojun Yu and Lin Shao and Wenhai Liu and Wenqiang Xu and Cewu Lu},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={rigidbody,differentiable_simulation,embodied_ai},
doi={10.1109/icra46639.2022.9811859},
link={https://doi.org/10.1109/icra46639.2022.9811859},
openalex_id={W3216834690},
abstract={Building general-purpose robots to perform a diverse range of tasks in a large variety of environments in the physical world at the human level is extremely challenging. According to [1], it requires the robot learning to be sample-efficient, generalizable, compositional, and incremental. In this work, we introduce a systematic learning framework called SAGCI-system towards achieving these above four requirements. Our system first takes the raw point clouds gathered by the camera mounted on the robot's wrist as the inputs and produces initial modeling of the surrounding environment represented as a file of Unified Robot Description Format (URDF). Our system adopts a learning-augmented differentiable simulation that loads the URDF. The robot then utilizes the interactive perception to interact with the environment to online verify and modify the URDF. Leveraging the differentiable simulation, we propose a model-based learning algorithm combining object-centric and robot-centric stages to efficiently produce policies to accomplish manipulation tasks. We apply our system to perform articulated object manipulation tasks, both in the simulation and the real world. Extensive experiments demonstrate the effectiveness of our proposed learning framework. Supplemental materials and videos are available on our project webpage https://sites.google.com/view/egci.}
}
@inproceedings{schaff2022softrobots,
title={Soft Robots Learn to Crawl: Jointly Optimizing Design and Control with Sim-to-Real Transfer},
author={Charles Schaff and Audrey Sedal and Matthew R. Walter},
booktitle={Robotics: Science and Systems XVIII},
year={2022},
category={softbody,control,embodied_ai},
tag={soft_robot,reinforce_learning},
doi={10.15607/rss.2022.xviii.062},
link={https://doi.org/10.15607/rss.2022.xviii.062},
openalex_id={W4283784756},
abstract={This work provides a complete framework for the simulation, co-optimization, and sim-to-real transfer of the design and control of soft legged robots. The compliance of soft robots provides a form of "mechanical intelligence"-the ability to passively exhibit behaviors that would otherwise be difficult to program. Exploiting this capacity requires careful consideration of the coupling between mechanical design and control. Cooptimization provides a promising means to generate sophisticated soft robots by reasoning over this coupling. However, the complex nature of soft robot dynamics makes it difficult to achieve a simulation environment that is both sufficiently accurate to allow for sim-to-real transfer and fast enough for contemporary co-optimization algorithms. In this work, we describe a modularized model order reduction algorithm that significantly improves the efficiency of finite element simulation, while preserving the accuracy required to successfully learn effective soft robot design-control pairs that transfer to reality. We propose a reinforcement learning-based framework for co-optimization and demonstrate successful optimization, construction, and zero-shot sim-to-real transfer of several soft crawling robots. Our learned robot outperforms an expert-designed crawling robot, showing that our approach can generate novel, high-performing designs even in well-understood domains.}
}
@inproceedings{preiss2022trackingfast,
title={Tracking Fast Trajectories with a Deformable Object using a Learned Model},
author={James A. Preiss and David Millard and Tao Yao and Gaurav S. Sukhatme},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={softbody,neural_solver,embodied_ai},
doi={10.1109/icra46639.2022.9812189},
link={https://doi.org/10.1109/icra46639.2022.9812189},
project={https://uscresl.org/publication/tracking-fast-trajectories-with-a-deformable-object-using-a-learned-model},
openalex_id={W4285102175},
abstract={We propose a method for robotic control of deformable objects using a learned nonlinear dynamics model. After collecting a dataset of trajectories from the real system, we train a recurrent neural network (RNN) to approximate its input-output behavior with a latent state-space model. The RNN internal state is low-dimensional enough to enable realtime nonlinear control methods. We demonstrate a closed-loop control scheme with the RNN model using a standard nonlinear state observer and model-predictive controller. We apply our method to track a highly dynamic trajectory with a point on the deformable object, in real time and on real hardware. Our experiments show that the RNN model captures the true system's frequency response and can be used to track trajectories outside the training distribution. In an ablation study, we find that the full method improves tracking accuracy compared to an open-loop version without the state observer.},
project_kind={webpage}
}
@inproceedings{shen2022acidaction,
title={ACID: Action-Conditional Implicit Visual Dynamics for Deformable Object Manipulation},
author={Bokui Shen and Zhenyu Jiang and Christopher Choy and Silvio Savarese and Leonidas Guibas and Anima Anandkumar and Yuke Zhu},
booktitle={Robotics: Science and Systems XVIII},
year={2022},
category={fluid,neural_solver,reconstruction,embodied_ai},
tag={real2sim,fluid_reconstruction},
doi={10.15607/rss.2022.xviii.001},
link={https://doi.org/10.15607/rss.2022.xviii.001},
openalex_id={W4229002925},
abstract={Manipulating volumetric deformable objects in the real world, like plush toys and pizza dough, bring substantial challenges due to infinite shape variations, non-rigid motions, and partial observability. We introduce ACID, an actionconditional visual dynamics model for volumetric deformable objects based on structured implicit neural representations. ACID integrates two new techniques: implicit representations for action-conditional dynamics and geodesics-based contrastive learning. To represent deformable dynamics from partial RGB-D observations, we learn implicit representations of occupancy and flow-based forward dynamics. To accurately identify state change under large non-rigid deformations, we learn a correspondence embedding field through a novel geodesics-based contrastive loss. To evaluate our approach, we develop a simulation framework for manipulating complex deformable shapes in realistic scenes and a benchmark containing over 17,000 action trajectories with six types of plush toys and 78 variants. Our model achieves the best performance in geometry, correspondence, and dynamics predictions over existing approaches. The ACID dynamics models are successfully employed to goal-conditioned deformable manipulation tasks, resulting in a 30% increase in task success rate over the strongest baseline. For more results and information, please visit https://b0ku1.github.io/acid/.}
}
@article{arnavaz2022differentiabledepth,
title={Differentiable Depth for Real2Sim Calibration of Soft Body Simulations},
author={Kasra Arnavaz and Mads Nielsen and Paul G. Kry and Miles Macklin and Kenny Erleben},
journal={Computer Graphics Forum},
year={2022},
category={softbody,differentiable_simulation,reconstruction},
doi={10.1111/cgf.14720},
link={https://doi.org/10.1111/cgf.14720},
project={https://researchprofiles.ku.dk/en/publications/differentiable-depth-for-real2sim-calibration-of-soft-body-simula},
openalex_id={W4309790058},
abstract={Abstract In this work, we present a novel approach for calibrating material model parameters for soft body simulations using real data. We use a fully differentiable pipeline, combining a differentiable soft body simulator and differentiable depth rendering, which permits fast gradient‐based optimizations. Our method requires no data pre‐processing, and minimal experimental set‐up, as we directly minimize the L2‐norm between raw LIDAR scans and rendered simulation states. In essence, we provide the first marker‐free approach for calibrating a soft‐body simulator to match observed real‐world deformations. Our approach is inexpensive as it solely requires a consumer‐level LIDAR sensor compared to acquiring a professional marker‐based motion capture system. We investigate the effects of different material parameterizations and evaluate convergence for parameter optimization in both single and multi‐material scenarios of varying complexity. Finally, we show that our set‐up can be extended to optimize for dynamic behaviour as well.},
project_kind={webpage}
}
@inproceedings{chen2022virtualelastic,
title={Virtual Elastic Objects},
author={Hsiao-yu Chen and Edith Tretschk and Tuur Stuyck and Petr Kadleček and Ladislav Kavan and Etienne Vouga and Christoph Lassner},
booktitle={2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2022},
category={softbody,differentiable_simulation,reconstruction},
tag={real2sim},
doi={10.1109/cvpr52688.2022.01537},
link={https://doi.org/10.1109/cvpr52688.2022.01537},
project={https://hsiaoyu.github.io/VEO},
openalex_id={W4313023542},
abstract={We present Virtual Elastic Objects (VEOs): virtual objects that not only look like their real-world counterparts but also behave like them, even when subject to novel interactions. Achieving this presents multiple challenges: not only do objects have to be captured including the physical forces acting on them, then faithfully reconstructed and rendered, but also plausible material parameters found and simulated. To create VEOs, we built a multi-view capture system that captures objects under the influence of a compressed air stream. Building on recent advances in model-free, dynamic Neural Radiance Fields, we reconstruct the objects and corresponding deformation fields. We propose to use a differentiable, particle-based simulator to use these deformation fields to find representative material parameters, which enable us to run new simulations. To render simulated objects, we devise a method for integrating the simulation results with Neural Radiance Fields. The resulting method is applicable to a wide range of scenarios: it can handle objects composed of inhomogeneous material, with very different shapes, and it can simulate interactions with other virtual objects. We present our results using a newly collected dataset of 12 objects under a variety of force fields, which will be made available upon publication.},
project_kind={project_page}
}
@inproceedings{grigorev2023hoodhierarchical,
title={HOOD: Hierarchical Graphs for Generalized Modelling of Clothing Dynamics},
author={Artur Grigorev and Michael J. Black and Otmar Hilliges},
booktitle={2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2023},
category={cloth, neural_solver},
tag={gnn},
doi={10.1109/cvpr52729.2023.01627},
link={https://doi.org/10.1109/cvpr52729.2023.01627},
project={https://dolorousrtur.github.io/hood/?trk=public_post_comment-text},
openalex_id={W4386066472},
abstract={We propose a method that leverages graph neural networks, multi-level message passing, and unsupervised training to enable efficient prediction of realistic clothing dynamics. Whereas existing methods based on linear blend skinning must be trained for specific garments, our method, called HOOD, is agnostic to body shape and applies to tight-fitting garments as well as loose, free-flowing clothing. Furthermore, HOOD handles changes in topology (e.g., garments with buttons or zippers) and material properties at inference time. As one key contribution, we propose a hierarchical message-passing scheme that efficiently propagates stiff stretching modes while preserving local detail. We empirically show that HOOD outperforms strong baselines quantitatively and that its results are perceived as more realistic than state-of-the-art methods.},
project_kind={project_page}
}
@article{rodrguezpardo2023howwill,
title={How Will It Drape Like? Capturing Fabric Mechanics from Depth Images},
author={Carlos Rodríguez-Pardo and Melania Prieto-Martín and Dan Casas and Elena Garcés},
journal={Computer Graphics Forum},
year={2023},
category={cloth,reconstruction},
tag={cloth_reconstruction},
doi={10.1111/cgf.14750},
link={http://dx.doi.org/10.1111/cgf.14750},
project={https://carlosrodriguezpardo.es/projects/MechFromDepth},
openalex_id={W4365606148},
abstract={Abstract We propose a method to estimate the mechanical parameters of fabrics using a casual capture setup with a depth camera. Our approach enables to create mechanically‐correct digital representations of real‐world textile materials, which is a fundamental step for many interactive design and engineering applications. As opposed to existing capture methods, which typically require expensive setups, video sequences, or manual intervention, our solution can capture at scale, is agnostic to the optical appearance of the textile, and facilitates fabric arrangement by non‐expert operators. To this end, we propose a sim‐to‐real strategy to train a learning‐based framework that can take as input one or multiple images and outputs a full set of mechanical parameters. Thanks to carefully designed data augmentation and transfer learning protocols, our solution generalizes to real images despite being trained only on synthetic data, hence successfully closing the sim‐to‐real loop. Key in our work is to demonstrate that evaluating the regression accuracy based on the similarity at parameter space leads to an inaccurate distances that do not match the human perception. To overcome this, we propose a novel metric for fabric drape similarity that operates on the image domain instead on the parameter space, allowing us to evaluate our estimation within the context of a similarity rank. We show that out metric correlates with human judgments about the perception of drape similarity, and that our model predictions produce perceptually accurate results compared to the ground truth parameters.},
project_kind={webpage}
}
@inproceedings{zhao2023learninganchor,
title={Learning Anchor Transformations for 3D Garment Animation},
author={Fang Zhao and Zekun Li and Shaoli Huang and Junwu Weng and Tianfei Zhou and Guo-Sen Xie and Jue Wang and Ying Shan},
booktitle={2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2023},
category={cloth, neural_solver, avatar},
doi={10.1109/cvpr52729.2023.00055},
link={https://doi.org/10.1109/cvpr52729.2023.00055},
project={https://semanticdh.github.io/AnchorDEF},
openalex_id={W4386071679},
abstract={This paper proposes an anchor-based deformation model, namely AnchorDEF, to predict 3D garment animation from a body motion sequence. It deforms a garment mesh template by a mixture of rigid transformations with extra nonlinear displacements. A set of anchors around the mesh surface is introduced to guide the learning of rigid transformation matrices. Once the anchor transformations are found, per-vertex nonlinear displacements of the garment template can be regressed in a canonical space, which reduces the complexity of deformation space learning. By explicitly constraining the transformed anchors to satisfy the consistencies of position, normal and direction, the physical meaning of learned anchor transformations in space is guaranteed for better generalization. Furthermore, an adaptive anchor updating is proposed to optimize the anchor position by being aware of local mesh topology for learning representative anchor transformations. Qualitative and quantitative experiments on different types of garments demonstrate that AnchorDEF achieves the state-of-the-art performance on 3D garment deformation prediction in motion, especially for loose-fitting garments.},
project_kind={project_page}
}
@article{li2023swingarspectrum,
title={SwinGar: Spectrum-Inspired Neural Dynamic Deformation for Free-Swinging Garments},
author={Tianxing Li and Rui Shi and Qing Zhu and Takashi Kanai},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2023},
category={cloth, neural_solver},
doi={10.1109/tvcg.2023.3346055},
link={https://doi.org/10.1109/tvcg.2023.3346055},
openalex_id={W4390120214},
abstract={Our work presents a novel spectrum-inspired learning-based approach for generating clothing deformations with dynamic effects and personalized details. Existing methods in the field of clothing animation are limited to either static behavior or specific network models for individual garments, which hinders their applicability in real-world scenarios where diverse animated garments are required. Our proposed method overcomes these limitations by providing a unified framework that predicts dynamic behavior for different garments with arbitrary topology and looseness, resulting in versatile and realistic deformations. First, we observe that the problem of bias towards low frequency always hampers supervised learning and leads to overly smooth deformations. To address this issue, we introduce a frequency-control strategy from a spectral perspective that enhances the generation of high-frequency details of the deformation. In addition, to make the network highly generalizable and able to learn various clothing deformations effectively, we propose a spectral descriptor to achieve a generalized description of the global shape information. Building on the above strategies, we develop a dynamic clothing deformation estimator that integrates graph attention mechanisms with long short-term memory. The estimator takes as input expressive features from garments and human bodies, allowing it to automatically output continuous deformations for diverse clothing types, independent of mesh topology or vertex count. Finally, we present a neural collision handling method to further enhance the realism of garments. Our experimental results demonstrate the effectiveness of our approach on a variety of free-swinging garments and its superiority over state-of-the-art methods.}
}
@inproceedings{shao2023towardsmulti,
title={Towards Multi-Layered 3D Garments Animation},
author={Yidi Shao and Chen Change Loy and Bo Dai},
booktitle={2023 IEEE/CVF International Conference on Computer Vision (ICCV)},
year={2023},
category={cloth, neural_solver, contact},
doi={10.1109/iccv51070.2023.01321},
link={https://doi.org/10.1109/iccv51070.2023.01321},
project={https://mmlab-ntu.github.io/project/layersnet},
openalex_id={W4390872908},
abstract={Mimicking realistic dynamics in 3D garment animations is a challenging task due to the complex nature of multi-layered garments and the variety of outer forces involved. Existing approaches mostly focus on single-layered garments driven by only human bodies and struggle to handle general scenarios. In this paper, we propose a novel data-driven method, called LayersNet, to model garment-level animations as particle-wise interactions in a micro physics system. We improve simulation efficiency by representing garments as patch-level particles in a two-level structural hierarchy. Moreover, we introduce a novel Rotation Equivalent Transformation with Rotation Invariant Attention that leverage the rotation invariance and additivity of physics systems to better model outer forces. To verify the effectiveness of our approach and bridge the gap between experimental environments and real-world scenarios, we introduce a new challenging dataset, D-LAYERS, containing 700K frames of dynamics of 4,900 combinations of multi-layered garments driven by human bodies and randomly sampled wind. Our LayersNet achieves superior performance both quantitatively and qualitatively. Project page: www.mmlab-ntu.com/project/layersnet/index.html.},
project_kind={project_page}
}
@inproceedings{lee2023aquariumfully,
title={Aquarium: A Fully Differentiable Fluid-Structure Interaction Solver for Robotics Applications},
author={Jeong Hun Lee and Mike Y. Michelis and Robert K. Katzschmann and Zachary Manchester},
booktitle={2023 IEEE International Conference on Robotics and Automation (ICRA)},
year={2023},
category={multiphys,differentiable_simulation,embodied_ai},
tag={underwater_robot},
doi={10.1109/icra48891.2023.10161494},
link={https://doi.org/10.1109/icra48891.2023.10161494},
project={https://github.com/RoboticExplorationLab/Aquarium.jl},
openalex_id={W4383066557},
abstract={We present Aquarium, a differentiable fluid-structure interaction solver for robotics that offers stable simulation, accurately coupled fluid-robot physics in two dimensions, and full differentiability with respect to fluid and robot states and parameters. Aquarium achieves stable simulation with accurate flow physics by directly integrating over the incompressible Navier-Stokes equations using a fully implicit Crank-Nicolson scheme with a second-order finite-volume spa-tial discretization. The fluid and robot physics are coupled using the immersed-boundary method by formulating the no-slip condition as an equality constraint applied directly to the Navier-Stokes system. This choice of coupling allows the fluid-structure interaction to be posed and solved as a nonlinear optimization problem. This optimization-based formulation is then exploited using the implicit-function theorem to compute derivatives. Derivatives can then be passed to downstream gradient-based optimization or learning algorithms. We demon-strate Aquarium's ability to accurately simulate coupled fluid-robot physics with numerous 2D examples, including a cylinder in free stream and a soft robotic fish tail with hardware validation. We also demonstrate Aquarium's ability to provide analytical gradients by performing gradient-based shape-and-gait optimization of an oscillating diamond foil to maximize its generated thrust.},
project_kind={code_repo}
}
@inproceedings{mayr2023boundarygraph,
title={Boundary Graph Neural Networks for 3D Simulations},
author={Andreas Mayr and Sebastian Lehner and Arno Mayrhofer and Christoph Kloss and Sepp Hochreiter and J. Brandstetter},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
year={2023},
category={fluid, neural_solver},
tag={gnn},
doi={10.1609/aaai.v37i8.26092},
link={https://doi.org/10.1609/aaai.v37i8.26092},
project={https://ml-jku.github.io/bgnn},
openalex_id={W3175078562},
abstract={The abundance of data has given machine learning considerable momentum in natural sciences and engineering, though modeling of physical processes is often difficult. A particularly tough problem is the efficient representation of geometric boundaries. Triangularized geometric boundaries are well understood and ubiquitous in engineering applications. However, it is notoriously difficult to integrate them into machine learning approaches due to their heterogeneity with respect to size and orientation. In this work, we introduce an effective theory to model particle-boundary interactions, which leads to our new Boundary Graph Neural Networks (BGNNs) that dynamically modify graph structures to obey boundary conditions. The new BGNNs are tested on complex 3D granular flow processes of hoppers, rotating drums and mixers, which are all standard components of modern industrial machinery but still have complicated geometry. BGNNs are evaluated in terms of computational efficiency as well as prediction accuracy of particle flows and mixing entropies. BGNNs are able to accurately reproduce 3D granular flows within simulation uncertainties over hundreds of thousands of simulation timesteps. Most notably, in our experiments, particles stay within the geometric objects without using handcrafted conditions or restrictions.},
project_kind={project_page}
}
@article{rosset2023interactivedesign,
title={Interactive design of 2D car profiles with aerodynamic feedback},
author={Nicolas Rosset and Guillaume Cordonnier and Régis Duvigneau and Adrien Bousseau},
journal={Computer Graphics Forum},
year={2023},
category={fluid,neural_solver,neural_representation},
tag={shape_design,user_interaction},
doi={10.1111/cgf.14772},
link={https://doi.org/10.1111/cgf.14772},
project={https://hal.science/hal-03975369},
openalex_id={W4327888675},
abstract={Abstract The design of car shapes requires a delicate balance between aesthetic and performance. While fluid simulation provides the means to evaluate the aerodynamic performance of a given shape, its computational cost hinders its usage during the early explorative phases of design, when aesthetic is decided upon. We present an interactive system to assist designers in creating aerodynamic car profiles. Our system relies on a neural surrogate model to predict fluid flow around car shapes, providing fluid visualization and shape optimization feedback to designers as soon as they sketch a car profile. Compared to prior work that focused on time‐averaged fluid flows, we describe how to train our model on instantaneous, synchronized observations extracted from multiple pre‐computed simulations, such that we can visualize and optimize for dynamic flow features, such as vortices. Furthermore, we architectured our model to support gradient‐based shape optimization within a learned latent space of car profiles. In addition to regularizing the optimization process, this latent space and an associated encoder‐decoder allows us to input and output car profiles in a bitmap form, without any explicit parameterization of the car boundary. Finally, we designed our model to support pointwise queries of fluid properties around car shapes, allowing us to adapt computational cost to application needs. As an illustration, we only query our model along streamlines for flow visualization, we query it in the vicinity of the car for drag optimization, and we query it behind the car for vortex attenuation.},
project_kind={webpage}
}
@inproceedings{huang2023defgraspnetsgrasp,
title={DefGraspNets: Grasp Planning on 3D Fields with Graph Neural Nets},
author={Isabella Huang and Yashraj Narang and Růžena Bajcsy and Fábio Ramos and Tucker Hermans and Dieter Fox},
booktitle={2023 IEEE International Conference on Robotics and Automation (ICRA)},
year={2023},
category={softbody,differentiable_simulation,control,embodied_ai},
tag={gnn},
doi={10.1109/icra48891.2023.10160986},
link={https://doi.org/10.1109/icra48891.2023.10160986},
project={https://research.nvidia.com/publication/2023-05_defgraspnets-grasp-planning-3d-fields-graph-neural-nets},
openalex_id={W4383097520},
abstract={Robotic grasping of 3D deformable objects is critical for real-world applications such as food handling and robotic surgery. Unlike rigid and articulated objects, 3D deformable objects have infinite degrees of freedom. Fully defining their state requires 3D deformation and stress fields, which are exceptionally difficult to analytically compute or experimentally measure. Thus, evaluating grasp candidates for grasp planning typically requires accurate, but slow 3D finite element method (FEM) simulation. Sampling-based grasp planning is often impractical, as it requires evaluation of a large number of grasp candidates. Gradient-based grasp planning can be more efficient, but requires a differentiable model to synthesize optimal grasps from initial candidates. Differentiable FEM simulators may fill this role, but are typically no faster than standard FEM. In this work, we propose learning a predictive graph neural network (GNN), DefGraspNets, to act as our differentiable model. We train DefGraspNets to predict 3D stress and deformation fields based on FEM-based grasp simulations. DefGraspNets not only runs up to 1500x faster than the FEM simulator, but also enables fast gradient-based grasp optimization over 3D stress and deformation metrics. We design DefGraspNets to align with real-world grasp planning practices and demonstrate generalization across multiple test sets, including real-world experiments.},
project_kind={webpage}
}
@inproceedings{lee2023differentiabledynamics,
title={Differentiable Dynamics Simulation Using Invariant Contact Mapping and Damped Contact Force},
author={Minji Lee and Jeong Min Lee and Dongjun Lee},
booktitle={2023 IEEE International Conference on Robotics and Automation (ICRA)},
year={2023},
category={rigidbody,differentiable_simulation,contact},
doi={10.1109/icra48891.2023.10161519},
link={https://doi.org/10.1109/icra48891.2023.10161519},
openalex_id={W4383108951},
abstract={The gradient of typical differentiable simulation is uninformative for two reasons: 1) non-smoothness in contact dynamics not considered properly, and 2) excessive local minima generated from the smoothing procedure. To tackle this issue, we first propose differentiable contact dynamics with an invariant contact set and coordinate differentiation using a signed distance function (SDF). Also, to eliminate the undesirable jittering caused by the smoothing procedure, which induces extra local minima, and to achieve a smooth and informative gradient, we further endow our framework with a novel damped contact model. Various optimization problems are implemented to demonstrate the usefulness and efficacy of our differentiable framework.}
}
@inproceedings{wang2023dynamicresolution,
title={Dynamic-Resolution Model Learning for Object Pile Manipulation},
author={Yixuan Wang and Yunzhu Li and Katherine Driggs-Campbell and Li Fei-Fei and Jiajun Wu},
booktitle={Robotics: Science and Systems XIX},
year={2023},
category={rigidbody,neural_solver,control,embodied_ai},
tag={robot_control},
doi={10.15607/rss.2023.xix.047},
link={https://doi.org/10.15607/rss.2023.xix.047},
project={https://github.com/WangYixuan12/dyn-res-pile-manip},
openalex_id={W4385430450},
project_kind={code_repo},
abstract={Dynamics models learned from visual observations have shown to be effective in various robotic manipulation tasks. One of the key questions for learning such dynamics models is what scene representation to use. Prior works typically assume representation at a fixed dimension or resolution, which may be inefficient for simple tasks and ineffective for more complicated tasks. In this work, we investigate how to learn dynamic and adaptive representations at different levels of abstraction to achieve the optimal trade-off between efficiency and effectiveness. Specifically, we construct dynamic-resolution particle representations of the environment and learn a unified dynamics model using graph neural networks (GNNs) that allows continuous selection of the abstraction level. During test time, the agent can adaptively determine the optimal resolution at each model-predictive control (MPC) step. We evaluate our method in object pile manipulation, a task we commonly encounter in cooking, agriculture, manufacturing, and pharmaceutical applications. Through comprehensive evaluations both in the simulation and the real world, we show that our method achieves significantly better performance than state-of-the-art fixed-resolution baselines at the gathering, sorting, and redistribution of granular object piles made with various instances like coffee beans, almonds, corn, etc.}
}
@inproceedings{longhini2023elasticcontext,
title={Elastic Context: Encoding Elasticity for Data-driven Models of Textiles Elastic Context: Encoding Elasticity for Data-driven Models of Textiles},
author={Alberta Longhini and Marco Moletta and Alfredo Reichlin and Michael C. Welle and Alexander Kravberg and Yufei Wang and David Held and Zackory Erickson and Danica Kragić},
booktitle={2023 IEEE International Conference on Robotics and Automation (ICRA)},
year={2023},
category={softbody, reconstruction, embodied_ai},
doi={10.1109/icra48891.2023.10160740},
link={https://doi.org/10.1109/icra48891.2023.10160740},
openalex_id={W4383066164},
abstract={Physical interaction with textiles, such as assistive dressing or household tasks, requires advanced dexterous skills. The complexity of textile behavior during stretching and pulling is influenced by the material properties of the yarn and by the textile's construction technique, which are often unknown in real-world settings. Moreover, identification of physical properties of textiles through sensing commonly available on robotic platforms remains an open problem. To address this, we introduce Elastic Context (EC), a method to encode the elasticity of textiles using stress-strain curves adapted from textile engineering for robotic applications. We employ EC to learn generalized elastic behaviors of textiles and examine the effect of EC dimension on accurate force modeling of real-world non-linear elastic behaviors.}
}
@inproceedings{turpin2023fastgrasp,
title={Fast-Grasp'D: Dexterous Multi-finger Grasp Generation Through Differentiable Simulation},
author={Dylan Turpin and Tao Zhong and Shutong Zhang and Guanglei Zhu and Eric Heiden and Miles Macklin and Stavros Tsogkas and Sven Dickinson and Animesh Garg},
booktitle={2023 IEEE International Conference on Robotics and Automation (ICRA)},
year={2023},
category={rigidbody,differentiable_simulation,embodied_ai,contact},
doi={10.1109/icra48891.2023.10160314},
link={https://doi.org/10.1109/icra48891.2023.10160314},
openalex_id={W4383108955},
abstract={Multi-finger grasping relies on high quality training data, which is hard to obtain: human data is hard to transfer and synthetic data relies on simplifying assumptions that reduce grasp quality. By making grasp simulation differentiable, and contact dynamics amenable to gradient-based optimization, we accelerate the search for high-quality grasps with fewer limiting assumptions. We present Grasp'D-1M: a large-scale dataset for multi-finger robotic grasping, synthesized with Fast-Grasp'D, a novel differentiable grasping simulator. Grasp'D-1M contains one million training examples for three robotic hands (three, four and five-fingered), each with multimodal visual inputs (RGB+depth+segmentation, available in mono and stereo). Grasp synthesis with Fast-Grasp'D is 10x faster than GraspIt! [1] and 20x faster than the prior Grasp'D differentiable simulator [2]. Generated grasps are more stable and contact-rich than GraspIt! grasps, regardless of the distance threshold used for contact generation. We validate the usefulness of our dataset by retraining an existing vision-based grasping pipeline [3] on Grasp'D-1M, and showing a dramatic increase in model performance, predicting grasps with 30% more contact, a 33% higher epsilon metric, and 35% lower simulated displacement. Additional details at fast-graspd.github.io.}
}
@inproceedings{xu2023roboninjalearning,
title={RoboNinja: Learning an Adaptive Cutting Policy for Multi-Material Objects},
author={Zhenjia Xu and Xian Zhou and Xingyu Lin and Cheng Chi and Zhiao Huang and Chuang Gan and Shuran Song},
booktitle={Robotics: Science and Systems XIX},
year={2023},
category={rigidbody,differentiable_simulation,reconstruction,embodied_ai},
tag={real2sim},
doi={10.15607/rss.2023.xix.046},
link={https://doi.org/10.15607/rss.2023.xix.046},
openalex_id={W4385403858},
abstract={We introduce RoboNinja, a learning-based cutting system for multi-material objects (i.e., soft objects with rigid cores such as avocados or mangos).In contrast to prior works using open-loop cutting actions to cut through single-material objects (e.g., slicing a cucumber), RoboNinja aims to remove the soft part of an object while preserving the rigid core, thereby maximizing the yield.To achieve this, our system closes the perceptionaction loop by utilizing an interactive state estimator and an adaptive cutting policy.The system first employs sparse collision information to iteratively estimate the position and geometry of an object's core and then generates closed-loop cutting actions based on the estimated state and a tolerance value.The "adaptiveness" of the policy is achieved through the tolerance value, which modulates the policy's conservativeness when encountering collisions, maintaining an adaptive safety distance from the estimated core.Learning such cutting skills directly on a realworld robot is challenging.Yet, existing simulators are limited in simulating multi-material objects or computing the energy consumption during the cutting process.To address this issue, we develop a differentiable cutting simulator that supports multimaterial coupling and allows for the generation of optimized trajectories as demonstrations for policy learning.Furthermore, by using a low-cost force sensor to capture collision feedback, we were able to successfully deploy the learned model in real-world scenarios, including objects with diverse core geometries and soft materials.}
}
@inproceedings{lv2023samrl,
title={SAM-RL: Sensing-Aware Model-Based Reinforcement Learning via Differentiable Physics-Based Simulation and Rendering},
author={Jun Lv and Yunhai Feng and Cheng Zhang and Shuang Zhao and Lin Shao and Cewu Lu},
booktitle={Robotics: Science and Systems XIX},
year={2023},
category={softbody,differentiable_simulation,embodied_ai},
tag={reinforce_learning},
doi={10.15607/rss.2023.xix.040},
link={https://doi.org/10.15607/rss.2023.xix.040},
openalex_id={W4385430519},
abstract={Model-based reinforcement learning (MBRL) is recognized with the potential to be significantly more sample efficient than model-free RL.How an accurate model can be developed automatically and efficiently from raw sensory inputs (such as images), especially for complex environments and tasks, is a challenging problem that hinders the broad application of MBRL in the real world.In this work, we propose a sensingaware model-based reinforcement learning system called SAM-RL.Leveraging the differentiable physics-based simulation and rendering, SAM-RL automatically updates the model by comparing rendered images with real raw images and produces the policy efficiently.With the sensing-aware learning pipeline, SAM-RL allows a robot to select an informative viewpoint to monitor the task process.We apply our framework to real world experiments for accomplishing three manipulation tasks: robotic assembly, tool manipulation, and deformable object manipulation.We demonstrate the effectiveness of SAM-RL via extensive experiments.}
}
@article{tang2023beyondchainmail,
title={Beyond Chainmail: Computational Modeling of Discrete Interlocking Materials},
author={Pengbin Tang and Stelian Coros and Bernhard Thomaszewski},
journal={ACM Transactions on Graphics},
year={2023},
category={softbody, contact},
doi={10.1145/3592112},
link={https://doi.org/10.1145/3592112},
openalex_id={W4385320774},
abstract={We present a method for computational modeling, mechanical characterization, and macro-scale simulation of discrete interlocking materials (DIM)---3D-printed chainmail fabrics made of quasi-rigid interlocking elements. Unlike conventional elastic materials for which deformation and restoring force are directly coupled, the mechanics of DIM are governed by contacts between individual elements that give rise to anisotropic deformation constraints. To model the mechanical behavior of these materials, we propose a computational approach that builds on three key components. ( a ): we explore the space of feasible deformations using native-scale simulations at the per-element level. ( b ): based on this simulation data, we introduce the concept of strain-space boundaries to represent deformation limits for in- and out-of-plane deformations, and ( c ): we use the strain-space boundaries to drive an efficient macro-scale simulation model based on homogenized deformation constraints. We evaluate our method on a set of representative discrete interlocking materials and validate our findings against measurements on physical prototypes.}
}
@inproceedings{grigorev2024contourcraftlearning,
title={ContourCraft: Learning to Resolve Intersections in Neural Multi-Garment Simulations},
author={Artur Grigorev and Giorgio Becherini and Michael J. Black and Otmar Hilliges and Bernhard Thomaszewski},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2024},
category={cloth, neural_solver, contact},
tag={gnn},
doi={10.1145/3641519.3657408},
link={https://doi.org/10.1145/3641519.3657408},
openalex_id={W4396987717},
abstract={Learning-based approaches to cloth simulation have started to show their potential in recent years. However, handling collisions and intersections in neural simulations remains a largely unsolved problem. In this work, we present ContourCraft, a learning-based solution for handling intersections in neural cloth simulations. Unlike conventional approaches that critically rely on intersection-free inputs, ContourCraft robustly recovers from intersections introduced through missed collisions, self-penetrating bodies, or errors in manually designed multi-layer outfits. The technical core of ContourCraft is a novel intersection contour loss that penalizes interpenetrations and encourages rapid resolution thereof. We integrate our intersection loss with a collision-avoiding repulsion objective into a neural cloth simulation method based on graph neural networks (GNNs). We demonstrate our method’s ability across a challenging set of diverse multi-layer outfits under dynamic human motions. Our extensive analysis indicates that ContourCraft significantly improves collision handling for learned simulation and produces visually compelling results.}
}
@inproceedings{li2024efficientdeformation,
title={Efficient Deformation Learning of Varied Garments with a Structure-Preserving Multilevel Framework},
author={Tianxing Li and Rui Shi and Zihui Li and Takashi Kanai and Qing Zhu},
booktitle={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
year={2024},
category={cloth, neural_solver},
tag={gnn},
doi={10.1145/3651286},
link={https://doi.org/10.1145/3651286},
project={https://li-tianxing.github.io/publication/psdunet},
openalex_id={W4396853002},
abstract={Due to the highly nonlinear behavior of clothing, modelling fine-scale garment deformation on arbitrary meshes under varied conditions within a unified network poses a significant challenge. Existing methods often compromise on either model generalization, deformation quality, or runtime speed, making them less suitable for real-world applications. To address it, we propose to incorporate multi-source graph construction and pooling to achieve a novel graph learning scheme. We first introduce methods for extracting cues from different deformation correlations and transform the garment mesh into a comprehensive graph enriched with deformation-related information. To enhance the learning capability and generalizability of the model, we present structure-preserving pooling and unpooling strategies for the mesh deformation task, thereby improving information propagation across the mesh and enhancing the realism of deformation. Lastly, we conduct an attribution analysis and visualize the contribution of various vertices in the graph to the output, providing insights into the deformation behavior. The experimental results demonstrate superior performance against state-of-the-art methods.},
project_kind={project_page}
}
@article{ju2024estimatingcloth,
title={Estimating Cloth Simulation Parameters From Tag Information and Cusick Drape Test},
author={Eunjung Ju and Kwang‐yun Kim and S. W. Yoon and Eungjune Shim and Gyeongjin Kang and Phil Sik Chang and Myung Geol Choi},
journal={Computer Graphics Forum},
year={2024},
category={cloth,reconstruction},
tag={cloth_reconstruction},
doi={10.1111/cgf.15027},
link={https://doi.org/10.1111/cgf.15027},
project={https://mingry.github.io/Fabrics5k},
openalex_id={W4396512560},
abstract={Abstract In recent years, the fashion apparel industry has been increasingly employing virtual simulations for the development of new products. The first step in virtual garment simulation involves identifying the optimal simulation parameters that accurately reproduce the drape properties of the actual fabric. Recent techniques advocate for a data‐driven approach, estimating parameters from outcomes of a Cusick drape test. Such methods deviate from standard Cusick drape tests, introducing high‐cost tools, which reduces practicality. Our research presents a more practical model, utilizing 2D silhouette images from the ISO‐standardized Cusick drape test. Notably, while past models have shown limitations in estimating stretching parameters, our novel approach leverages the fabric's tag information including fabric type and fiber composition. Our proposed model functions as a cascaded system: first, it estimates stretching parameters using tag information, then, in the subsequent step, it considers the estimated stretching parameters alongside the fabric sample's Cusick drape test results to determine bending parameters. We validated our model against existing methods and applied it in practical scenarios, showing promising outcomes. (see https://www.acm.org/publications/class-2012 )},
project_kind={project_page}
}
@article{wang2024garmentanimation,
title={Garment Animation NeRF with Color Editing},
author={Renke Wang and Meng Zhang and Jun Li and Jian Yang},
journal={Computer Graphics Forum},
year={2024},
category={cloth,reconstruction},
tag={cloth_reconstruction,user_interaction,nerf},
doi={10.1111/cgf.15178},
link={https://doi.org/10.1111/cgf.15178},
project={https://mengzephyr.com/Garment-Animation-NeRF-With-Color-Editing},
openalex_id={W4403249593},
abstract={Abstract Generating high‐fidelity garment animations through traditional workflows, from modeling to rendering, is both tedious and expensive. These workflows often require repetitive steps in response to updates in character motion, rendering viewpoint changes, or appearance edits. Although recent neural rendering offers an efficient solution for computationally intensive processes, it struggles with rendering complex garment animations containing fine wrinkle details and realistic garment‐and‐body occlusions, while maintaining structural consistency across frames and dense view rendering. In this paper, we propose a novel approach to directly synthesize garment animations from body motion sequences without the need for an explicit garment proxy. Our approach infers garment dynamic features from body motion, providing a preliminary overview of garment structure. Simultaneously, we capture detailed features from synthesized reference images of the garment's front and back, generated by a pre‐trained image model. These features are then used to construct a neural radiance field that renders the garment animation video. Additionally, our technique enables garment recoloring by decomposing its visual elements. We demonstrate the generalizability of our method across unseen body motions and camera views, ensuring detailed structural consistency. Furthermore, we showcase its applicability to color editing on both real and synthetic garment data. Compared to existing neural rendering techniques, our method exhibits qualitative and quantitative improvements in garment dynamics and wrinkle detail modeling. Code is available at https://github.com/wrk226/GarmentAnimationNeRF .},
project_kind={webpage}
}
@inproceedings{zhang2024neuralgarment,
title={Neural Garment Dynamic Super-Resolution},
author={Meng Zhang and Jun Li},
booktitle={SIGGRAPH Asia 2024 Conference Papers},
year={2024},
category={cloth},
tag={super_resolution},
doi={10.1145/3680528.3687610},
link={https://doi.org/10.1145/3680528.3687610},
project={https://mengzephyr.com/Neural-Garment-Dynamic-Super-Reslution},
openalex_id={W4404965530},
abstract={shapes, motions, and garment types not present in the training data.We demonstrate significant improvements over state-of-the-art alternatives, particularly in enhancing the quality of high-frequency, fine-grained wrinkle details.Code and data is released in https://github.com/MengZephyr/Neural-Garment-},
project_kind={webpage}
}
@article{li2024neuralgarment,
title={Neural Garment Dynamics via Manifold‐Aware Transformers},
author={Peizhuo Li and Tuanfeng Y. Wang and Timur Levent Kesdogan and Duygu Ceylan and Olga Sorkine‐Hornung},
journal={Computer Graphics Forum},
year={2024},
category={cloth, neural_solver, contact, avatar},
tag={transformer},
doi={10.1111/cgf.15028},
link={https://doi.org/10.1111/cgf.15028},
project={https://github.com/PeizhuoLi/manifold-aware-transformers},
openalex_id={W4396512549},
abstract={Abstract Data driven and learning based solutions for modeling dynamic garments have significantly advanced, especially in the context of digital humans. However, existing approaches often focus on modeling garments with respect to a fixed parametric human body model and are limited to garment geometries that were seen during training. In this work, we take a different approach and model the dynamics of a garment by exploiting its local interactions with the underlying human body. Specifically, as the body moves, we detect local garment‐body collisions, which drive the deformation of the garment. At the core of our approach is a mesh‐agnostic garment representation and a manifold‐aware transformer network design, which together enable our method to generalize to unseen garment and body geometries. We evaluate our approach on a wide variety of garment types and motion sequences and provide competitive qualitative and quantitative results with respect to the state of the art.},
project_kind={code_repo}
}
@article{chen2024parametriclinear,
title={Parametric Linear Blend Skinning Model for Multiple-Shape 3D Garments},
author={X. Chen and Guangrun Wang and Xiaogang Xu and Philip H. S. Torr and Liang Lin},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2024},
category={cloth, avatar},
doi={10.1109/tvcg.2024.3478852},
link={https://doi.org/10.1109/tvcg.2024.3478852},
project={https://www.sysu-hcp.net/projects/cv/126.html},
openalex_id={W4403534387},
abstract={We present a novel data-driven Parametric Linear Blend Skinning (PLBS) model meticulously crafted for generalized 3D garment dressing and animation. Previous data-driven methods are impeded by certain challenges including overreliance on human body modeling and limited adaptability across different garment shapes. Our method resolves these challenges via two goals: 1) Develop a model based on garment modeling rather than human body modeling. 2) Separately construct low-dimensional sub-spaces for modeling in-plane deformation (such as variation in garment shape and size) and out-of-plane deformation (such as deformation due to varied body size and motion). Therefore, we formulate garment deformation as a PLBS model controlled by canonical 3D garment mesh, vertex-based skinning weights and associated local patch transformation. Unlike traditional LBS models specialized for individual objects, PLBS model is capable of uniformly expressing varied garments and bodies, the in-plane deformation is encoded on the canonical 3D garment and the out-of-plane deformation is controlled by the local patch transformation. Besides, we propose novel 3D garment registration and skinning weight decomposition strategies to obtain adequate data to build PLBS model under different garment categories. Furthermore, we employ dynamic fine-tuning to complement high-frequency signals missing from LBS for unseen testing data. Experiments illustrate that our method is capable of modeling dynamics for loose-fitting garments, outperforming previous data-driven modeling methods using different sub-space modeling strategies. We showcase that our method can factorize and be generalized for varied body sizes, garment shapes, garment sizes and human motions under different garment categories.},
project_kind={webpage}
}
@inproceedings{stotko2024physicsguided,
title={Physics-guided Shape-from-Template: Monocular Video Perception through Neural Surrogate Models},
author={David Stotko and Nils Wandel and Reinhard Klein},
booktitle={2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2024},
category={cloth,neural_solver,reconstruction},
tag={real2sim,cloth_reconstruction},
doi={10.1109/cvpr52733.2024.01130},
link={https://doi.org/10.1109/cvpr52733.2024.01130},
project={https://github.com/vc-bonn/Physics-guided-Shape-from-Template},
openalex_id={W4402727387},
abstract={3D reconstruction of dynamic scenes is a long-standing problem in computer graphics and increasingly difficult the less information is available. Shape-from-Template (SfT) methods aim to reconstruct a template-based geometry from RGB images or video sequences, often leveraging just a single monocular camera without depth information, such as regular smartphone recordings. Unfortunately, existing reconstruction methods are either unphysical and noisy or slow in optimization. To solve this problem, we propose a novel SfT reconstruction algorithm for cloth using a pre-trained neural surrogate model that is fast to evaluate, stable, and produces smooth reconstructions due to a regularizing physics simulation. Differentiable rendering of the simulated mesh enables pixel-wise comparisons between the reconstruction and a target video sequence that can be used for a gradient-based optimization procedure to extract not only shape information but also physical parameters such as stretching, shearing, or bending stiffness of the cloth. This allows to retain a precise, stable, and smooth reconstructed geometry while reducing the runtime by a factor of 400–500 compared to ϕ-SfT, a state-of-the-art physics-based SfT approach.},
project_kind={code_repo}
}
@article{cen2024realtime,
title={A Real-Time and Interactive Fluid Modeling System for Mixed Reality},
author={Yunchi Cen and Hanchen Deng and Yue Ma and Xiaohui Liang},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2024},
category={fluid,differentiable_simulation},
tag={user_interaction},
doi={10.1109/tvcg.2024.3456140},
link={https://doi.org/10.1109/tvcg.2024.3456140},
project={https://github.com/cenyc/Interactive-Fluid-Modeling},
openalex_id={W4402351307},
abstract={Within the realm of mixed reality, the capability to dynamically render environmental effects with high realism plays a crucial role in amplifying user engagement and interaction. Fluid dynamics, in particular, stand out as essential elements for crafting immersive virtual settings. This includes the simulation of phenomena like smoke, fire, and clouds, which are instrumental in enriching the virtual experience. This work showcases a cutting-edge system developed to produce dynamic and interactive fluid effects that mirror real captured data in real-time for mixed reality applications. This innovative system seamlessly incorporates fluid reconstruction alongside velocity estimation processes within the Unity engine environment. Our approach leverages a novel physics-based differentiable rendering technique, grounded in the principles of light transport in participating media, to simulate the intricate behaviors of fluid while ensuring high fidelity in visual appearance. To further enhance realism, we have expanded our framework to include the estimation of velocity fields, addressing the critical need for fluid motion simulation. The practical application of these techniques demonstrates the system's capacity to offer a robust platform for fluid modeling in mixed reality environments. Through extensive evaluations, we illustrate the effectiveness of our approach in various scenes, underscoring its potential to transform mixed reality content creation by providing developers with the tools to incorporate highly realistic and interactive fluid seamlessly.},
project_kind={code_repo}
}
@inproceedings{qiu2024neusmokeefficient,
title={NeuSmoke: Efficient Smoke Reconstruction and View Synthesis with Neural Transportation Fields},
author={Jiaxiong Qiu and Ruihong Cen and Zhong Li and Han Yan and Ming‐Ming Cheng and Bo Ren},
booktitle={SIGGRAPH Asia 2024 Conference Papers},
year={2024},
category={fluid,reconstruction},
tag={fluid_reconstruction},
doi={10.1145/3680528.3687667},
link={https://doi.org/10.1145/3680528.3687667},
project={https://github.com/JiaxiongQ/NeuSmoke},
openalex_id={W4404966498},
project_kind={code_repo}
}
@inproceedings{wang2024physicsinformed,
title={Physics-Informed Learning of Characteristic Trajectories for Smoke Reconstruction},
author={Yiming Wang and Siyu Tang and Mengyu Chu},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2024},
category={fluid,differentiable_simulation,neural_representation,reconstruction},
tag={fluid_reconstruction,nerf},
doi={10.1145/3641519.3657483},
link={https://doi.org/10.1145/3641519.3657483},
project={https://github.com/19reborn/PICT_smoke},
openalex_id={W4400573511},
abstract={We delve into the physics-informed neural reconstruction of smoke and obstacles through sparse-view RGB videos, tackling challenges arising from limited observation of complex dynamics. Existing physics-informed neural networks often emphasize short-term physics constraints, leaving the proper preservation of long-term conservation less explored. We introduce Neural Characteristic Trajectory Fields, a novel representation utilizing Eulerian neural fields to implicitly model Lagrangian fluid trajectories. This topology-free, auto-differentiable representation facilitates efficient flow map calculations between arbitrary frames as well as efficient velocity extraction via auto-differentiation. Consequently, it enables end-to-end supervision covering long-term conservation and short-term physics priors. Building on the representation, we propose physics-informed trajectory learning and integration into NeRF-based scene reconstruction. We enable advanced obstacle handling through self-supervised scene decomposition and seamless integrated boundary constraints. Our results showcase the ability to overcome challenges like occlusion uncertainty, density-color ambiguity, and static-dynamic entanglements. Code and sample tests are at https://github.com/19reborn/PICT_smoke.},
project_kind={code_repo}
}
@article{zhao2024reconstructionimplicit,
title={Reconstruction of implicit surfaces from fluid particles using convolutional neural networks},
author={Chunming Zhao and Tamar Shinar and Craig Schroeder},
journal={Computer Graphics Forum},
year={2024},
category={fluid,reconstruction},
tag={fluid_reconstruction,cnn},
doi={10.1111/cgf.15181},
link={https://doi.org/10.1111/cgf.15181},
project={https://www.cs.ucr.edu/~craigs/research.html},
openalex_id={W4403249663},
abstract={Abstract In this paper, we present a novel network‐based approach for reconstructing signed distance functions from fluid particles. The method uses a weighting kernel to transfer particles to a regular grid, which forms the input to a convolutional neural network. We propose a regression‐based regularization to reduce surface noise without penalizing high‐curvature features. The reconstruction exhibits improved spatial surface smoothness and temporal coherence compared with existing state of the art surface reconstruction methods. The method is insensitive to particle sampling density and robustly handles thin features, isolated particles, and sharp edges.},
project_kind={webpage}
}
@inproceedings{choi2024snnpde,
title={SNN-PDE: Learning Dynamic PDEs from Data with Simplicial Neural Networks},
author={Jae Woong Choi and Yuzhou Chen and Huikyo Lee and Hyun Kim and Yulia R. Gel},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
year={2024},
category={fluid, neural_solver},
tag={gnn},
doi={10.1609/aaai.v38i10.29038},
link={http://dx.doi.org/10.1609/aaai.v38i10.29038},
openalex_id={W4393147400},
abstract={Dynamics of many complex systems, from weather and climate to spread of infectious diseases, can be described by partial differential equations (PDEs). Such PDEs involve unknown function(s), partial derivatives, and typically multiple independent variables. The traditional numerical methods for solving PDEs assume that the data are observed on a regular grid. However, in many applications, for example, weather and air pollution monitoring delivered by the arbitrary located weather stations of the National Weather Services, data records are irregularly spaced. Furthermore, in problems involving prediction analytics such as forecasting wildfire smoke plumes, the primary focus may be on a set of irregular locations associated with urban development. In recent years, deep learning (DL) methods and, in particular, graph neural networks (GNNs) have emerged as a new promising tool that can complement traditional PDE solvers in scenarios of the irregular spaced data, contributing to the newest research trend of physics informed machine learning (PIML). However, most existing PIML methods tend to be limited in their ability to describe higher dimensional structural properties exhibited by real world phenomena, especially, ones that live on manifolds. To address this fundamental challenge, we bring the elements of the Hodge theory and, in particular, simplicial convolution defined on the Hodge Laplacian to the emerging nexus of DL and PDEs. In contrast to conventional Laplacian and the associated convolution operation, the simplicial convolution allows us to rigorously describe diffusion across higher order structures and to better approximate the complex underlying topology and geometry of the data. The new approach, Simplicial Neural Networks for Partial Differential Equations (SNN PDE) offers a computationally efficient yet effective solution for time dependent PDEs. Our studies of a broad range of synthetic data and wildfire processes demonstrate that SNN PDE improves upon state of the art baselines in handling unstructured grids and irregular time intervals of complex physical systems and offers competitive forecasting capabilities for weather and air quality forecasting.}
}
@inproceedings{seker2024estimatingmaterial,
title={Estimating Material Properties of Interacting Objects Using Sum-GP-UCB},
author={M. Yunus Seker and Oliver Kroemer},
booktitle={2024 IEEE International Conference on Robotics and Automation (ICRA)},
year={2024},
category={rigidbody,reconstruction},
tag={real2sim,material_estimation},
doi={10.1109/icra57147.2024.10610129},
link={https://doi.org/10.1109/icra57147.2024.10610129},
project={https://myunusseker.github.io/SumGP},
openalex_id={W4401415778},
abstract={Robots need to estimate the material and dynamic properties of objects from observations in order to simulate them accurately. We present a Bayesian optimization approach to identifying the material property parameters of objects based on a set of observations. Our focus is on estimating these properties based on observations of scenes with different sets of interacting objects. We propose an approach that exploits the structure of the reward function by modeling the reward for each observation separately and using only the parameters of the objects in that scene as inputs. The resulting lower-dimensional models generalize better over the parameter space, which in turn results in a faster optimization. To speed up the optimization process further, and reduce the number of simulation runs needed to find good parameter values, we also propose partial evaluations of the reward function, wherein the selected parameters are only evaluated on a subset of real world evaluations. The approach was successfully evaluated on a set of scenes with a wide range of object interactions, and we showed that our method can effectively perform incremental learning without resetting the rewards of the gathered observations.},
project_kind={project_page}
}
@inproceedings{gjoka2024softpneumatic,
title={Soft Pneumatic Actuator Design using Differentiable Simulation},
author={Arvi Gjoka and Espen Knoop and Moritz Bächer and Denis Zorin and Daniele Panozzo},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers 24},
year={2024},
category={rigidbody,differentiable_simulation,optimization,embodied_ai},
tag={shape_design},
doi={10.1145/3641519.3657467},
link={https://doi.org/10.1145/3641519.3657467},
project={https://la.disneyresearch.com/publication/soft-pneumatic-actuator-design-using-differentiable-simulation},
openalex_id={W4400573493},
abstract={We propose a computational design pipeline for pneumatically-actuated soft robots interacting with their environment through contact. We optimize the shape of the robot with a shape optimization approach, using a physically-accurate high-order finite element model for the forward simulation. Our approach enables fine-grained control over both deformation and contact forces by optimizing the shape of internal cavities, which we exploit to design pneumatically-actuated robots that can assume user-prescribed poses, or apply user-controlled forces. We demonstrate the efficacy of our method on two artistic and two functional examples.},
project_kind={webpage}
}
@inproceedings{jin2024diffsounddifferentiable,
title={DiffSound: Differentiable Modal Sound Rendering and Inverse Rendering for Diverse Inference Tasks},
author={Xutong Jin and Chenxi Xu and Ruohan Gao and Jiajun Wu and Guoping Wang and Sheng Li},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2024},
category={softbody,differentiable_simulation,reconstruction,sound},
tag={real2sim,parameter_estimation},
doi={10.1145/3641519.3657493},
link={https://doi.org/10.1145/3641519.3657493},
project={https://hellojxt.github.io/DiffSound/},
openalex_id={W4400582185},
abstract={Accurately estimating and simulating the physical properties of objects from real-world sound recordings is of great practical importance in the fields of vision, graphics, and robotics. However, the progress in these directions has been limited—prior differentiable rigid or soft body simulation techniques cannot be directly applied to modal sound synthesis due to the high sampling rate of audio, while previous audio synthesizers often do not fully model the accurate physical properties of the sounding objects. We propose DiffSound, a differentiable sound rendering framework for physics-based modal sound synthesis, which is based on an implicit shape representation, a new high-order finite element analysis module, and a differentiable audio synthesizer. Our framework can solve a wide range of inverse problems thanks to the differentiability of the entire pipeline, including physical parameter estimation, geometric shape reasoning, and impact position prediction. Experimental results demonstrate the effectiveness of our approach, highlighting its ability to accurately reproduce the target sound in a physics-based manner. DiffSound serves as a valuable tool for various sound synthesis and analysis applications.},
project_kind={project_page}
}
@inproceedings{wang2024neuralmodes,
title={Neural Modes: Self-supervised Learning of Nonlinear Modal Subspaces},
author={Jiahong Wang and Yinwei Du and Stelian Coros and Bernhard Thomaszewski},
booktitle={2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2024},
category={softbody,neural_representation},
doi={10.1109/cvpr52733.2024.02185},
link={https://doi.org/10.1109/cvpr52733.2024.02185},
project={https://github.com/jiahong-w/neural-modes},
openalex_id={W4402703151},
abstract={We propose a self-supervised approach for learning physics-based subspaces for real-time simulation. Existing learning-based methods construct subspaces by approximating pre-defined simulation data in a purely geometric way. However; this approach tends to produce highenergy configurations, leads to entangled latent space dimensions, and generalizes poorly beyond the training set. To overcome these limitations, we propose a self-supervised approach that directly minimizes the system's mechanical energy during training. We show that our method leads to learned subspaces that reflect physical equilibrium constraints, resolve overfitting issues of previous methods, and offer interpretable latent space parameters.},
project_kind={code_repo}
}
@inproceedings{chen2024realtime,
title={Real-time Wing Deformation Simulations for Flying Insects},
author={Qiang Chen and Zhigang Deng and Feng Li and Yuming Fang and Tingsong Lu and Yang Tong and Yifan Zuo},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2024},
category={softbody, control},
doi={10.1145/3641519.3657434},
link={https://doi.org/10.1145/3641519.3657434},
project={https://graphics.cs.uh.edu/article/2024/2640/2024-siggraph-insectwingdeformation/},
openalex_id={W4400582037},
abstract={Realistic simulation of the intricate wing deformations seen in flying insects not only deepens our comprehension of insect flight mechanics but also opens up numerous applications in fields such as computer animation and virtual reality. Despite its importance, this research area has been relatively underexplored due to the complex and diverse wing structures and the intricate patterns of deformation. This paper presents an efficient skeleton-driven model specifically designed to real-time simulate realistic wing deformations across a wide range of flying insects. Our approach begins with the construction of a virtual skeleton that accurately reflects the distinct morphological characteristics of individual insect species. This skeleton serves as the foundation for the simulation of the intricate deformation wave propagation often observed in wing deformations. To faithfully reproduce the bending effect seen in these deformations, we introduce both internal and external forces that act on the wing joints, drawing on periodic wing-beat motion and a simplified aerodynamics model. Additionally, we utilize mass-spring algorithms to simulate the inherent elasticity of the wings, helping to prevent excessive twisting. Through various simulation experiments, comparisons, and user studies, we demonstrate the effectiveness, robustness, and adaptability of our model.},
project_kind={webpage}
}
@inproceedings{chen2025dressanyone,
title={Dress Anyone : Automatic Physically-Based Garment Pattern Refitting 56},
author={Hsiao-yu Chen and Egor Larionov and Ladislav Kavan and Gene Wei-Chin Lin and Doug Roble and Olga Sorkine‐Hornung and Tuur Stuyck},
booktitle={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
year={2025},
category={cloth, differentiable_simulation, optimization, avatar},
doi={10.1145/3747858},
link={https://doi.org/10.1145/3747858},
project={https://igl.ethz.ch/projects/dress_anyone},
openalex_id={W4413244246},
abstract={Well-fitted clothing is essential for both real and virtual garments to enable self-expression and accurate representation for a large variety of body types. Common practice in the industry is to provide a pre-made selection of distinct garment sizes such as small, medium and large. While these may cater to certain groups of individuals that fall within this distribution, they often exclude large sections of the population. In contrast, individually tailored clothing offers a solution to obtain custom-fit garments that are tailored to each individual. However, manual tailoring is time-consuming and requires specialized knowledge, prohibiting the approach from being applied to produce fitted clothing at scale. To address this challenge, we propose a novel method leveraging differentiable simulation for refitting and draping 3D garments and their corresponding 2D pattern panels onto a new body shape. This enables a workflow where garments only need to be designed once, in a single size, and they can be automatically refitted to support numerous body size and shape variations. Our method enables downstream applications, where our optimized 3D drape can be directly ingested into game engines or other applications. Our 2D sewing patterns allow for accurate physics-based simulations and enables manufacturing clothing for the real world.},
project_kind={webpage}
}
@article{li2025dress1,
title={Dress-1-to-3: Single Image to Simulation-Ready 3D Outfit with Diffusion Prior and Differentiable Physics},
author={Xuan Li and Chang Yu and Wenxin Du and Ying Jiang and Tianyi Xie and Yunuo Chen and Yin Yang and Chenfanfu Jiang},
journal={ACM Transactions on Graphics},
year={2025},
category={cloth, differentiable_simulation, reconstruction, avatar},
tag={real2sim},
doi={10.1145/3731177},
link={https://doi.org/10.1145/3731177},
project={https://dress-1-to-3.github.io/},
openalex_id={W4412673739},
abstract={Recent advances in large models have significantly advanced image-to-3D reconstruction. However, the generated models are often fused into a single piece, limiting their applicability in downstream tasks. This paper focuses on 3D garment generation, a key area for applications like virtual try-on with dynamic garment animations, which require garments to be separable and simulation-ready. We introduce Dress-1-to-3, a novel pipeline that reconstructs physics-plausible, simulation-ready separated garments with sewing patterns and humans from an in-the-wild image. Starting with the image, our approach combines a pre-trained image-to-sewing pattern generation model for creating coarse sewing patterns with a pre-trained multi-view diffusion model to produce multi-view images. The sewing pattern is further refined using a differentiable garment simulator based on the generated multi-view images. Versatile experiments demonstrate that our optimization approach substantially enhances the geometric alignment of the reconstructed 3D garments and humans with the input image. Furthermore, by integrating a texture generation module and a human motion generation module, we produce customized physics-plausible and realistic dynamic garment demonstrations. Our project page is https://dress-1-to-3.github.io/.}
}
@article{li2025frequencydivided,
title={Frequency-Divided Learning of Fine-Grained Clothing Behavior via Flexible Dynamic Graphs},
author={Tianxing Li and Rui Shi and Takashi Kanai and Qing Zhu},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2025},
category={cloth, neural_solver},
tag={gnn},
doi={10.1109/tvcg.2025.3591816},
link={https://doi.org/10.1109/tvcg.2025.3591816},
project={https://shirui-homepage.com/publication/2025-freq-div-TVCG},
openalex_id={W4412605474},
abstract={Despite significant advancements in neural simulation techniques for clothing animation, these methods struggle to capture the dynamic details of garments during movement. This limitation restricts their applicability in scenarios where high-quality garment deformation is essential. To address this challenge, we introduce a novel graph learning-based approach to enhance deformation realism through designed mechanisms for mesh information propagation and external optimization strategies during model training. First, we address the issue of over-smoothing common in conventional graph processing techniques by introducing a flexible message-passing method. This approach effectively manages node interactions within the mesh, thereby improving the expressiveness of the model. Furthermore, acknowledging that uniform model supervision typically neglects high-frequency details during optimization, we analyze the spectral properties of clothing meshes. Based on this analysis, we introduce a frequency-division constraint aligned with the characteristics of different frequency bands, which aids in precisely controlling the generation of details. Our model further integrates self-collision and other physics-aware losses, enabling the learning of generalized and fine-grained dynamic deformations. Extensive evaluations and comparisons demonstrate the effectiveness of our approach, showing notable improvements over existing state-of-the-art solutions.},
project_kind={webpage}
}
@article{peng2025picaphysics,
title={PICA: Physics-Integrated Clothed Avatar},
author={Bo Peng and Yunfan Tao and Haoyu Zhan and Yudong Guo and Juyong Zhang},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2025},
category={cloth,avatar},
tag={3dgs,gnn},
doi={10.1109/tvcg.2025.3614642},
link={https://doi.org/10.1109/tvcg.2025.3560241},
project={https://ustc3dv.github.io/PICA},
openalex_id={W4414538549},
abstract={We introduce PICA, a novel representation for high-fidelity animatable clothed human avatars with physics-plausible dynamics, even for loose clothing. Previous neural rendering-based representations of animatable clothed humans typically employ a single model to represent both the clothing and the underlying body. While efficient, these approaches often fail to represent complex garment dynamics, leading to incorrect deformations and noticeable rendering artifacts, especially for sliding or loose garments. Furthermore, most previous works represent garment dynamics as pose-dependent deformations and facilitate novel pose animations in a data-driven manner. This often results in outcomes that do not faithfully represent the mechanics of motion and are prone to generating artifacts in out-of-distribution poses. To address these issues, we employ two individual 2D Gaussian Splatting (2DGS) models with different deformation characteristics, modeling the human body and clothing separately. This distinction allows for better handling of their respective motion characteristics. With this representation, we integrate a graph neural network (GNN)-based clothing physics simulation module to ensure a better representation of clothing dynamics. Our method, through its carefully designed features, achieves high-fidelity rendering of clothed human bodies in complex and novel driving poses, outperforming previous methods under the same settings.},
project_kind={project_page}
}
@article{shi2025selfsupervised,
title={Self‐Supervised Humidity‐Controllable Garment Simulation via Capillary Bridge Modeling},
author={Min Shi and Xinyuan Wang and J. Zhang and Lin Gao and Dengming Zhu and Hui Zhang},
journal={Computer Graphics Forum},
year={2025},
category={cloth, neural_solver, contact, avatar},
tag={gnn},
doi={10.1111/cgf.70236},
link={https://doi.org/10.1111/cgf.70236},
openalex_id={W4415077393},
abstract={Abstract Simulating wet clothing remains a significant challenge due to the complex physical interactions between moist fabric and the human body, compounded by the lack of dedicated datasets for training data‐driven models. Existing self‐supervised approaches struggle to capture moisture‐induced dynamics such as skin adhesion, anisotropic surface resistance, and non‐linear wrinkling, leading to limited accuracy and efficiency. To address this, we present SHGS, a novel self‐supervised framework for humidity‐controllable clothing simulation grounded in the physical modeling of capillary bridges that form between fabric and skin. We abstract the forces induced by wetness into two physically motivated components: a normal adhesive force derived from Laplace pressure and a tangential shear‐resistance force that opposes relative motion along the fabric surface. By formulating these forces as potential energy for conservative effects and as mechanical work for non‐conservative effects, we construct a physics‐consistent wetness loss. This enables self‐supervised training without requiring labeled data of wet clothing. Our humidity‐sensitive dynamics are driven by a multi‐layer graph neural network, which facilitates a smooth and physically realistic transition between different moisture levels. This architecture decouples the garment's dynamics in wet and dry states through a local weight interpolation mechanism, adjusting the fabric's behavior in response to varying humidity conditions. Experiments demonstrate that SHGS outperforms existing methods in both visual fidelity and computational efficiency, marking a significant advancement in realistic wet‐cloth simulation.}
}
@article{chen2025neuralparticle,
title={A Neural Particle Level Set Method for Dynamic Interface Tracking},
author={Duowen Chen and Junwei Zhou and Bo Zhu},
journal={ACM Transactions on Graphics},
year={2025},
category={fluid, neural_representation},
doi={10.1145/3730399},
link={https://doi.org/10.1145/3730399},
project={https://cdwj.github.io/projects/neural-pls-project-page/index.html},
openalex_id={W4409623878},
abstract={We propose a neural particle level set (Neural PLS) method to accommodate tracking and evolving dynamic neural representations. At the heart of our approach is a set of oriented particles serving dual roles of interface trackers and sampling seeders. These dynamic particles are used to evolve the interface and construct neural representations on a multi-resolution grid-hash structure to hybridize coarse sparse distance fields and multi-scale feature encoding. Based on these parallel implementations and neural-network-friendly architectures, our neural particle level set method combines the computational merits on both ends of the traditional particle level sets and the modern implicit neural representations, in terms of feature representation and dynamic tracking. We demonstrate the efficacy of our approach by showcasing its performance surpassing traditional level-set methods in both benchmark tests and physical simulations.},
project_kind={project_page}
}
@inproceedings{yu2025pioneeringneural,
title={A Pioneering Neural Network Method for Efficient and Robust Fuel Sloshing Simulation in Aircraft},
author={Chen Yu and Shuai Zheng and Nianyi Wang and Menglong Jin and Yan Chang},
booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
year={2025},
category={fluid, neural_solver},
doi={10.1609/aaai.v39i15.33752},
link={https://doi.org/10.1609/aaai.v39i15.33752},
project={https://github.com/chenyu-xjtu/A-Pioneering-Neural-Network-Method-for-Efficient-and-Robust-Fuel-Sloshing-Simulation-in-Aircraft},
openalex_id={W4409347271},
abstract={Simulating fuel sloshing within aircraft tanks during flight is crucial for aircraft safety research. Traditional methods based on Navier-Stokes equations are computationally expensive. In this paper, we treat fluid motion as point cloud transformation and propose the first neural network method specifically designed for simulating fuel sloshing in aircraft. This model is also the first deep learning model capable of stably modeling fluid particle dynamics in such complex scenarios. Our triangle feature fusion design achieves an optimal balance among fluid dynamics modeling, momentum conservation constraints, and global stability control. Additionally, we constructed the Fueltank dataset, the first dataset for aircraft fuel surface sloshing. It comprises 320,000 frames across four typical tank types and covers a wide range of flight maneuvers, including multi-directional rotations. We conducted comprehensive experiments on both our dataset and the take-off scenario of the aircraft. Compared to existing neural network-based fluid simulation algorithms, we significantly enhanced accuracy while maintaining high computational speed. Compared to traditional SPH methods, our speed improved approximately 10 times. Furthermore, compared to traditional fluid simulation software such as Flow3D, our computation speed increased by more than 300 times.},
project_kind={code_repo}
}
@inproceedings{ni2025representingflow,
title={Representing Flow Fields with Divergence-Free Kernels for Reconstruction},
author={Xingyu Ni and Jingrui Xing and X. Li and Bin Wang and Baoquan Chen},
booktitle={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
year={2025},
category={fluid,reconstruction},
tag={fluid_reconstruction,super_resolution},
doi={10.1145/3747872},
link={https://doi.org/10.1145/3747872},
project={https://www.physicsbasedanimation.com/2025/08/10/representing-flow-fields-with-divergence-free-kernels-for-reconstruction},
openalex_id={W4413244297},
abstract={Accurately reconstructing continuous flow fields from sparse or indirect measurements remains an open challenge, as existing techniques often suffer from oversmoothing artifacts, reliance on heterogeneous architectures, and the computational burden of enforcing physics-informed losses in implicit neural representations (INRs). In this paper, we introduce a novel flow field reconstruction framework based on divergence-free kernels (DFKs), which inherently enforce incompressibility while capturing fine structures without relying on hierarchical or heterogeneous representations. Through qualitative analysis and quantitative ablation studies, we identify the matrix-valued radial basis functions derived from Wendland’s \\(\\mathcal \{C\}^4\\) polynomial (DFKs-Wen4) as the optimal form of analytically divergence-free approximation for velocity fields, owing to their favorable numerical properties, including compact support, positive definiteness, and second-order differentiability. Experiments across various reconstruction tasks, spanning data compression, inpainting, super-resolution, and time-continuous flow inference, have demonstrated that DFKs-Wen4 outperform INRs and other divergence-free representations in both reconstruction accuracy and computational efficiency, while requiring the fewest trainable parameters.},
project_kind={webpage}
}
@article{holz2025multiphysicssimulation,
title={Multiphysics Simulation Methods in Computer Graphics},
author={Daniel Holz and Stefan Rhys Jeske and Fabian Löschner and Jan Bender and Yin Yang and Sheldon Andrews},
journal={Computer Graphics Forum},
year={2025},
category={multiphys, survey},
doi={10.1111/cgf.70082},
link={https://doi.org/10.1111/cgf.70082},
project={https://multi.physics-simulation.org/},
openalex_id={W4409586884},
abstract={Abstract Physics simulation is a cornerstone of many computer graphics applications, ranging from video games and virtual reality to visual effects and computational design. The number of techniques for physically‐based modeling and animation has thus skyrocketed over the past few decades, facilitating the simulation of a wide variety of materials and physical phenomena. This report captures the state‐of‐the‐art of multiphysics simulation for computer graphics applications. Although a lot of work has focused on simulating individual phenomena, here we put an emphasis on methods developed by the computer graphics community for simulating various physical phenomena and materials, as well as the interactions between them. These include combinations of discretization schemes, mathematical modeling frameworks, and coupling techniques. For the most commonly used methods we provide an overview of the state‐of‐the‐art and deliver valuable insights into the various approaches. A selection of software frameworks that offer out‐of‐the‐box multiphysics modeling capabilities is also presented. Finally, we touch on emerging trends in physics‐based animation that affect multiphysics simulation, including machine learning‐based methods which have become increasingly popular in recent years.},
project_kind={webpage}
}
@article{xu2025differentiablematerial,
title={A Differentiable Material Point Method Framework for Shape Morphing},
author={Min Xu and Chongmin Song and David I. W. Levin and David Hyde},
journal={IEEE Transactions on Visualization and Computer Graphics},
year={2025},
category={softbody,differentiable_simulation,engine},
doi={10.1109/tvcg.2025.3591729},
link={https://doi.org/10.1109/tvcg.2025.3591729},
project={https://chayo.oopy.io/1e7e3760-68e9-808e-b30a-f89ba08d6193},
openalex_id={W4412567585},
abstract={We present a novel, physically-based morphing technique for elastic shapes, leveraging the differentiable material point method (MPM) with space-time control through per-particle deformation gradients to accommodate complex topology changes. This approach, grounded in MPM's natural handling of dynamic topologies, is enhanced by a chained iterative optimization technique, allowing for the creation of both succinct and extended morphing sequences that maintain coherence over time. Demonstrated across various challenging scenarios, our method is able to produce detailed elastic deformation and topology transitions, all grounded within our physics-based simulation framework.},
project_kind={webpage}
}
@inproceedings{chen2025learningobject,
title={Learning Object Properties Using Robot Proprioception via Differentiable Robot-Object Interaction},
author={Peter Yichen Chen and Lijun Liu and Pingchuan Ma and J. J. Eastman and Daniela Rus and Dylan Randle and Yuri Ivanov and Wojciech Matusik},
booktitle={2025 IEEE International Conference on Robotics and Automation (ICRA)},
year={2025},
category={softbody,differentiable_simulation,reconstruction,embodied_ai},
tag={real2sim,parameter_estimation},
doi={10.1109/icra55743.2025.11127955},
link={https://doi.org/10.1109/icra55743.2025.11127955},
project={https://warpdiffrobot.github.io/},
openalex_id={W4413944832},
abstract={Differentiable simulation has become a powerful tool for system identification. While prior work has focused on identifying robot properties using robot-specific data or object properties using object-specific data, our approach calibrates object properties by using information from the robot, without relying on data from the object itself. Specifically, we utilize robot joint encoder information, which is commonly available in standard robotic systems. Our key observation is that by analyzing the robot's reactions to manipulated objects, we can infer properties of those objects, such as inertia and softness. Leveraging this insight, we develop differentiable simulations of robot-object interactions to inversely identify the properties of the manipulated objects. Our approach relies solely on proprioception – the robot's internal sensing capabilities – and does not require external measurement tools or vision-based tracking systems. This general method is applicable to any articulated robot and requires only joint position information. We demonstrate the effectiveness of our method on a low-cost robotic platform, achieving accurate mass and elastic modulus estimations of manipulated objects with just a few seconds of computation on a laptop.},
project_kind={project_page}
}
@article{romanyserrasolsas2025painlessdifferentiable,
title={Painless Differentiable Rotation Dynamics},
author={Magí Romanyà-Serrasolsas and Juan J. Casafranca and Miguel Á. Otaduy},
journal={ACM Transactions on Graphics},
year={2025},
category={rigidbody, differentiable_simulation},
doi={10.1145/3730944},
link={https://doi.org/10.1145/3730944},
project={https://mslab.es/projects/Painless/},
openalex_id={W4412673705},
abstract={We propose the formulation of forward and differentiable rigid-body dynamics using Lie-algebra rotation derivatives. In particular, we show how this approach can easily be applied to incremental-potential formulations of forward dymamics, and we introduce a novel definition of adjoints for differentiable dynamics. In contrast to other parameterizations of rotations (notably the popular rotation-vector parameterization), our approach leads to painlessly simple and compact derivatives, better conditioning, and higher runtime efficiency. We demonstrate our approach on fundamental rigid-body problems, but also on Cosserat rods as an example of multi-rigid-body dynamics.},
project_kind={webpage}
}
@inproceedings{ruan2025profrobot,
title={Prof. Robot: Differentiable Robot Rendering Without Static and Self-Collisions},
author={Qi Ruan and Jiabao Lei and Wenhao Yuan and Yanglin Zhang and Dekun Lu and Guiliang Liu and Kui Jia},
booktitle={2025 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2025},
category={rigidbody,reconstruction,embodied_ai},
doi={10.1109/cvpr52734.2025.02101},
link={https://doi.org/10.1109/cvpr52734.2025.02101},
openalex_id={W4413144620},
abstract={Differentiable rendering has gained significant attention in the field of robotics, with differentiable robot rendering emerging as an effective paradigm for learning robotic actions from image-space supervision. However, the lack of physical world perception in this approach may lead to potential collisions during action optimization. In this work, we introduce a novel improvement on previous efforts by incorporating physical awareness of collisions through the learning of a neural robotic collision classifier. This enables the optimization of actions that avoid collisions with static, non-interactable environments as well as the robot itself. To facilitate effective gradient optimization with the classifier, we identify the underlying issue and propose leveraging Eikonal regularization to ensure consistent gradients for optimization. Our solution can be seamlessly integrated into existing differentiable robot rendering frameworks, utilizing gradients for optimization and providing a foundation for future applications of differentiable rendering in robotics with improved reliability of interactions with the physical world. Both qualitative and quantitative experiments demonstrate the necessity and effectiveness of our method compared to previous solutions.}
}
@inproceedings{shen2025elasticlocomotion,
title={Elastic Locomotion with Mixed Second-order Differentiation},
author={Siyuan Shen and Tianjia Shao and Kun Zhou and Chenfanfu Jiang and Sheldon Andrews and Victor Zordan and Yin Yang},
booktitle={Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2025},
category={softbody, differentiable_simulation, control, embodied_ai},
doi={10.1145/3721238.3730685},
link={https://doi.org/10.1145/3721238.3730685},
openalex_id={W4412596298}
}
@inproceedings{tang2025inversedesign,
title={Inverse Design of Discrete Interlocking Materials with Desired Mechanical Behavior},
author={Pengbin Tang and Bernhard Thomaszewski and Stelian Coros and Bernd Bickel},
booktitle={Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2025},
category={softbody,optimization},
tag={shape_design},
doi={10.1145/3721238.3730675},
link={https://doi.org/10.1145/3721238.3730675},
project={https://tangpengbin.github.io/publications/InverseDIM/index.html},
openalex_id={W4412587883},
project_kind={project_page}
}
@inproceedings{liu2025precisegradient,
title={Precise Gradient Discontinuities in Neural Fields for Subspace Physics},
author={Mengfei Liu and Yue Chang and Zhecheng Wang and Peter Yichen Chen and Eitan Grinspun},
booktitle={Proceedings of the SIGGRAPH Asia 2025 Conference Papers},
year={2025},
category={softbody,neural_representation},
tag={neural_field},
doi={10.1145/3757377.3763810},
link={https://doi.org/10.1145/3757377.3763810},
project={https://www.dgp.toronto.edu/projects/discont_grad},
openalex_id={W7110006315},
abstract={Discontinuities in spatial derivatives appear in a wide range of physical systems, from creased thin sheets to materials with sharp stiffness transitions. Accurately modeling these features is essential for simulation but remains challenging for traditional mesh-based methods, which require discontinuity-aligned remeshing—entangling geometry with simulation and hindering generalization across shape families.},
project_kind={webpage}
}
@inproceedings{stuyck2025quaffurereal,
title={Quaffure: Real-Time Quasi-Static Neural Hair Simulation},
author={Tuur Stuyck and Gene Wei-Chin Lin and Egor Larionov and Hsiao-yu Chen and Aljaž Božič and Nikolaos Sarafianos and Doug Roble},
booktitle={2025 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2025},
category={softbody,neural_solver,avatar},
tag={hair},
doi={10.1109/cvpr52734.2025.00031},
link={https://doi.org/10.1109/cvpr52734.2025.00031},
project={https://tuurstuyck.github.io/quaffure/quaffure.html},
openalex_id={W4413156022},
abstract={Realistic hair motion is crucial for high-quality avatars, but it is often limited by the computational resources available for real-time applications. To address this challenge, we propose a novel neural approach to predict physically plausible hair deformations that generalizes to various body poses, shapes, and hairstyles. Our model is trained using a self-supervised loss, eliminating the need for expensive data generation and storage. We demonstrate our method’s effectiveness through numerous results across a wide range of pose and shape variations, showcasing its robust generalization capabilities and temporally smooth results. Our approach is highly suitable for real-time applications with an inference time of only a few milliseconds on consumer hardware and its ability to scale to predicting the drape of 1000 grooms in 0.3 seconds.},
project_kind={project_page}
}
@inproceedings{li2025selfsupervised,
title={Self-supervised Learning of Latent Space Dynamics},
author={Yue Li and Gene Wei-Chin Lin and Egor Larionov and Aljaž Božič and Doug Roble and Ladislav Kavan and Stelian Coros and Bernhard Thomaszewski and Tuur Stuyck and Hsiao-yu Chen},
booktitle={Proceedings of the ACM on Computer Graphics and Interactive Techniques},
year={2025},
category={softbody,neural_solver,neural_representation},
doi={10.1145/3747854},
link={https://doi.org/10.1145/3747854},
openalex_id={W4413131481},
abstract={Modeling the dynamic behavior of deformable objects is crucial for creating realistic digital worlds. While conventional simulations produce high-quality motions, their computational costs are often prohibitive. Subspace simulation techniques address this challenge by restricting deformations to a lower-dimensional space, improving performance while maintaining visually compelling results. However, even subspace methods struggle to meet the stringent performance demands of portable devices such as virtual reality headsets and mobile platforms. To overcome this limitation, we introduce a novel subspace simulation framework powered by a neural latent-space integrator. Our approach leverages self-supervised learning to enhance inference stability and generalization. By operating entirely within latent space, our method eliminates the need for full-space computations, resulting in a highly efficient method well-suited for deployment on portable devices. We demonstrate the effectiveness of our approach on challenging examples involving rods, shells, and solids, showcasing its versatility and potential for widespread adoption.}
}
@inproceedings{mittal2025uniphylearning,
title={UniPhy: Learning a Unified Constitutive Model for Inverse Physics Simulation},
author={Himangi Mittal and Peiye Zhuang and Hsin-Ying Lee and Shubham Tulsiani},
booktitle={2025 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2025},
category={fluid,differentiable_simulation},
tag={material_estimation,neural_material},
doi={10.1109/cvpr52734.2025.01511},
link={https://doi.org/10.1109/cvpr52734.2025.01511},
project={https://himangim.github.io/UniPhy},
openalex_id={W4413155956},
abstract={We propose UniPhy, a common latent-conditioned neural constitutive model that can encode the physical properties of diverse materials. At inference UniPhy allows ‘inverse simulation’ i.e. inferring material properties by optimizing the scene-specific latent to match the available observations via differentiable simulation. In contrast to existing methods that treat such inference as system identification, UniPhy does not rely on user-specified material type information. Compared to prior neural constitutive modeling approaches which learn instance specific networks, the shared training across materials improves both, robustness and accuracy of the estimates. We train UniPhy using simulated trajectories across diverse geometries and materials - elastic, plasticine, sand, and fluids (Newtonian & non-Newtonian). At inference, given an object with unknown material properties, UniPhy can infer the material properties via latent optimization to match the motion observations, and can then allow re-simulating the object under diverse scenarios. We compare UniPhy against prior inverse simulation methods, and show that the inference from UniPhy enables more accurate replay and re-simulation under novel conditions.},
project_kind={project_page}
}
@article{lin2026neuralocksreal,
title={Neuralocks: Real‐Time Dynamic Neural Hair Simulation},
author={Gene Wei‐Chin Lin and Egor Larionov and Hsiao‐yu Chen and Doug Roble and Tuur Stuyck},
journal={Computer Graphics Forum},
year={2026},
category={softbody,neural_solver,avatar},
tag={hair},
doi={10.1111/cgf.70407},
link={https://doi.org/10.1111/cgf.70407},
openalex_id={W7143446781},
abstract={Abstract Real‐time hair simulation is a vital component in creating believable virtual avatars, as it provides a sense of immersion and authenticity. The dynamic behavior of hair, such as bouncing or swaying in response to character movements like jumping or walking, plays a significant role in enhancing the overall realism and engagement of virtual experiences. Current methods for simulating hair have been constrained by two primary approaches: highly optimized physics‐based systems and neural methods. However, state‐of‐the‐art neural techniques have been limited to quasi‐static solutions, failing to capture the dynamic behavior of hair. This paper introduces a novel neural method that breaks through these limitations, achieving efficient and stable dynamic hair simulation while outperforming existing approaches. We propose a fully self‐supervised method which can be trained without any manual intervention or artist generated training data allowing the method to be integrated with hair reconstruction methods to enable automatic end‐to‐end methods for avatar reconstruction. Our approach harnesses the power of compact, memory‐efficient neural networks to simulate hair at the strand level, allowing for the simulation of diverse hairstyles without excessive computational resources or memory requirements. We validate the effectiveness of our method through a variety of hairstyle examples, showcasing its potential for real‐world applications.}
}
@inproceedings{jain2024neuralmonte,
title={Neural Monte Carlo Fluid Simulation},
author={Pranav Jain and Ziyin Qu and Peter Yichen Chen and Oded Stein},
booktitle={Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers},
year={2024},
category={fluid,differentiable_simulation,neural_representation},
tag={neural_field},
doi={10.1145/3641519.3657438},
link={https://doi.org/10.1145/3641519.3657438},
project={https://github.com/Pranav-Jain/Neural-Monte-Carlo-Fluid-Simulation},
openalex_id={W4400580579},
abstract={The idea of using a neural network to represent continuous vector fields (i.e., neural fields) has become popular for solving PDEs arising from physics simulations. Here, the classical spatial discretization (e.g., finite difference) of PDE solvers is replaced with a neural network that models a differentiable function, so the spatial gradients of the PDEs can be readily computed via autodifferentiation. When used in fluid simulation, however, neural fields fail to capture many important phenomena, such as the vortex shedding experienced in the von Kármán vortex street experiment. We present a novel neural network representation for fluid simulation that augments neural fields with explicitly enforced boundary conditions as well as a Monte Carlo pressure solver to get rid of all weakly enforced boundary conditions. Our method, the Neural Monte Carlo method (NMC), is completely mesh-free, i.e., it doesn’t depend on any grid-based discretization. While NMC does not achieve the state-of-the-art accuracy of the well-established grid-based methods, it significantly outperforms previous mesh-free neural fluid methods on fluid flows involving intricate boundaries and turbulence regimes.},
}
@inproceedings{li2025adjoint,
title={An Adjoint Method for Differentiable Fluid Simulation on Flow Maps},
author={Zhiqi Li and Jinjin He and Barnab{\'a}s B{\"o}rcs{\"o}k and Taiyuan Zhang and Duowen Chen and Tao Du and Ming C. Lin and Greg Turk and Bo Zhu},
booktitle={Proceedings of the SIGGRAPH Asia 2025 Conference Papers},
year={2025},
category={fluid,differentiable_simulation},
doi={10.1145/3757377.3763903},
link={https://arxiv.org/abs/2511.01259},
project={https://pearseven.github.io/DiffFMProject/},
abstract={This paper presents a novel adjoint solver for differentiable fluid simulation based on bidirectional flow maps. Our key observation is that the forward fluid solver and its corresponding backward, adjoint solver share the same flow map as the forward simulation. In the forward pass, this map transports fluid impulse variables from the initial frame to the current frame to simulate vortical dynamics. In the backward pass, the same map propagates adjoint variables from the current frame back to the initial frame to compute gradients. This shared long-range map allows the accuracy of gradient computation to benefit directly from improvements in flow map construction. Building on this insight, we introduce a novel adjoint solver that solves the adjoint equations directly on the flow map, enabling long-range and accurate differentiation of incompressible flows without differentiating intermediate numerical steps or storing intermediate variables, as required in conventional adjoint methods. To further improve efficiency, we propose a long-short time-sparse flow map representation for evolving adjoint variables. Our approach has low memory usage, requiring only 6.53GB of data at a resolution of $192^3$ while preserving high accuracy in tracking vorticity, enabling new differentiable simulation tasks that require precise identification, prediction, and control of vortex dynamics.},
}
@article{li2024neuralfluidic,
title={NeuralFluid: Neural Fluidic System Design and Control with Differentiable Simulation},
author={Li, Yifei and Sun, Yuchen and Ma, Pingchuan and Sifakis, Eftychios and Du, Tao and Zhu, Bo and Matusik, Wojciech},
journal={Advances in Neural Information Processing Systems},
volume={37},
year={2024},
category={fluid,differentiable_simulation,neural_solver,control,embodied_ai},
tag={fluid_control},
link={https://papers.nips.cc/paper_files/paper/2024/hash/9a379c1b05793d1c42dc832269834515-Abstract-Conference.html},
project={https://people.csail.mit.edu/liyifei/publication/neuralfluid/},
abstract={We present a novel framework to explore neural control and design of complex fluidic systems with dynamic solid boundaries. Our system features a fast differentiable Navier-Stokes solver with solid-fluid interface handling, a low-dimensional differentiable parametric geometry representation, a control-shape co-design algorithm, and gym-like simulation environments to facilitate various fluidic control design applications. Additionally, we present a benchmark of design, control, and learning tasks on high-fidelity, high-resolution dynamic fluid environments that pose challenges for existing differentiable fluid simulators. These tasks include designing the control of artificial hearts, identifying robotic end-effector shapes, and controlling a fluid gate. By seamlessly incorporating our differentiable fluid simulator into a learning framework, we demonstrate successful design, control, and learning results that surpass gradient-free solutions in these benchmark tasks.},
}
@article{park2024nearrealtime,
title={Near-realtime Facial Animation by Deep 3D Simulation Super-Resolution},
author={Hyojoon Park and Sangeetha Grama Srinivasan and Matthew Cong and Doyub Kim and Byungsoo Kim and Jonathan Swartz and Ken Museth and Eftychios Sifakis},
journal={ACM Transactions on Graphics},
year={2024},
category={softbody,avatar},
tag={super_resolution},
doi={10.1145/3670687},
link={https://doi.org/10.1145/3670687},
project={https://github.com/hjoonpark/3d-sim-super-res},
openalex_id={W4400557967},
abstract={We present a neural network-based simulation super-resolution framework that can efficiently and realistically enhance a facial performance produced by a low-cost, real-time physics-based simulation to a level of detail that closely approximates that of a reference-quality off-line simulator with much higher resolution (27× element count in our examples) and accurate physical modeling. Our approach is rooted in our ability to construct a training set of paired frames, from the low- and high-resolution simulators respectively, that are in semantic correspondence with each other. We use face animation as an exemplar of such a simulation domain, where creating this semantic congruence is achieved by simply dialing in the same muscle actuation controls and skeletal pose in the two simulators. Our proposed neural network super-resolution framework generalizes from this training set to unseen expressions, compensates for modeling discrepancies between the two simulations due to limited resolution or cost-cutting approximations in the real-time variant, and does not require any semantic descriptors or parameters to be provided as input, other than the result of the real-time simulation. We evaluate the efficacy of our pipeline on a variety of expressive performances and provide comparisons and ablation experiments for plausible variations and alternatives to our proposed scheme. Our code is available at https://github.com/hjoonpark/3d-sim-super- res.git.},
}
@inproceedings{xu2025amrtransformer,
title={AMR-Transformer: Enabling Efficient Long-range Interaction for Complex Neural Fluid Simulation},
author={Zeyi Xu and Jinfan Liu and Kuangxu Chen and Ye Chen and Zhangli Hu and Bingbing Ni},
booktitle={2025 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2025},
category={fluid,neural_solver},
tag={transformer},
doi={10.1109/cvpr52734.2025.00545},
link={https://doi.org/10.1109/cvpr52734.2025.00545},
project={https://github.com/JfanLiu/AMR_Transformer},
openalex_id={W4413155346},
abstract={Accurately and efficiently simulating complex fluid dynamics is a challenging task that has traditionally relied on computationally intensive methods. Neural network-based approaches, such as convolutional and graph neural networks, have partially alleviated this burden by enabling efficient local feature extraction. However, they struggle to capture long-range dependencies due to limited receptive fields, and Transformer-based models, while providing global context, incur prohibitive computational costs. To tackle these challenges, we propose AMR-Transformer, an efficient and accurate neural CFD-solving pipeline that integrates a novel adaptive mesh refinement scheme with a Navier-Stokes constraint-aware fast pruning module. This design encourages long-range interactions between simulation cells and facilitates the modeling of global fluid wave patterns, such as turbulence and shockwaves. Experiments show that our approach achieves significant gains in efficiency while preserving critical details, making it suitable for high-resolution physical simulations with long-range dependencies. On CFDBench, PDEBench and a new shock wave dataset, our pipeline demonstrates up to an order-of- magnitude improvement in accuracy over baseline models. Additionally, compared to ViT, our approach achieves a reduction in FLOPs of up to 60 times.},
}
@inproceedings{liu2025neuralkinematicbases,
title={Neural Kinematic Bases for Fluids},
author={Yibo Liu and Zhixin Fang and Sune Darkner and Noam Aigerman and Kenny Erleben and Paul Kry and Teseo Schneider},
booktitle={Proceedings of the SIGGRAPH Asia 2025 Conference Papers},
year={2025},
category={fluid, neural_representation},
doi={10.1145/3757377.3763925},
link={https://arxiv.org/abs/2504.15657},
abstract={We propose mesh-free fluid simulations that exploit a kinematic neural basis for velocity fields represented by an MLP. We design a set of losses that ensures that these neural bases approximate fundamental physical properties such as orthogonality, divergence-free, boundary alignment, and smoothness. Our neural bases can then be used to fit an input sketch of a flow, which will inherit the same fundamental properties from the bases. We then can animate such flow in real-time using standard time integrators. Our neural bases can accommodate different domains, moving boundaries, and naturally extend to three dimensions.},
}
@article{yue2025shapespace,
title={Shape Space Spectra},
author={Chang Yue and Otman Benchekroun and Maurizio M. Chiaramonte and Peter Yichen Chen and Eitan Grinspun},
journal={ACM Transactions on Graphics},
year={2025},
category={softbody,differentiable_simulation,neural_representation,embodied_ai,sound},
tag={shape_design,neural_field},
doi={10.1145/3731148},
link={https://doi.org/10.1145/3731148},
project={https://www.dgp.toronto.edu/projects/sss},
openalex_id={W4412673659},
abstract={Eigenanalysis of differential operators, such as the Laplace operator or elastic energy Hessian, is typically restricted to a single shape and its discretization, limiting reduced order modeling (ROM). We introduce the first eigenanalysis method for continuously parameterized shape families. Given a parametric shape, our method constructs spatial neural fields that represent eigen-functions across the entire shape space. It is agnostic to the specific shape representation, requiring only an inside/outside indicator function that depends on shape parameters. Eigenfunctions are computed by minimizing a variational principle over nested spaces with orthogonality constraints. Since eigenvalues may swap dominance at points of multiplicity, we jointly train multiple eigenfunctions while dynamically reordering them based on their eigenvalues at each step. Through causal gradient filtering , this reordering is reflected in backpropagation. Our method enables applications to operate over shape space, providing a single ROM that encapsulates vibration modes for all shapes, including previously unseen ones. Since our eigenanalysis is differentiable with respect to shape parameters, it facilitates eigenfunction-aware shape optimization. We evaluate our approach on shape optimization for sound synthesis and locomotion, as well as reduced-order modeling for elastodynamic simulation.},
}
@inproceedings{tao2024nirfs,
title={Neural Implicit Reduced Fluid Simulation},
author={Yuanyuan Tao and Ivan Puhachov and Derek Nowrouzezahrai and Paul Kry},
booktitle={Proceedings of the SIGGRAPH Asia 2024 Conference Papers},
year={2024},
category={fluid,neural_solver,neural_representation,control},
tag={fluid_control,neural_field},
doi={10.1145/3680528.3687628},
link={https://doi.org/10.1145/3680528.3687628},
project={https://yuanyuantao.github.io/Neural-Implicit-Reduced-Fluid-Simulation/},
abstract={High-fidelity simulation of fluid dynamics is challenging because of the high dimensional state data needed to capture fine details and the large computational cost associated with advancing the system in time. We present neural implicit reduced fluid simulation (NIRFS), a reduced fluid simulation technique that combines an implicit neural representation of fluid shapes and a neural ordinary differential equation to model the dynamics of fluid in the reduced latent space. The latent trajectories are computed at very little cost in comparison to simulations for training, while preserving fine physical details. We show that this approach can work well, capturing the shapes and dynamics involved in a variety of scenarios with constrained initial conditions, e.g., droplet-droplet collisions, crown splashes, and fluid slosh in a container. In each scenario, we learn the latent implicit representation of fluid shapes with a deep-network signed distance function, as well as the energy function and parameters of a damped Hamiltonian system, which helps guarantee desirable properties of the latent dynamics. To ensure that latent shape representations form smooth and physically meaningful trajectories, we simultaneously learn the latent representation and dynamics. We evaluate novel simulations for conservation of volume and momentum conservation, discuss design decisions, and demonstrate an application of our method to fluid control.},
}
@inproceedings{mani2023surfsup,
title={SurfsUp: Learning Fluid Simulation for Novel Surfaces},
author={Arjun Mani and Ishaan Preetam Chandratreya and Elliot Creager and Carl Vondrick and Richard Zemel},
booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
year={2023},
category={fluid,differentiable_simulation},
doi={10.1109/ICCV51070.2023.01308},
link={https://doi.org/10.1109/ICCV51070.2023.01308},
project={https://surfsup.cs.columbia.edu/},
abstract={Modeling the mechanics of fluid in complex scenes is vital to applications in design, graphics, and robotics. Learning-based methods provide fast and differentiable fluid simulators, however most prior work is unable to accurately model how fluids interact with genuinely novel surfaces not seen during training. We introduce SurfsUp, a framework that represents objects implicitly using signed distance functions (SDFs), rather than an explicit representation of meshes or particles. This continuous representation of geometry enables more accurate simulation of fluid-object interactions over long time periods while simultaneously making computation more efficient. Moreover, SurfsUp trained on simple shape primitives generalizes considerably out-of-distribution, even complex real-world scenes and objects. Finally, we show we can invert our model to design simple objects to manipulate fluid flow.},
}
@inproceedings{jia2025implicitphysics,
title={Learning an Implicit Physical Model for Image-based Fluid Simulation},
author={Emily Yue-ting Jia and Jiageng Mao and Zhiyuan Gao and Yajie Zhao and Yue Wang},
booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
year={2025},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction},
link={https://arxiv.org/abs/2508.08254},
project={https://physfluid.github.io/},
abstract={Humans possess an exceptional ability to visualize 4D scenes, encompassing both motion and 3D geometry, from a single still image. This ability is rooted in our accumulated observations of similar scenes and an intuitive understanding of physics. In this paper, we aim to replicate this capacity in neural networks, specifically focusing on natural fluid imagery. Existing methods for this task typically employ simplistic 2D motion estimators to animate the image, leading to motion predictions that often defy physical principles, resulting in unrealistic animations. Our approach introduces a novel method for generating 4D scenes with physics-consistent animation from a single image. We propose the use of a physics-informed neural network that predicts motion for each point, guided by a loss term derived from fundamental physical principles, including the Navier-Stokes equations. To reconstruct the 3D geometry, we predict feature-based 3D Gaussians from the input image, which are then animated using the predicted motions and rendered from any desired camera perspective. Experimental results highlight the effectiveness of our method in producing physically plausible animations, showcasing significant performance improvements over existing methods.},
}
@inproceedings{lee2024realtimeneuralcloth,
title={Real-Time Neural Cloth Deformation Using a Compact Latent Space and a Latent Vector Predictor},
author={Chanhaeng Lee and Maksym Perepichka and Saeed Ghorbani and Sudhir Mudur and Eric Paquette and Tiberiu Popa},
booktitle={European Conference on Computer Vision (ECCV) Workshops},
year={2024},
category={cloth,neural_representation},
tag={gnn},
doi={10.1007/978-3-031-92387-6_25},
link={https://doi.org/10.1007/978-3-031-92387-6_25},
abstract={We propose a method for real-time cloth deformation using neural networks. The computational overhead of most of the existing learning methods for cloth simulation often limits their use in interactive applications. Employing a two-stage training process, our method predicts garment deformations in real-time. In the first stage, a graph neural network extracts cloth vertex features which are compressed into a latent vector with a mesh convolution network. We then decode the latent vector to blend shape weights, which are fed to a trainable blend shape module. In the second stage, we freeze the latent extraction and train a temporal latent predictor network. The temporal latent predictor uses a subset of the inputs from the first stage, ensuring that inputs are restricted to those which are readily available in a typical game engine. Then, during inference, the latent predictor predicts the compacted latent which is processed by the decoder and blend shape networks from the first stage. The latent predictor is the crucial component to speed up our inference time by replacing the resource-intensive graph neural network from the first stage. Our experiments demonstrate that our method effectively balances computational efficiency and realistic cloth deformation, making it suitable for real-time use in applications such as games.},
}
@article{kairanda2024neuralclothsim,
title={NeuralClothSim: Neural Deformation Fields Meet the Thin Shell Theory},
author={Navami Kairanda and Marc Habermann and Christian Theobalt and Vladislav Golyanik},
journal={Advances in Neural Information Processing Systems},
volume={37},
year={2024},
category={cloth,neural_representation},
tag={neural_field},
link={https://proceedings.neurips.cc/paper_files/paper/2024/hash/c7649eeb93d2fad0ced9a3b974260710-Abstract-Conference.html},
project={https://4dqv.mpi-inf.mpg.de/NeuralClothSim/},
abstract={Despite existing 3D cloth simulators producing realistic results, they predominantly operate on discrete surface representations (e.g. points and meshes) with a fixed spatial resolution, which often leads to large memory consumption and resolution-dependent simulations. Moreover, back-propagating gradients through the existing solvers is difficult, and they hence cannot be easily integrated into modern neural architectures. In response, this paper re-thinks physically plausible cloth simulation: We propose NeuralClothSim, i.e., a new quasistatic cloth simulator using thin shells, in which surface deformation is encoded in neural network weights in the form of a neural field. Our memory-efficient solver operates on a new continuous coordinate-based surface representation called neural deformation fields (NDFs); it supervises NDF equilibria with the laws of the non-linear Kirchhoff-Love shell theory with a non-linear anisotropic material model. NDFs are adaptive: They allocate their capacity to the deformation details and allow surface state queries at arbitrary spatial resolutions without re-training. We show how to train NeuralClothSim while imposing hard boundary conditions and demonstrate multiple applications, such as material interpolation and simulation editing. The experimental results highlight the effectiveness of our continuous neural formulation.},
}
@inproceedings{jatavallabhula2021gradsim,
title={gradSim: Differentiable Simulation for System Identification and Visuomotor Control},
author={Krishna Murthy Jatavallabhula and Miles Macklin and Florian Golemo and Vikram Voleti and Linda Petrini and Martin Weiss and Breandan Considine and Jerome Parent-Levesque and Kevin Xie and Kenny Erleben and Liam Paull and Florian Shkurti and Derek Nowrouzezahrai and Sanja Fidler},
booktitle={International Conference on Learning Representations},
year={2021},
category={cloth,differentiable_simulation,reconstruction},
tag={real2sim,cloth_reconstruction,parameter_estimation},
link={https://openreview.net/forum?id=c_E8kFWfhp0},
project={https://gradsim.github.io/},
abstract={We consider the problem of estimating an object's physical properties such as mass, friction, and elasticity directly from video sequences. Such a system identification problem is fundamentally ill-posed due to the loss of information during image formation. Current solutions require precise 3D labels which are labor-intensive to gather, and infeasible to create for many systems such as deformable solids or cloth. We present gradSim, a framework that overcomes the dependence on 3D supervision by leveraging differentiable multiphysics simulation and differentiable rendering to jointly model the evolution of scene dynamics and image formation. This novel combination enables backpropagation from pixels in a video sequence through to the underlying physical attributes that generated them. Moreover, our unified computation graph spanning from the dynamics and through the rendering process enables learning in challenging visuomotor control tasks, without relying on state-based 3D supervision, while obtaining performance competitive to or better than techniques that rely on precise 3D labels.},
}
@inproceedings{coros2021differentiablesimulationcourse,
title={Differentiable Simulation},
author={Stelian Coros and Miles Macklin and Bernhard Thomaszewski and Nils Th{\"u}rey},
booktitle={SIGGRAPH Asia 2021 Courses},
year={2021},
category={multiphys,differentiable_simulation,contact},
doi={10.1145/3476117.3483433},
link={https://doi.org/10.1145/3476117.3483433},
abstract={Differentiable simulation is emerging as a fundamental building block for many cutting-edge applications in computer graphics, vision and robotics, among others. This course provides an introduction to this topic and an overview of state-of-the-art methods in this context. Starting with the basics of dynamic mechanical systems, we will present a general theoretical framework for differentiable simulation, which we will specialize to rigid bodies, deformable solids, and fluids. A particular focus will be on the different alternatives for computing simulation derivatives, ranging from analytical expressions via sensitivity analysis to reverse-mode automatic differentiation. As an important step towards real-world applications, we also present extensions to non-smooth phenomena such as frictional contact. Finally, we will discuss different ways of integrating differentiable simulation into machine learning frameworks.}
}
@inproceedings{xing2025stabilizing,
title={Stabilizing Reinforcement Learning in Differentiable Multiphysics Simulation},
author={Eliot Xing and Vernon Luk and Jean Oh},
booktitle={International Conference on Learning Representations},
year={2025},
category={multiphys,differentiable_simulation,embodied_ai},
tag={reinforce_learning},
link={https://arxiv.org/abs/2412.12089},
project={https://rewarped.github.io/},
abstract={We present a maximum entropy first-order reinforcement learning algorithm, alongside a parallel differentiable multiphysics simulation platform for reinforcement learning that supports simulating various materials beyond just rigid bodies. Existing first-order reinforcement learning algorithms are prone to suboptimal convergence to local minima in the reward landscape, so we formulate Soft Analytic Policy Optimization (SAPO), a maximum entropy first-order RL algorithm. SAPO uses first-order analytic gradients from differentiable simulation to train a stochastic actor to maximize expected return and entropy. We also introduce Rewarped, a parallel differentiable multiphysics simulation platform that provides GPU-accelerated parallel environments for reinforcement learning and enables computing batched simulation gradients efficiently. Across challenging manipulation and locomotion tasks involving rigid bodies, soft bodies, and fluids, SAPO shows more stable training, improved sample efficiency, and higher task performance than strong baselines.},
}
@misc{newton2025engine,
author={newton-physics contributors},
title={Newton: An Open-Source, GPU-Accelerated Physics Simulation Engine Built upon NVIDIA Warp},
year={2025},
category={rigidbody, embodied_ai, engine},
url={https://github.com/newton-physics/newton},
link={https://github.com/newton-physics/newton},
abstract={An open-source, GPU-accelerated physics simulation engine built upon NVIDIA Warp, specifically targeting roboticists and simulation researchers.}
}
@article{lecleach2023differentiable,
title={Differentiable Physics Simulation of Dynamics-Augmented Neural Objects},
author={Le Cleac'h, Simon and Yu, Hong-Xing and Guo, Michelle and Howell, Taylor A. and Gao, Ruohan and Wu, Jiajun and Manchester, Zachary and Schwager, Mac},
journal={IEEE Robotics and Automation Letters},
volume={8},
number={5},
pages={2780--2787},
year={2023},
category={rigidbody,differentiable_simulation,reconstruction},
tag={real2sim},
doi={10.1109/LRA.2023.3257707},
link={https://doi.org/10.1109/LRA.2023.3257707},
abstract={This paper presents a differentiable simulator for rigid body dynamics with collision handling that simulates dynamic interactions involving dynamics-augmented neural object representations. These representations describe an object's geometry as a continuous density field parameterized as a deep network and capture dynamics-relevant object properties such as friction, compliant contacts, and object shape. By augmenting neural objects with object-specific dynamic information, we obtain dynamics-augmented neural objects (DANOs), which are compatible with differentiable simulation and improve simulation fidelity. We demonstrate DANOs through learning object geometry and physical properties directly from still images and videos, making it possible to improve the fidelity of a simulation of a specific object through real-to-simulation transfer.},
}
@misc{bertiche2021implicit,
title={Neural Implicit Surfaces for Efficient and Accurate Collisions in Physically Based Simulations},
author={Bertiche, Hugo and Madadi, Meysam and Escalera, Sergio},
year={2021},
category={cloth, neural_representation, contact},
eprint={2110.01614},
archivePrefix={arXiv},
primaryClass={cs.GR},
link={https://arxiv.org/abs/2110.01614},
project={https://hbertiche.github.io/NeuralColliders/},
abstract={Current trends in physically based simulation seek to abandon explicit data structures in favor of neural networks. However, one key issue is collisions. Existing approaches use low-resolution geometries to compute collisions, leading to poor dynamics. Here, we use neural networks to model the colliding surfaces and propose Neural Implicit Surfaces for computing collisions using the resulting representations. We show that our approach improves the efficiency and accuracy of physically based simulations, while enabling several applications that are unique to the capabilities of neural networks. Specifically, our method can simulate at different levels of detail, lead to reduced memory footprint, and allow interpolation between collision models, among other features.},
}
@inproceedings{xu2022accelerated,
title={Accelerated Policy Learning with Parallel Differentiable Simulation},
author={Jie Xu and Viktor Makoviychuk and Yashraj Narang and Fábio Ramos and Wojciech Matusik and Animesh Garg and Miles Macklin},
booktitle={Conference on Robot Learning (CoRL)},
year={2022},
category={rigidbody,differentiable_simulation,control,embodied_ai,contact},
tag={robot_control,reinforce_learning},
doi={10.48550/arXiv.2204.07137},
link={https://arxiv.org/abs/2204.07137},
project={https://short-horizon-actor-critic.github.io/},
openalex_id={W4224879709},
abstract={Deep reinforcement learning can generate complex control policies, but requires large amounts of training data to work effectively. Recent work has attempted to address this issue by leveraging differentiable simulators. However, inherent problems such as local minima and exploding/vanishing numerical gradients prevent these methods from being generally applied to control tasks with complex contact-rich dynamics, such as humanoid locomotion in classical RL benchmarks. In this work we present a high-performance differentiable simulator and a new policy learning algorithm (SHAC) that can effectively leverage simulation gradients, even in the presence of non-smoothness. Our learning algorithm alleviates problems with local minima through a smooth critic function, avoids vanishing/exploding gradients through a truncated learning window, and allows many physical environments to be run in parallel. We evaluate our method on classical RL control tasks, and show substantial improvements in sample efficiency and wall-clock time over state-of-the-art RL and differentiable simulation-based algorithms. In addition, we demonstrate the scalability of our method by applying it to the challenging high-dimensional problem of muscle-actuated locomotion with a large action space, achieving a greater than 17x reduction in training time over the best-performing established RL algorithm.},
project_kind={project_page}
}
@inproceedings{lin2022diffskill,
title={DiffSkill: Skill Abstraction from Differentiable Physics for Deformable Object Manipulations with Tools},
author={Xingyu Lin and Zhiao Huang and Yunzhu Li and Joshua B. Tenenbaum and David Held and Chuang Gan},
booktitle={International Conference on Learning Representations (ICLR)},
year={2022},
category={softbody,differentiable_simulation,neural_solver,embodied_ai},
tag={reinforce_learning},
doi={10.48550/arXiv.2203.17275},
link={https://arxiv.org/abs/2203.17275},
project={https://xingyu-lin.github.io/diffskill/},
openalex_id={W4226396459},
abstract={We consider the problem of sequential robotic manipulation of deformable objects using tools. Previous works have shown that differentiable physics simulators provide gradients to the environment state and help trajectory optimization to converge orders of magnitude faster than model-free reinforcement learning algorithms for deformable object manipulation. However, such gradient-based trajectory optimization typically requires access to the full simulator states and can only solve short-horizon, single-skill tasks due to local optima. In this work, we propose a novel framework, named DiffSkill, that uses a differentiable physics simulator for skill abstraction to solve long-horizon deformable object manipulation tasks from sensory observations. In particular, we first obtain short-horizon skills using individual tools from a gradient-based optimizer, using the full state information in a differentiable simulator; we then learn a neural skill abstractor from the demonstration trajectories which takes RGBD images as input. Finally, we plan over the skills by finding the intermediate goals and then solve long-horizon tasks. We show the advantages of our method in a new set of sequential deformable object manipulation tasks compared to previous reinforcement learning algorithms and compared to the trajectory optimizer.},
project_kind={project_page}
}
@inproceedings{liu2022fishgym,
title={FishGym: A High-Performance Physics-based Simulation Framework for Underwater Robot Learning},
author={Wenji Liu and Kai Bai and Xuming He and Shuran Song and Changxi Zheng and Xiaopei Liu},
booktitle={2022 International Conference on Robotics and Automation (ICRA)},
year={2022},
category={multiphys,control,embodied_ai,engine},
tag={underwater_robot,reinforce_learning},
doi={10.1109/ICRA46639.2022.9812066},
link={https://doi.org/10.1109/ICRA46639.2022.9812066},
project={https://github.com/fish-gym/gym-fish},
openalex_id={W4285102606},
abstract={Bionic underwater robots have demonstrated their superiority in many applications. Yet, training their intelligence for a variety of tasks that mimic the behavior of underwater creatures poses a number of challenges in practice, mainly due to lack of a large amount of available training data as well as the high cost in real physical environment. Alternatively, simulation has been considered as a viable and important tool for acquiring datasets in different environments, but it mostly targeted rigid and soft body systems. There is currently dearth of work for more complex fluid systems interacting with immersed solids that can be efficiently and accurately simulated for robot training purposes. In this paper, we propose a new platform called “FishGym”, which can be used to train fish-like underwater robots. The framework consists of a robotic fish modeling module using articulated body with skinning, a GPU-based high-performance localized two-way coupled fluid-structure interaction simulation module that handles both finite and infinitely large domains, as well as a reinforcement learning module. We leveraged existing training methods with adaptations to underwater fish-like robots and obtained learned control policies for multiple benchmark tasks. The training results are demonstrated with reasonable motion trajectories, with comparisons and analyses to empirical models as well as known real fish swimming behaviors to highlight the advantages of the proposed platform.},
project_kind={code_repo}
}
@article{newbury2024review,
title={A Review of Differentiable Simulators},
author={R. Newbury and Jack Collins and Kerry He and Jiahe Pan and Ingmar Posner and David Howard and Akansel Cosgun},
journal={IEEE Access},
year={2024},
category={multiphys,survey},
doi={10.1109/ACCESS.2024.3425448},
link={https://doi.org/10.1109/ACCESS.2024.3425448},
project={https://rhys-newbury.github.io/projects/DiffSim},
openalex_id={W4400447170},
abstract={Differentiable simulators continue to push the state of the art across a range of domains including computational physics, robotics, and machine learning. Their main value is the ability to compute gradients of physical processes, which allows differentiable simulators to be readily integrated into commonly employed gradient-based optimization schemes. To achieve this, a number of design decisions need to be considered representing trade-offs in versatility, computational speed, and accuracy of the gradients obtained. This paper presents an in-depth review of the evolving landscape of differentiable physics simulators. We introduce the foundations and core components of differentiable simulators alongside common design choices. This is followed by a practical guide and overview of open-source differentiable simulators that have been used across past research. Finally, we review and contextualize prominent applications of differentiable simulation. By offering a comprehensive review of the current state-of-the-art in differentiable simulation, this work aims to serve as a resource for researchers and practitioners looking to understand and integrate differentiable physics within their research. We conclude by highlighting current limitations as well as providing insights into future directions for the field.},
project_kind={project_page}
}
@inproceedings{liu2024softmac,
title={SoftMAC: Differentiable Soft Body Simulation with Forecast-based Contact Model and Two-way Coupling with Articulated Rigid Bodies and Clothes},
author={Min Liu and Gang Yang and Siyuan Luo and Lin Shao},
booktitle={2024 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
year={2024},
category={multiphys,differentiable_simulation,embodied_ai,contact},
doi={10.1109/IROS58592.2024.10801308},
link={https://doi.org/10.1109/IROS58592.2024.10801308},
project={https://minliu01.github.io/SoftMAC/},
openalex_id={W4389471087},
abstract={Differentiable physics simulation provides an avenue to tackle previously intractable challenges through gradient-based optimization, thereby greatly improving the efficiency of solving robotics-related problems. To apply differentiable simulation in diverse robotic manipulation scenarios, a key challenge is to integrate various materials in a unified framework. We present SoftMAC, a differentiable simulation framework that couples soft bodies with articulated rigid bodies and clothes. SoftMAC simulates soft bodies with the continuum-mechanics-based Material Point Method (MPM). We provide a novel forecast-based contact model for MPM, which effectively reduces penetration without introducing other artifacts like unnatural rebound. To couple MPM particles with deformable and non-volumetric clothes meshes, we also propose a penetration tracing algorithm that reconstructs the signed distance field in local area. Diverging from previous works, SoftMAC simulates the complete dynamics of each modality and incorporates them into a cohesive system with an explicit and differentiable coupling mechanism. The feature empowers SoftMAC to handle a broader spectrum of interactions, such as soft bodies serving as manipulators and engaging with underactuated systems. We conducted comprehensive experiments to validate the effectiveness and accuracy of the proposed differentiable pipeline in downstream robotic manipulation applications. Supplementary materials and videos are available on our project website at https://minliu01.github.io/SoftMAC.},
project_kind={project_page}
}
@inproceedings{mnager2025differentiable,
title={Differentiable Simulation of Soft Robots with Frictional Contacts},
author={Etienne Ménager and Louis Montaut and Quentin Le Lidec and Justin Carpentier},
booktitle={2025 IEEE 8th International Conference on Soft Robotics (RoboSoft)},
year={2025},
category={softbody,differentiable_simulation},
tag={reinforce_learning},
doi={10.1109/ROBOSOFT63089.2025.11020844},
link={https://doi.org/10.1109/ROBOSOFT63089.2025.11020844},
project={https://simple-robotics.github.io/publications/differentiable-soft-robotics/},
openalex_id={W4407054423},
abstract={In recent years, soft robotics simulators have evolved to offer various functionalities, including the simulation of different material types (e.g., elastic, hyper-elastic) and actuation methods (e.g., pneumatic, cable-driven, servomotor). These simulators also provide tools for various tasks, such as calibration, design, and control. However, efficiently and accurately computing derivatives within these simulators remains a challenge, particularly in the presence of physical contact interactions. Incorporating these derivatives can, for instance, significantly improve the convergence speed of control methods like reinforcement learning and trajectory optimization, enable gradient-based techniques for design, or facilitate end-to-end machine-learning approaches for model reduction. This paper addresses these challenges by introducing a unified method for computing the derivatives of mechanical equations within the finite element method framework, including contact interactions modeled as a nonlinear complementarity problem. The proposed approach handles both collision and friction phases, accounts for their nonsmooth dynamics, and leverages the sparsity introduced by mesh-based models. Its effectiveness is demonstrated through several examples of controlling and calibrating soft systems.},
project_kind={project_page}
}
@article{yifei2025neural,
title={Neural Modular Physics for Elastic Simulation},
author={Li, Yifei and Wu, Haixu and Xu, Zeyi and Stuyck, Tuur and Matusik, Wojciech},
journal={ArXiv.org},
year={2025},
category={softbody,neural_solver},
link={https://arxiv.org/abs/2512.15083},
project={https://people.csail.mit.edu/liyifei/publication/nmp},
openalex_id={W7116210361},
abstract={Learning-based methods have made significant progress in physics simulation, typically approximating dynamics with a monolithic end-to-end optimized neural network. Although these models offer an effective way to simulation, they may lose essential features compared to traditional numerical simulators, such as physical interpretability and reliability. Drawing inspiration from classical simulators that operate in a modular fashion, this paper presents Neural Modular Physics (NMP) for elastic simulation, which combines the approximation capacity of neural networks with the physical reliability of traditional simulators. Beyond the previous monolithic learning paradigm, NMP enables direct supervision of intermediate quantities and physical constraints by decomposing elastic dynamics into physically meaningful neural modules connected through intermediate physical quantities. With a specialized architecture and training strategy, our method transforms the numerical computation flow into a modular neural simulator, achieving improved physical consistency and generalizability. Experimentally, NMP demonstrates superior generalization to unseen initial conditions and resolutions, stable long-horizon simulation, better preservation of physical properties compared to other neural simulators, and greater feasibility in scenarios with unknown underlying dynamics than traditional simulators.},
project_kind={webpage}
}
@article{daviet2025neurally,
title={Neurally Integrated Finite Elements for Differentiable Elasticity on Evolving Domains},
author={Gilles Daviet and Tianchang Shen and Nicholas Sharp and David I. W. Levin},
journal={ACM Transactions on Graphics},
year={2025},
category={softbody,differentiable_simulation,reconstruction},
tag={real2sim},
doi={10.1145/3727874},
link={https://doi.org/10.1145/3727874},
project={https://research.nvidia.com/labs/toronto-ai/flexisim/},
openalex_id={W4409087605},
abstract={We present an elastic simulator for domains defined as evolving implicit functions, which is efficient, robust, and differentiable with respect to both shape and material. This simulator is motivated by applications in 3D reconstruction: it is increasingly effective to recover geometry from observed images as implicit functions, but physical applications require accurately simulating and optimizing-for the behavior of such shapes under deformation, which has remained challenging. Our key technical innovation is to train a small neural network to fit quadrature points for robust numerical integration on implicit grid cells. When coupled with a Mixed Finite Element formulation, this yields a smooth, fully differentiable simulation model connecting the evolution of the underlying implicit surface to its elastic response. We demonstrate the efficacy of our approach on forward simulation of implicits, direct simulation of 3D shapes during editing, and novel physics-based shape and topology optimizations in conjunction with differentiable rendering.},
project_kind={project_page}
}
@inproceedings{ye2026efficient,
title={Efficient Differentiable Contact Model with Long-range Influence},
author={Xiaohan Ye and Kui Wu and Zherong Pan and Taku Komura},
booktitle={International Conference on Learning Representations (ICLR)},
year={2026},
category={rigidbody,differentiable_simulation,contact},
doi={10.48550/arXiv.2509.20917},
link={https://arxiv.org/abs/2509.20917},
abstract={With the maturation of differentiable physics, its role in various downstream applications: such as model predictive control, robotic design optimization, and neural PDE solvers, has become increasingly important. However, the derivative information provided by differentiable simulators can exhibit abrupt changes or vanish altogether, impeding the convergence of gradient-based optimizers. In this work, we demonstrate that such erratic gradient behavior is closely tied to the design of contact models. We further introduce a set of properties that a contact model must satisfy to ensure well-behaved gradient information. Lastly, we present a practical contact model for differentiable rigid-body simulators that satisfies all of these properties while maintaining computational efficiency. Our experiments show that, even from simple initializations, our contact model can discover complex, contact-rich control signals, enabling the successful execution of a range of downstream locomotion and manipulation tasks.}
}
@inproceedings{weib2020correspondence,
title={Correspondence-Free Material Reconstruction using Sparse Surface Constraints},
author={Sebastian Weiß and Robert Maier and Daniel Cremers and Rüdiger Westermann and Nils Thuerey},
booktitle={2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2020},
category={softbody,reconstruction},
tag={real2sim,material_estimation},
doi={10.1109/cvpr42600.2020.00474},
link={https://doi.org/10.1109/cvpr42600.2020.00474},
project={https://ge.in.tum.de/publications/2020-cvpr-weiss},
openalex_id={W3034237986},
abstract={We present a method to infer physical material parameters, and even external boundaries, from the scanned motion of a homogeneous deformable object via the solution of an inverse problem. Parameters are estimated from real-world data sources such as sparse observations from a Kinect sensor without correspondences. We introduce a novel Lagrangian-Eulerian optimization formulation, including a cost function that penalizes differences to observations during an optimization run. This formulation matches correspondence-free, sparse observations from a single-view depth image with a finite element simulation of deformable bodies. In a number of tests using synthetic datasets and real-world measurements, we analyse the robustness of our approach and the convergence behavior of the numerical optimization scheme.},
project_kind={webpage}
}
@inproceedings{guo2021inverse,
title={Inverse Simulation: Reconstructing Dynamic Geometry of Clothed Humans via Optimal Control},
author={Jingfan Guo and Jie Li and Rahul Narain and Hyun Soo Park},
booktitle={2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2021},
category={cloth,reconstruction,avatar},
tag={real2sim,cloth_reconstruction},
doi={10.1109/cvpr46437.2021.01446},
link={https://doi.org/10.1109/cvpr46437.2021.01446},
openalex_id={W3179683226},
abstract={This paper studies the problem of inverse cloth simulation—to estimate shape and time-varying poses of the underlying body that generates physically plausible cloth motion, which matches to the point cloud measurements on the clothed humans. A key innovation is to represent the dynamics of the cloth geometry using a dynamical system that is controlled by the body states (shape and pose). This allows us to express the cloth motion as a resultant of external (skin friction and gravity) and internal (elasticity) forces. Inspired by the theory of optimal control, we optimize the body states such that the simulated cloth motion is matched to the point cloud measurements, and the analytic gradient of the simulator is back-propagated to update the body states. We propose a cloth relaxation scheme to initialize the cloth state, which ensures the physical validity. Our method produces physically plausible and temporally smooth cloth and body movements that are faithful to the measurements, and shows superior performance compared to the existing methods. As a byproduct, the stress and strain that are applied to the body and clothes can be recovered.}
}
s
@inproceedings{mezghanni2022physical,
title={Physical Simulation Layer for Accurate 3D Modeling},
author={Mariem Mezghanni and Théo Bodrito and Malika Boulkenafed and Maks Ovsjanikov},
booktitle={2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2022},
category={rigidbody,differentiable_simulation},
doi={10.1109/cvpr52688.2022.01315},
link={https://doi.org/10.1109/cvpr52688.2022.01315},
openalex_id={W4312332873},
abstract={We introduce a novel approach for generative 3D modeling that explicitly encourages the physical and thus functional consistency of the generated shapes. To this end, we advocate the use of online physical simulation as part of learning a generative model. Unlike previous related methods, our approach is trained end-to-end with a fully differentiable physical simulator in the training loop. We accomplish this by leveraging recent advances in differentiable programming, and introducing a fully differentiable point-based physical simulation layer, which accurately evaluates the shape's stability when subjected to gravity. We then incorporate this layer in a signed distance function (SDF) shape decoder. By augmenting a conventional SDF decoder with our simulation layer, we demonstrate through extensive experiments that online physical simulation improves the accuracy, visual plausibility and physical validity of the resulting shapes, while requiring no additional data or annotation effort.}
}
@inproceedings{huang2022mesh,
title={Mesh-based Dynamics with Occlusion Reasoning for Cloth Manipulation},
author={Zixuan Huang and Xingyu Lin and David Held},
booktitle={Robotics: Science and Systems XVIII},
year={2022},
category={cloth,reconstruction,embodied_ai},
tag={cloth_reconstruction},
doi={10.15607/rss.2022.xviii.011},
link={https://doi.org/10.15607/rss.2022.xviii.011},
openalex_id={W4281765718},
abstract={Self-occlusion is challenging for cloth manipulation, as it makes it difficult to estimate the full state of the cloth. Ideally, a robot trying to unfold a crumpled or folded cloth should be able to reason about the cloth's occluded regions. We leverage recent advances in pose estimation for cloth to build a system that uses explicit occlusion reasoning to unfold a crumpled cloth. Specifically, we first learn a model to reconstruct the mesh of the cloth. However, the model will likely have errors due to the complexities of the cloth configurations and due to ambiguities from occlusions. Our main insight is that we can further refine the predicted reconstruction by performing test-time finetuning with self-supervised losses. The obtained reconstructed mesh allows us to use a mesh-based dynamics model for planning while reasoning about occlusions. We evaluate our system both on cloth flattening as well as on cloth canonicalization, in which the objective is to manipulate the cloth into a canonical pose. Our experiments show that our method significantly outperforms prior methods that do not explicitly account for occlusions or perform test-time optimization. Videos and visualizations can be found on our project website.}
}
@inproceedings{kaneko2024improving,
title={Improving Physics-Augmented Continuum Neural Radiance Field-Based Geometry-Agnostic System Identification with Lagrangian Particle Optimization},
author={Takuhiro Kaneko},
booktitle={2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2024},
category={softbody,reconstruction},
tag={real2sim,parameter_estimation,nerf},
doi={10.1109/cvpr52733.2024.00523},
link={https://doi.org/10.1109/cvpr52733.2024.00523},
openalex_id={W4402716344},
abstract={Geometry-agnostic system identification is a technique for identifying the geometry and physical properties of an object from video sequences without any geometric as-sumptions. Recently, physics-augmented continuum neural radiance fields (PAC-NeRF) has demonstrated promising results for this technique by utilizing a hybrid Eulerian-Lagrangian representation, in which the geometry is represented by the Eulerian grid representations of NeRF, the physics is described by a material point method (MPM), and they are connected via Lagrangian particles. However, a notable limitation of PAC-NeRF is that its performance is sensitive to the learning of the geometry from the first frames owing to its two-step optimization. First, the grid representations are optimized with the first frames of video sequences, and then the physical properties are optimized through video sequences utilizing the fixed first-frame grid representations. This limitation can be critical when learning of the geometric structure is difficult, for example, in a few-shot (sparse view) setting. To overcome this limitation, we propose Lagrangian particle optimization (LPO), in which the positions and features of particles are opti-mized through video sequences in Lagrangian space. This method allows for the optimization of the geometric structure across the entire video sequence within the physical constraints imposed by the MPM. The experimental results demonstrate that the LPO is useful for geometric correction and physical identification in sparse-view settings. 11Video samples are available at https://www.keel.ntt.co.jp/people/kaneko.takuhiro/projeets/lpo/}
}
@article{yu2024inverse,
title={Inverse Garment and Pattern Modeling with a Differentiable Simulator},
author={Boyang Yu and Frédéric Cordier and Hyewon Seo},
journal={Computer Graphics Forum},
year={2024},
category={cloth,differentiable_simulation,reconstruction},
tag={cloth_reconstruction},
doi={10.1111/cgf.15249},
link={https://doi.org/10.1111/cgf.15249},
openalex_id={W4404195038},
abstract={Abstract The capability to generate simulation‐ready garment models from 3D shapes of clothed people will significantly enhance the interpretability of captured geometry of real garments, as well as their faithful reproduction in the digital world. This will have notable impact on fields like shape capture in social VR, and virtual try‐on in the fashion industry. To align with the garment modeling process standardized by the fashion industry and cloth simulation software, it is required to recover 2D patterns, which are then placed around the wearer's body model and seamed prior to the draping simulation. This involves an inverse garment design problem, which is the focus of our work here: Starting with an arbitrary target garment geometry, our system estimates its animatable replica along with its corresponding 2D pattern. Built upon a differentiable cloth simulator, it runs an optimization process that is directed towards minimizing the deviation of the simulated garment shape from the target geometry, while maintaining desirable properties such as left‐to‐right symmetry. Experimental results on various real‐world and synthetic data show that our method outperforms state‐of‐the‐art methods in producing both high‐quality garment models and accurate 2D patterns.}
}
@article{tao2025flowcapx,
title={FlowCapX: Physics‐Grounded Flow Capture with Long‐Term Consistency},
author={N. Tao and L. Zhang and Xingyu Ni and Mengyu Chu and Baoquan Chen},
journal={Computer Graphics Forum},
year={2025},
category={fluid,reconstruction},
tag={real2sim,fluid_reconstruction},
doi={10.1111/cgf.70274},
link={https://doi.org/10.1111/cgf.70274},
project={https://github.com/taoningxiao/FlowCapX},
openalex_id={W4415077326},
abstract={Abstract We present FlowCapX , a physics‐enhanced framework for flow reconstruction from sparse video inputs, addressing the challenge of jointly optimizing complex physical constraints and sparse observational data over long time horizons. Existing methods often struggle to capture turbulent motion while maintaining physical consistency, limiting reconstruction quality and downstream tasks. Focusing on velocity inference, our approach introduces a hybrid framework that strategically separates representation and supervision across spatial scales. At the coarse level, we resolve sparse‐view ambiguities via a novel optimization strategy that aligns long‐term observation with physics‐grounded velocity fields. By emphasizing vorticity‐based physical constraints, our method enhances physical fidelity and improves optimization stability. At the fine level, we prioritize observational fidelity to preserve critical turbulent structures. Extensive experiments demonstrate state‐of‐the‐art velocity reconstruction, enabling velocity‐aware downstream tasks, e.g., accurate flow analysis, scene augmentation with tracer visualization and re‐simulation. Our implementation is released at ://github.com/taoningxiao/FlowCapX.git .},
project_kind={code_repo}
}
@inproceedings{kaneko2025structure,
title={Structure from Collision},
author={Takuhiro Kaneko},
booktitle={2025 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year={2025},
category={softbody,reconstruction,contact},
tag={material_estimation,nerf,3dgs},
doi={10.1109/cvpr52734.2025.01521},
link={https://doi.org/10.1109/cvpr52734.2025.01521},
openalex_id={W4413157470},
abstract={Recent advancements in neural 3D representations, such as neural radiance fields (NeRF) and 3D Gaussian splatting (3DGS), have made accurate estimation of the 3D structure from multiview images possible. However, this capability is limited to estimating the visible external structure, and it is still difficult to identify the invisible internal structure hidden behind the surface. To overcome this limitation, we address a new task called structure from collision (SfC), which aims to estimate the structure (including the invisible internal one) of an object from the appearance changes at collision. To solve this task, we propose a novel model called SfC-NeRF, which optimizes the invisible internal structure of the object through a video sequence under physical, appearance (i.e., visible external structure)-preserving, and keyframe constraints. In particular, to avoid falling into undesirable local optima owing to its ill-posed nature, we propose volume annealing, i.e., searching for the global optima by repeatedly reducing and expanding the volume. Extensive experiments on 115 objects involving diverse structures (i.e., various cavity shapes, locations, and sizes) and various material properties reveal the properties of SfC and demonstrate the effectiveness of the proposed SfC-NeRF. 1}
}
@article{yu2024super,
title={Super-Resolution Cloth Animation with Spatial and Temporal Coherence},
author={Jiawang Yu and Zhendong Wang},
journal={ACM Transactions on Graphics},
year={2024},
category={cloth},
tag={super_resolution},
doi={10.1145/3658143},
link={https://doi.org/10.1145/3658143},
project={https://www.physicsbasedanimation.com/2024/07/09/super-resolution-cloth-animation-with-spatial-and-temporal-coherence},
openalex_id={W4400821249},
abstract={Creating super-resolution cloth animations, which refine coarse cloth meshes with fine wrinkle details, faces challenges in preserving spatial consistency and temporal coherence across frames. In this paper, we introduce a general framework to address these issues, leveraging two core modules. The first module interleaves a simulator and a corrector. The simulator handles cloth dynamics, while the corrector rectifies differences in low-frequency features across various resolutions. This interleaving ensures prompt correction of spatial errors from the coarse simulation, effectively preventing their temporal propagation. The second module performs mesh-based super-resolution for detailed wrinkle enhancements. We decompose garment meshes into overlapping patches for adaptability to various styles and geometric continuity. Our method achieves an 8× improvement in resolution for cloth animations. We showcase the effectiveness of our method through diverse animation examples, including simple cloth pieces and intricate garments.},
project_kind={webpage}
}