From: Jan Grulich Date: Thu, 27 Aug 2026 08:06:00 +0000 Subject: Bug 2054622 - WebRTC backport: PipeWire mmap improvements r=pehrsons PipeWire: encapsulate mmap in ScopedBuf and fix mapoffset handling Move mmap into ScopedBuf::initialize so callers no longer manage mmap flags and cleanup directly. Use MAP_PRIVATE for read-only MemFd consumers and MAP_SHARED for DmaBuf and read-write consumers. Fix screen capture's double mapoffset application by passing the offset to mmap directly. Add AccessMode and BufferType enums for clarity and delete copy/move to prevent double-munmap. PipeWire: validate fd backing size before mmap Validate that the fd actually backs the requested mapping range by calling fstat and checking mapoffset + maxsize <= st_size before mmap, matching what PipeWire does in its own memory pool (src/pipewire/mem.c). Skip the check for DmaBuf fds where fstat does not reliably report the buffer size. This is a simple backport of WebRTC upstream changes. Upstream commits: c84dbe590a5519bc1404a268dd8816088ffe8980 5c10f93376c1e9f0ff339bd1facf5b6d7d978f25 Differential Revision: https://phabricator.services.mozilla.com/D321791 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/abff9e232490c530fa9693f3b6792966ab8a877b --- .../linux/wayland/shared_screencast_stream.cc | 11 ++--- .../linux/wayland/test/test_egl_dmabuf.cc | 5 +-- .../test/test_screencast_stream_provider.cc | 7 ++-- modules/portal/pipewire_utils.cc | 42 +++++++++++++++++++ modules/portal/pipewire_utils.h | 33 ++++++++------- .../linux/video_capture_pipewire.cc | 12 +++--- 6 files changed, 72 insertions(+), 38 deletions(-) diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc index b42fa2e139..c35f9370e9 100644 --- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc +++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc @@ -1109,20 +1109,15 @@ bool SharedScreenCastStreamPrivate::ProcessMemFDBuffer( uint8_t* src = nullptr; const uint64_t maxsize = static_cast(spa_buffer->datas[0].maxsize); - const uint64_t mapoffset = - static_cast(spa_buffer->datas[0].mapoffset); - - map.initialize( - static_cast(mmap(nullptr, maxsize + mapoffset, PROT_READ, - MAP_PRIVATE, spa_buffer->datas[0].fd, 0)), - maxsize + mapoffset, spa_buffer->datas[0].fd); + map.initialize(spa_buffer->datas[0].fd, maxsize, + spa_buffer->datas[0].mapoffset, ScopedBuf::BufferType::kMemFd); if (!map) { RTC_LOG(LS_ERROR) << "Failed to mmap the memory: " << std::strerror(errno); return false; } - src = SPA_MEMBER(map.get(), mapoffset, uint8_t); + src = map.get(); const uint64_t src_stride = spa_buffer->datas[0].chunk->stride; diff --git a/modules/desktop_capture/linux/wayland/test/test_egl_dmabuf.cc b/modules/desktop_capture/linux/wayland/test/test_egl_dmabuf.cc index 36be3fa8ee..58f745ede5 100644 --- a/modules/desktop_capture/linux/wayland/test/test_egl_dmabuf.cc +++ b/modules/desktop_capture/linux/wayland/test/test_egl_dmabuf.cc @@ -60,10 +60,9 @@ bool TestEglDrmDevice::ImageFromDmaBuf( const size_t plane_stride = plane.stride; const size_t buffer_size_bytes = plane_stride * size.height(); - uint8_t* map = static_cast( - mmap(nullptr, buffer_size_bytes, PROT_READ, MAP_SHARED, plane.fd, 0)); ScopedBuf scoped_buf; - scoped_buf.initialize(map, buffer_size_bytes, plane.fd, true); + scoped_buf.initialize(plane.fd, buffer_size_bytes, 0, + ScopedBuf::BufferType::kDmaBuf); if (!scoped_buf) { RTC_LOG(LS_ERROR) << "TestEglDrmDevice: Failed to mmap DMA-BUF"; diff --git a/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc b/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc index 7253cbf166..3b95eb7a2c 100644 --- a/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc +++ b/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc @@ -195,10 +195,9 @@ void TestScreenCastStreamProvider::RecordFrame(RgbaColor rgba_color, ScopedBuf scoped_buf; if (spa_data->type == SPA_DATA_DmaBuf) { - uint8_t* map = - static_cast(mmap(nullptr, buffer_size, PROT_READ | PROT_WRITE, - MAP_SHARED, spa_data->fd, 0)); - scoped_buf.initialize(map, buffer_size, spa_data->fd, true); + scoped_buf.initialize(spa_data->fd, buffer_size, 0, + ScopedBuf::BufferType::kDmaBuf, + ScopedBuf::AccessMode::kReadWrite); if (!scoped_buf) { RTC_LOG(LS_ERROR) << "Failed to mmap DMA-BUF for recording"; pw_stream_queue_buffer(pw_stream_, buffer); diff --git a/modules/portal/pipewire_utils.cc b/modules/portal/pipewire_utils.cc index 7008696b7d..bdea4318e7 100644 --- a/modules/portal/pipewire_utils.cc +++ b/modules/portal/pipewire_utils.cc @@ -11,13 +11,18 @@ #include "modules/portal/pipewire_utils.h" #include +#include +#include +#include +#include #include #include #include #include #include +#include "rtc_base/checks.h" #include "rtc_base/sanitizer.h" #include "rtc_base/string_encode.h" #include "rtc_base/string_to_number.h" @@ -120,4 +125,41 @@ PipeWireInitializer::~PipeWireInitializer() { } } +void ScopedBuf::initialize(int fd, + size_t maxsize, + off_t mapoffset, + BufferType buffer_type, + AccessMode mode) { + // PipeWire supports shared memory mappings <= 4GB in size. + RTC_CHECK_GE(mapoffset, 0); + RTC_CHECK_LE(mapoffset, UINT32_MAX); + RTC_CHECK_LE(maxsize, UINT32_MAX); + + if (buffer_type == BufferType::kMemFd) { + struct stat sb; + if (fstat(fd, &sb) != 0 || + static_cast(mapoffset) + static_cast(maxsize) > + static_cast(sb.st_size)) { + return; + } + } + + int prot = + mode == AccessMode::kReadWrite ? (PROT_READ | PROT_WRITE) : PROT_READ; + int flags = + (buffer_type == BufferType::kDmaBuf || mode == AccessMode::kReadWrite) + ? MAP_SHARED + : MAP_PRIVATE; + + map_ = + static_cast(mmap(nullptr, maxsize, prot, flags, fd, mapoffset)); + map_size_ = maxsize; + fd_ = fd; + buffer_type_ = buffer_type; + + if (buffer_type_ == BufferType::kDmaBuf && map_ != MAP_FAILED) { + SyncDmaBuf(fd_, DMA_BUF_SYNC_START); + } +} + } // namespace webrtc diff --git a/modules/portal/pipewire_utils.h b/modules/portal/pipewire_utils.h index d0734c19be..e3769294f2 100644 --- a/modules/portal/pipewire_utils.h +++ b/modules/portal/pipewire_utils.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -102,12 +103,17 @@ static bool SyncDmaBuf(int fd, uint64_t start_or_end) { class ScopedBuf { public: + enum class AccessMode { kReadOnly, kReadWrite }; + enum class BufferType { kMemFd, kDmaBuf }; + ScopedBuf() {} - ScopedBuf(uint8_t* map, int map_size, int fd, bool is_dma_buf = false) - : map_(map), map_size_(map_size), fd_(fd), is_dma_buf_(is_dma_buf) {} + ScopedBuf(const ScopedBuf&) = delete; + ScopedBuf& operator=(const ScopedBuf&) = delete; + ScopedBuf(ScopedBuf&&) = delete; + ScopedBuf& operator=(ScopedBuf&&) = delete; ~ScopedBuf() { if (map_ != MAP_FAILED) { - if (is_dma_buf_) { + if (buffer_type_ == BufferType::kDmaBuf) { SyncDmaBuf(fd_, DMA_BUF_SYNC_END); } munmap(map_, map_size_); @@ -116,24 +122,19 @@ class ScopedBuf { explicit operator bool() { return map_ != MAP_FAILED; } - void initialize(uint8_t* map, int map_size, int fd, bool is_dma_buf = false) { - map_ = map; - map_size_ = map_size; - is_dma_buf_ = is_dma_buf; - fd_ = fd; - - if (is_dma_buf_) { - SyncDmaBuf(fd_, DMA_BUF_SYNC_START); - } - } + void initialize(int fd, + size_t maxsize, + off_t mapoffset, + BufferType buffer_type, + AccessMode mode = AccessMode::kReadOnly); uint8_t* get() { return map_; } protected: uint8_t* map_ = static_cast(MAP_FAILED); - int map_size_; - int fd_; - bool is_dma_buf_; + size_t map_size_ = 0; + int fd_ = -1; + BufferType buffer_type_ = BufferType::kMemFd; }; } // namespace webrtc diff --git a/modules/video_capture/linux/video_capture_pipewire.cc b/modules/video_capture/linux/video_capture_pipewire.cc index 54586e2673..2b3b619c38 100644 --- a/modules/video_capture/linux/video_capture_pipewire.cc +++ b/modules/video_capture/linux/video_capture_pipewire.cc @@ -24,7 +24,6 @@ #include #include #include -#include #include #include @@ -483,12 +482,11 @@ void VideoCaptureModulePipeWire::ProcessBuffers() { if (spaBuffer->datas[0].type == SPA_DATA_DmaBuf || spaBuffer->datas[0].type == SPA_DATA_MemFd) { ScopedBuf frame; - frame.initialize( - static_cast( - mmap(nullptr, spaBuffer->datas[0].maxsize, PROT_READ, MAP_SHARED, - spaBuffer->datas[0].fd, spaBuffer->datas[0].mapoffset)), - spaBuffer->datas[0].maxsize, spaBuffer->datas[0].fd, - spaBuffer->datas[0].type == SPA_DATA_DmaBuf); + frame.initialize(spaBuffer->datas[0].fd, spaBuffer->datas[0].maxsize, + spaBuffer->datas[0].mapoffset, + spaBuffer->datas[0].type == SPA_DATA_DmaBuf + ? ScopedBuf::BufferType::kDmaBuf + : ScopedBuf::BufferType::kMemFd); if (!frame) { RTC_LOG(LS_ERROR) << "Failed to mmap the memory: "