/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "FFmpegLog.h" #include "FFmpegVideoDecoder.h" #include "mozilla/DataMutex.h" #include "mozilla/ScopeExit.h" #include "nsPrintfCString.h" #if LIBAVCODEC_VERSION_MAJOR >= 60 && !defined(FFVPX_VERSION) # if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK) # include # include // mozilla/widget/DMABufFormats.h (via FFmpegVideoDecoder.h -> DMABufDevice.h) // may define DRM_FORMAT_MOD_INVALID before libdrm; same pattern as // DMABufSurface.cpp / FFmpegVideoFramePool.cpp. # ifdef DRM_FORMAT_MOD_INVALID # undef DRM_FORMAT_MOD_INVALID # endif # include # ifndef DRM_FORMAT_MOD_INVALID # define DRM_FORMAT_MOD_INVALID ((1ULL << 56) - 1) # endif # include # include # include # include # include "libavutil/hwcontext.h" # include "libavutil/hwcontext_vulkan.h" # include "libavutil/macros.h" # include "libavutil/pixfmt.h" # include "libavutil/version.h" # include "mozilla/StaticPrefs_media.h" # ifdef __linux__ # include # elif defined(XP_SOLARIS) || defined(__sun) # include // major(), minor() for st_rdev # elif defined(XP_FREEBSD) || defined(XP_OPENBSD) || defined(XP_NETBSD) # include // major(), minor() for st_rdev (BSD) # endif # endif // MOZ_USE_HWDECODE && MOZ_WIDGET_GTK #endif // LIBAVCODEC_VERSION_MAJOR >= 60 && !defined(FFVPX_VERSION) namespace mozilla { #if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK) # if LIBAVCODEC_VERSION_MAJOR >= 60 && !defined(FFVPX_VERSION) FFmpegVideoDecoder< LIBAV_VER>::FFmpegVulkanVideoDecoder::~FFmpegVulkanVideoDecoder() { if (!StaticPrefs::media_ffvpx_hw_enabled()) { return; } // Resources should already be cleaned up by ProcessShutdown() // If mDevice is not null here, it means ProcessShutdown wasn't called // and the device may already be destroyed - don't try to clean up if (mDevice) { NS_WARNING( "~FFmpegVulkanVideoDecoder called with device still set - resources " "may leak"); } } void FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::Cleanup() { FFMPEGV_LOG("FFmpegVulkanVideoDecoder::Cleanup()"); if (mDevice) { if (mDeviceWaitIdle) { mDeviceWaitIdle(mDevice); } for (uint32_t qi = 0; qi < mCopyQueueCount; qi++) { if (mCopyCmdBuf[qi] && mCopyCmdPool[qi] && mFreeCommandBuffers) { mFreeCommandBuffers(mDevice, mCopyCmdPool[qi], 1, &mCopyCmdBuf[qi]); } if (mCopyCmdPool[qi] && mDestroyCommandPool) { mDestroyCommandPool(mDevice, mCopyCmdPool[qi], nullptr); } if (mCopyFence[qi] && mDestroyFence) { mDestroyFence(mDevice, mCopyFence[qi], nullptr); } } for (int i = 0; i < kNumBuffers; i++) { if (mCopyDoneSemFd[i] >= 0) { close(mCopyDoneSemFd[i]); mCopyDoneSemFd[i] = -1; } mCopyDoneSemValue[i] = 0; mCopyDoneSemSignaled[i] = false; if (mCopyDoneSem[i] && mDestroySemaphore) { mDestroySemaphore(mDevice, mCopyDoneSem[i], nullptr); mCopyDoneSem[i] = VK_NULL_HANDLE; } if (mNv12BaseFd[i] >= 0) { close(mNv12BaseFd[i]); } if (mNv12Image[i] && mDestroyImage) { mDestroyImage(mDevice, mNv12Image[i], nullptr); } if (mNv12Mem[i] && mFreeMemory) { mFreeMemory(mDevice, mNv12Mem[i], nullptr); } } } mDevice = nullptr; mCopyQueueCount = 0; mCopyQueueIsDedicatedTransfer = false; mCopyQueueRoundRobin = 0; mCopyQueue.Clear(); mCopyCmdPool.Clear(); mCopyCmdBuf.Clear(); mCopyFence.Clear(); mDeviceFunctions.Clear(); for (int i = 0; i < kNumBuffers; i++) { mNv12Image[i] = nullptr; mNv12Mem[i] = nullptr; mNv12BaseFd[i] = -1; mCopyDoneSem[i] = VK_NULL_HANDLE; mCopyDoneSemFd[i] = -1; mCopyDoneSemValue[i] = 0; mCopyDoneSemSignaled[i] = false; } mCurrentBuffer = 0; mWidth = 0; mHeight = 0; mTotalSize = 0; mUvOffset = 0; mYPitch = 0; mUvPitch = 0; } namespace { // Cached instance-level Vulkan function pointers, shared across all decoders // for the lifetime of the process as long as the VkInstance doesn't change. struct InstanceFunctionCache { VkInstance mInstance = VK_NULL_HANDLE; PFN_vkGetDeviceProcAddr mGetDeviceProcAddr = nullptr; PFN_vkGetPhysicalDeviceProperties mGetPhysicalDeviceProperties = nullptr; PFN_vkGetPhysicalDeviceQueueFamilyProperties mGetPhysicalDeviceQueueFamilyProperties = nullptr; PFN_vkGetPhysicalDeviceMemoryProperties mGetPhysicalDeviceMemoryProperties = nullptr; PFN_vkGetPhysicalDeviceFormatProperties2 mGetPhysicalDeviceFormatProperties2 = nullptr; PFN_vkGetPhysicalDeviceImageFormatProperties2 mGetPhysicalDeviceImageFormatProperties2 = nullptr; PFN_vkGetPhysicalDeviceExternalSemaphoreProperties mGetPhysicalDeviceExternalSemaphoreProperties = nullptr; // Flat array of all the above pointers for use by IsLoaded(). nsTArray mFnPtrs; }; } // namespace constinit static StaticDataMutex sInstanceFnCache{ "VulkanInstanceFunctions"}; void FFmpegVideoDecoder::FFmpegVulkanVideoDecoder:: LoadInstanceFunctions(PFN_vkGetInstanceProcAddr aGetProcAddr, VkInstance aInst, VkPhysicalDevice aPhysDev) { auto cache = sInstanceFnCache.Lock(); if (cache->mInstance == aInst && cache->mGetDeviceProcAddr) { mGetDeviceProcAddr = cache->mGetDeviceProcAddr; mGetPhysicalDeviceProperties = cache->mGetPhysicalDeviceProperties; mGetPhysicalDeviceQueueFamilyProperties = cache->mGetPhysicalDeviceQueueFamilyProperties; mGetPhysicalDeviceMemoryProperties = cache->mGetPhysicalDeviceMemoryProperties; mGetPhysicalDeviceFormatProperties2 = cache->mGetPhysicalDeviceFormatProperties2; mGetPhysicalDeviceImageFormatProperties2 = cache->mGetPhysicalDeviceImageFormatProperties2; mGetPhysicalDeviceExternalSemaphoreProperties = cache->mGetPhysicalDeviceExternalSemaphoreProperties; mInstanceFunctions = cache->mFnPtrs.Clone(); return; } mInstanceFunctions.Clear(); auto load = [&](T& fn, const char* name) { fn = reinterpret_cast(aGetProcAddr(aInst, name)); if (!fn) { NS_WARNING(nsPrintfCString("[VULKAN] Failed to load %s", name).get()); } mInstanceFunctions.AppendElement((PFN_vkVoidFunction)fn); }; load(mGetDeviceProcAddr, "vkGetDeviceProcAddr"); load(mGetPhysicalDeviceProperties, "vkGetPhysicalDeviceProperties"); load(mGetPhysicalDeviceQueueFamilyProperties, "vkGetPhysicalDeviceQueueFamilyProperties"); load(mGetPhysicalDeviceMemoryProperties, "vkGetPhysicalDeviceMemoryProperties"); load(mGetPhysicalDeviceFormatProperties2, "vkGetPhysicalDeviceFormatProperties2"); load(mGetPhysicalDeviceImageFormatProperties2, "vkGetPhysicalDeviceImageFormatProperties2"); load(mGetPhysicalDeviceExternalSemaphoreProperties, "vkGetPhysicalDeviceExternalSemaphoreProperties"); cache->mInstance = aInst; cache->mGetDeviceProcAddr = mGetDeviceProcAddr; cache->mGetPhysicalDeviceProperties = mGetPhysicalDeviceProperties; cache->mGetPhysicalDeviceQueueFamilyProperties = mGetPhysicalDeviceQueueFamilyProperties; cache->mGetPhysicalDeviceMemoryProperties = mGetPhysicalDeviceMemoryProperties; cache->mGetPhysicalDeviceFormatProperties2 = mGetPhysicalDeviceFormatProperties2; cache->mGetPhysicalDeviceImageFormatProperties2 = mGetPhysicalDeviceImageFormatProperties2; cache->mGetPhysicalDeviceExternalSemaphoreProperties = mGetPhysicalDeviceExternalSemaphoreProperties; cache->mFnPtrs = mInstanceFunctions.Clone(); } void FFmpegVideoDecoder< LIBAV_VER>::FFmpegVulkanVideoDecoder::LoadDeviceFunctions(VkDevice aDev) { mDeviceFunctions.Clear(); auto load = [&](T& fn, const char* name) { fn = (T)(void*)mGetDeviceProcAddr(aDev, name); if (!fn) { NS_WARNING(nsPrintfCString("[VULKAN] Failed to load %s", name).get()); } mDeviceFunctions.AppendElement((PFN_vkVoidFunction)fn); }; load(mCreateCommandPool, "vkCreateCommandPool"); load(mDestroyCommandPool, "vkDestroyCommandPool"); load(mAllocateCommandBuffers, "vkAllocateCommandBuffers"); load(mFreeCommandBuffers, "vkFreeCommandBuffers"); load(mBeginCommandBuffer, "vkBeginCommandBuffer"); load(mEndCommandBuffer, "vkEndCommandBuffer"); load(mGetDeviceQueue, "vkGetDeviceQueue"); load(mQueueSubmit, "vkQueueSubmit"); load(mCmdPipelineBarrier, "vkCmdPipelineBarrier"); load(mCmdCopyImage, "vkCmdCopyImage"); load(mDeviceWaitIdle, "vkDeviceWaitIdle"); load(mCreateImage, "vkCreateImage"); load(mDestroyImage, "vkDestroyImage"); load(mGetImageMemoryRequirements, "vkGetImageMemoryRequirements"); load(mGetImageMemoryRequirements2, "vkGetImageMemoryRequirements2"); load(mGetImageSubresourceLayout, "vkGetImageSubresourceLayout"); load(mBindImageMemory, "vkBindImageMemory"); load(mAllocateMemory, "vkAllocateMemory"); load(mFreeMemory, "vkFreeMemory"); load(mCreateFence, "vkCreateFence"); load(mDestroyFence, "vkDestroyFence"); load(mResetFences, "vkResetFences"); load(mWaitForFences, "vkWaitForFences"); load(mGetSemaphoreCounterValue, "vkGetSemaphoreCounterValue"); load(mGetMemoryFdKHR, "vkGetMemoryFdKHR"); load(mGetImageDrmFormatModifierPropertiesEXT, "vkGetImageDrmFormatModifierPropertiesEXT"); load(mCreateSemaphore, "vkCreateSemaphore"); load(mDestroySemaphore, "vkDestroySemaphore"); load(mWaitSemaphores, "vkWaitSemaphores"); load(mGetSemaphoreFdKHR, "vkGetSemaphoreFdKHR"); } bool FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::IsLoaded() const { if (mInstanceFunctions.IsEmpty() || mDeviceFunctions.IsEmpty()) { return false; } auto allValid = [](const auto& aFuncs) { return std::all_of(aFuncs.begin(), aFuncs.end(), [](auto aFn) { return aFn != nullptr; }); }; return allValid(mInstanceFunctions) && allValid(mDeviceFunctions); } void FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::InitDrmModifiers( VkPhysicalDevice aPhysDev, VkFormat aFormatForModifiers, const nsTArray* aCompositorMods, VkImageUsageFlags aImageUsages) { mDrmModifiers.clear(); mExportRequiresDedicatedByModifier.Clear(); FFMPEGV_LOG("[VULKAN] Compositor %zu modifier(s) for intersection", aCompositorMods ? aCompositorMods->Length() : 0); const bool isCompositorSupportsOnlyLinear = !aCompositorMods || aCompositorMods->IsEmpty() || (aCompositorMods->Length() == 1 && aCompositorMods->ElementAt(0) == DRM_FORMAT_MOD_LINEAR); if (isCompositorSupportsOnlyLinear) { FFMPEGV_LOG( "[VULKAN] Compositor supports only LINEAR modifier; negotiation will " "intersect with decoder (result at most LINEAR)"); } // Query decoder device for supported DRM modifiers, then intersect with // compositor. Format must match decoded stream (NV12 or P010) so we use // aFormatForModifiers from decoded info. Do not add // VK_IMAGE_USAGE_STORAGE_BIT to aImageUsages; the query must use only // transfer and video-decode usage bits. if (mGetPhysicalDeviceFormatProperties2) { const VkFormat formatForModifiers = aFormatForModifiers; VkDrmFormatModifierPropertiesListEXT modList = {}; modList.sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT; VkFormatProperties2 formatProps = {}; formatProps.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2; formatProps.pNext = &modList; mGetPhysicalDeviceFormatProperties2(aPhysDev, formatForModifiers, &formatProps); if (modList.drmFormatModifierCount > 0) { std::vector modProps( modList.drmFormatModifierCount); modList.pDrmFormatModifierProperties = modProps.data(); mGetPhysicalDeviceFormatProperties2(aPhysDev, formatForModifiers, &formatProps); NS_WARNING( nsPrintfCString("[VULKAN] Found %u DRM modifiers for format 0x%x", modList.drmFormatModifierCount, (unsigned)formatForModifiers) .get()); for (uint32_t i = 0; i < modList.drmFormatModifierCount; i++) { NS_WARNING( nsPrintfCString("[VULKAN] 0x%llx (planes=%u, features=0x%x)", (unsigned long long)modProps[i].drmFormatModifier, modProps[i].drmFormatModifierPlaneCount, modProps[i].drmFormatModifierTilingFeatures) .get()); if (aCompositorMods) { if (!aCompositorMods->Contains(modProps[i].drmFormatModifier)) { FFMPEGV_LOG( "[VULKAN] modifier 0x%llx: not supported by compositor", (unsigned long long)modProps[i].drmFormatModifier); continue; } } else if (modProps[i].drmFormatModifier != DRM_FORMAT_MOD_LINEAR) { FFMPEGV_LOG( "[VULKAN] modifier 0x%llx: skipped without compositor list", (unsigned long long)modProps[i].drmFormatModifier); continue; } if (!(modProps[i].drmFormatModifierTilingFeatures & (VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT))) { FFMPEGV_LOG( "[VULKAN] modifier 0x%llx: skipped, missing transfer " "src/dst tiling features", (unsigned long long)modProps[i].drmFormatModifier); continue; } if (mGetPhysicalDeviceImageFormatProperties2) { VkPhysicalDeviceImageDrmFormatModifierInfoEXT modInfo = {}; modInfo.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT; modInfo.drmFormatModifier = modProps[i].drmFormatModifier; modInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; VkPhysicalDeviceExternalImageFormatInfo extFormatInfo = {}; extFormatInfo.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO; extFormatInfo.pNext = &modInfo; extFormatInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; VkPhysicalDeviceImageFormatInfo2 formatInfo = {}; formatInfo.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2; formatInfo.pNext = &extFormatInfo; formatInfo.format = formatForModifiers; formatInfo.type = VK_IMAGE_TYPE_2D; formatInfo.tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; formatInfo.usage = aImageUsages; formatInfo.flags = 0; VkExternalImageFormatProperties extProps2 = { .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR, }; VkImageFormatProperties2 props2 = {}; props2.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2; props2.pNext = &extProps2; VkResult isFormatPropsSupported = mGetPhysicalDeviceImageFormatProperties2(aPhysDev, &formatInfo, &props2); const bool exportRequiresDedicated = !!(extProps2.externalMemoryProperties.externalMemoryFeatures & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT); FFMPEGV_LOG("modifier 0x%llx: DEDICATED_ONLY_BIT: %s", (unsigned long long)modProps[i].drmFormatModifier, exportRequiresDedicated ? "YES" : "NO"); FFMPEGV_LOG("modifier 0x%llx: DMA_BUF_BIT_EXT: %s", (unsigned long long)modProps[i].drmFormatModifier, extProps2.externalMemoryProperties.compatibleHandleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT ? "YES" : "NO"); FFMPEGV_LOG( "[VULKAN] modifier 0x%llx: image format props supported for " "usage 0x%x? %s", (unsigned long long)modProps[i].drmFormatModifier, (unsigned)aImageUsages, isFormatPropsSupported == VK_SUCCESS ? "YES" : "NO"); if (isFormatPropsSupported != VK_SUCCESS) { continue; } mExportRequiresDedicatedByModifier.InsertOrUpdate( modProps[i].drmFormatModifier, exportRequiresDedicated); } mDrmModifiers.push_back(modProps[i].drmFormatModifier); } } } if (mDrmModifiers.empty()) { mDrmModifiers.push_back(DRM_FORMAT_MOD_LINEAR); FFMPEGV_LOG("[VULKAN] No suitable modifiers found, using LINEAR"); } // NVIDIA: query may not expose tiled modifiers, add known-working one if RDD // and GPU share the same device (only when we had a real compositor list). if (aCompositorMods && mNegotiatedCompositorDecoderVendorID == 0x10de && mDecoderMatchesCompositor && mDrmModifiers[0] == DRM_FORMAT_MOD_LINEAR) { mDrmModifiers[0] = DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 2, 6, 4); } FFMPEGV_LOG("[VULKAN] Using %zu modifiers, first=0x%llx", mDrmModifiers.size(), (unsigned long long)mDrmModifiers[0]); } static void* sVulkanLib = nullptr; static bool sVulkanEnumerated = false; static bool PhysicalDeviceHasVulkanVideoDecodeStack( PFN_vkEnumerateDeviceExtensionProperties aEnumerateExt, VkPhysicalDevice aDevice, const char* aDeviceName) { if (!aEnumerateExt) { return false; } static const char* const kRequired[] = {"VK_KHR_video_queue", "VK_KHR_video_decode_queue"}; uint32_t extCount = 0; if (aEnumerateExt(aDevice, nullptr, &extCount, nullptr) != VK_SUCCESS || extCount == 0) { return false; } std::vector props(extCount); if (aEnumerateExt(aDevice, nullptr, &extCount, props.data()) != VK_SUCCESS) { return false; } for (const char* req : kRequired) { bool found = false; for (uint32_t i = 0; i < extCount; i++) { if (strcmp(props[i].extensionName, req) == 0) { found = true; break; } } if (!found) { FFMPEGV_LOG("Skipping %s: missing required extension %s", aDeviceName, req); return false; } } return true; } bool FFmpegVideoDecoder::FFmpegVulkanVideoDecoder:: SelectVulkanDecoderPhysicalDevice(const StaticMutexAutoLock& aProofOfLock, const nsCString& aRendererNode) { uint32_t rendererDrmMajor = 0, rendererDrmMinor = 0; # if defined(MOZ_WIDGET_GTK) if (!aRendererNode.IsEmpty()) { struct stat st = {}; if (stat(aRendererNode.get(), &st) == 0) { rendererDrmMajor = major(st.st_rdev); rendererDrmMinor = minor(st.st_rdev); FFMPEGV_LOG("Renderer device from GPU: %s (major=%u, minor=%u)", aRendererNode.get(), rendererDrmMajor, rendererDrmMinor); } else { FFMPEGV_LOG("Renderer device from GPU: %s - stat() failed (errno=%d)", aRendererNode.get(), errno); } } else { // Empty when renderer is llvmpipe or glxtest failed to detect a DRM device FFMPEGV_LOG("Renderer device from GPU: empty (gfxVars::DrmRenderDevice)"); } # endif const bool useCache = (rendererDrmMajor == 0 && rendererDrmMinor == 0); if (!sVulkanEnumerated || !useCache) { if (useCache) { sVulkanEnumerated = true; } if (!sVulkanLib) { sVulkanLib = dlopen("libvulkan.so.1", RTLD_LAZY); if (!sVulkanLib) { FFMPEGV_LOG("Failed to load libvulkan.so.1"); return false; } } auto vkGetInstanceProcAddr = (PFN_vkGetInstanceProcAddr)dlsym(sVulkanLib, "vkGetInstanceProcAddr"); if (!vkGetInstanceProcAddr) { FFMPEGV_LOG("Failed to get vkGetInstanceProcAddr"); return false; } auto vkCreateInstance = (PFN_vkCreateInstance)vkGetInstanceProcAddr( nullptr, "vkCreateInstance"); if (!vkCreateInstance) { FFMPEGV_LOG("Failed to get vkCreateInstance"); return false; } VkApplicationInfo appInfo = {}; appInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; appInfo.apiVersion = VK_API_VERSION_1_3; VkInstanceCreateInfo createInfo = {}; createInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; createInfo.pApplicationInfo = &appInfo; VkInstance instance = VK_NULL_HANDLE; if (vkCreateInstance(&createInfo, nullptr, &instance) != VK_SUCCESS) { FFMPEGV_LOG("Failed to create Vulkan instance"); return false; } auto vkDestroyInstance = (PFN_vkDestroyInstance)vkGetInstanceProcAddr( instance, "vkDestroyInstance"); auto destroyInstance = MakeScopeExit([&] { if (vkDestroyInstance && instance) { vkDestroyInstance(instance, nullptr); } }); auto vkEnumeratePhysicalDevices = (PFN_vkEnumeratePhysicalDevices)vkGetInstanceProcAddr( instance, "vkEnumeratePhysicalDevices"); auto vkGetPhysicalDeviceProperties = (PFN_vkGetPhysicalDeviceProperties)vkGetInstanceProcAddr( instance, "vkGetPhysicalDeviceProperties"); auto vkGetPhysicalDeviceProperties2 = (PFN_vkGetPhysicalDeviceProperties2)vkGetInstanceProcAddr( instance, "vkGetPhysicalDeviceProperties2"); auto vkEnumerateDeviceExtensionProperties = (PFN_vkEnumerateDeviceExtensionProperties)vkGetInstanceProcAddr( instance, "vkEnumerateDeviceExtensionProperties"); if (!vkEnumeratePhysicalDevices || !vkGetPhysicalDeviceProperties || !vkEnumerateDeviceExtensionProperties) { NS_WARNING("Failed to get Vulkan enumeration functions"); return false; } uint32_t count = 0; vkEnumeratePhysicalDevices(instance, &count, nullptr); if (count == 0) { FFMPEGV_LOG("No Vulkan devices found"); return false; } std::vector devices(count); vkEnumeratePhysicalDevices(instance, &count, devices.data()); // Collect valid devices (non-CPU, Vulkan 1.3+), sorted by type (discrete // first). std::vector> validDevices; for (uint32_t i = 0; i < count; i++) { VkPhysicalDeviceProperties p = {}; bool isDecoderMatchesRendererFound = false; if (rendererDrmMajor && rendererDrmMinor && vkGetPhysicalDeviceProperties2) { VkPhysicalDeviceDrmPropertiesEXT drmProps = {}; drmProps.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT; VkPhysicalDeviceProperties2 props2 = {}; props2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; props2.pNext = &drmProps; vkGetPhysicalDeviceProperties2(devices[i], &props2); p = props2.properties; isDecoderMatchesRendererFound = (drmProps.hasRender && rendererDrmMajor == drmProps.renderMajor && rendererDrmMinor == drmProps.renderMinor) || (drmProps.hasPrimary && rendererDrmMajor == drmProps.primaryMajor && rendererDrmMinor == drmProps.primaryMinor); } else { vkGetPhysicalDeviceProperties(devices[i], &p); } if (p.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU) { uint32_t major = VK_API_VERSION_MAJOR(p.apiVersion); uint32_t minor = VK_API_VERSION_MINOR(p.apiVersion); if (major > 1 || (major == 1 && minor >= 3)) { if (!PhysicalDeviceHasVulkanVideoDecodeStack( vkEnumerateDeviceExtensionProperties, devices[i], p.deviceName)) { continue; } validDevices.push_back( std::make_pair(p, isDecoderMatchesRendererFound)); } } } auto deviceTypePriority = [](VkPhysicalDeviceType t) -> int { switch (t) { case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return 3; case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return 2; case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return 1; default: return 0; } }; std::sort( validDevices.begin(), validDevices.end(), [&deviceTypePriority](const auto& p1, const auto& p2) { if (p1.second != p2.second) { return p1.second > p2.second; // renderer-matching device first } return deviceTypePriority(p1.first.deviceType) > deviceTypePriority(p2.first.deviceType); // discrete first }); if (validDevices.empty()) { FFMPEGV_LOG( "No suitable Vulkan device found (need 1.3+, non-CPU, " "VK_KHR_video_queue + VK_KHR_video_decode_queue)"); return false; } memcpy(mNegotiatedVulkanDeviceName, validDevices[0].first.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); mNegotiatedCompositorDecoderVendorID = validDevices[0].first.vendorID; mNegotiatedCompositorDecoderDeviceID = validDevices[0].first.deviceID; mDecoderMatchesCompositor = validDevices[0].second; FFMPEGV_LOG( "Selected Vulkan device for video decoding: %s (vendorID=0x%x, " "deviceID=0x%x), matches renderer: %s", mNegotiatedVulkanDeviceName, mNegotiatedCompositorDecoderVendorID, mNegotiatedCompositorDecoderDeviceID, mDecoderMatchesCompositor ? "true" : "false"); } return true; } bool FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::InitCtx( VkDevice aDevice, VkPhysicalDevice aPhysDev, PFN_vkGetInstanceProcAddr aGetProcAddr, VkInstance aInstance, uint32_t aCopyQueueFamilyIndex) { // Load instance-level functions once if (!mGetDeviceProcAddr) { LoadInstanceFunctions(aGetProcAddr, aInstance, aPhysDev); } // Reload mDevice-level functions when mDevice changes if (mDevice != aDevice) { // Cleanup old resources before switching mDevice PFN_vkGetDeviceProcAddr savedGetDeviceProcAddr = mGetDeviceProcAddr; PFN_vkGetPhysicalDeviceMemoryProperties savedGetMemProps = mGetPhysicalDeviceMemoryProperties; Cleanup(); mGetDeviceProcAddr = savedGetDeviceProcAddr; mGetPhysicalDeviceMemoryProperties = savedGetMemProps; LoadDeviceFunctions(aDevice); if (!IsLoaded()) { FFMPEGV_LOG("Failed to load required Vulkan device functions"); return false; } mDevice = aDevice; // Instead of forcing LINEAR tiling when the selected Vulkan device does // not match the renderer device, we negotiate DRM modifiers via IPC: // the decoder queries supported modifiers from the compositor and uses // the intersection. This removes the need for a forceLinear flag. uint32_t copyQueueFamilyIndex = aCopyQueueFamilyIndex; uint32_t transferOnlyQueueCount = 0; int32_t transferOnlyQueueFamilyIndex = -1; if (mGetPhysicalDeviceQueueFamilyProperties) { uint32_t queueFamilyCount = 0; mGetPhysicalDeviceQueueFamilyProperties(aPhysDev, &queueFamilyCount, nullptr); AutoTArray props; if (queueFamilyCount > 0) { props.SetLength(queueFamilyCount); mGetPhysicalDeviceQueueFamilyProperties(aPhysDev, &queueFamilyCount, props.Elements()); for (uint32_t i = 0; i < queueFamilyCount; i++) { if (props[i].queueCount > 0 && (props[i].queueFlags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)) == 0 && (props[i].queueFlags & VK_QUEUE_TRANSFER_BIT)) { copyQueueFamilyIndex = i; transferOnlyQueueFamilyIndex = static_cast(i); transferOnlyQueueCount = static_cast(props[i].queueCount); break; } } } } if (transferOnlyQueueFamilyIndex >= 0) { mQueueFamilyIndex = transferOnlyQueueFamilyIndex; mCopyQueueCount = transferOnlyQueueCount; } else { mQueueFamilyIndex = copyQueueFamilyIndex; } mCopyQueueCount = std::max(1u, mCopyQueueCount); mCopyQueue.SetLength(mCopyQueueCount); mCopyCmdPool.SetLength(mCopyQueueCount); mCopyCmdBuf.SetLength(mCopyQueueCount); mCopyFence.SetLength(mCopyQueueCount); VkCommandPoolCreateInfo poolInfo = {}; poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; poolInfo.queueFamilyIndex = mQueueFamilyIndex; auto cleanUp = MakeScopeExit([&] { Cleanup(); }); for (uint32_t qi = 0; qi < mCopyQueueCount; qi++) { VkResult poolRes = mCreateCommandPool(aDevice, &poolInfo, nullptr, &mCopyCmdPool[qi]); if (poolRes != VK_SUCCESS && copyQueueFamilyIndex != aCopyQueueFamilyIndex) { copyQueueFamilyIndex = aCopyQueueFamilyIndex; mQueueFamilyIndex = copyQueueFamilyIndex; poolInfo.queueFamilyIndex = mQueueFamilyIndex; poolRes = mCreateCommandPool(aDevice, &poolInfo, nullptr, &mCopyCmdPool[qi]); } if (poolRes != VK_SUCCESS) { FFMPEGV_LOG("Failed to create Vulkan command pool for queue %u", qi); return false; } mGetDeviceQueue(aDevice, mQueueFamilyIndex, qi, &mCopyQueue[qi]); VkCommandBufferAllocateInfo cmdAllocInfo = {}; cmdAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; cmdAllocInfo.commandPool = mCopyCmdPool[qi]; cmdAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; cmdAllocInfo.commandBufferCount = 1; if (mAllocateCommandBuffers(aDevice, &cmdAllocInfo, &mCopyCmdBuf[qi]) != VK_SUCCESS) { FFMPEGV_LOG("Failed to allocate Vulkan command buffer for queue %u", qi); return false; } VkFenceCreateInfo fenceInfo = {}; fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT; if (mCreateFence(aDevice, &fenceInfo, nullptr, &mCopyFence[qi]) != VK_SUCCESS) { FFMPEGV_LOG("Failed to create Vulkan copy fence for queue %u", qi); return false; } } cleanUp.release(); NS_WARNING(nsPrintfCString("[VULKAN] Initialized Vulkan Firefox context, " "vkGetMemoryFdKHR=%p\n", (void*)mGetMemoryFdKHR) .get()); } return mDevice != nullptr; } MediaResult FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::InitCopyRingBuffer( uint32_t aWidth, uint32_t aHeight, AVPixelFormat aSwFormat, AVBufferRef* aVulkanDevCtx) { VkPhysicalDevice physDev = ((AVVulkanDeviceContext*)((AVHWDeviceContext*)aVulkanDevCtx->data)->hwctx) ->phys_dev; if (mWidth == aWidth && mHeight == aHeight) { return MediaResult(NS_OK); } for (int buf = 0; buf < kNumBuffers; buf++) { if (mNv12BaseFd[buf] >= 0) { close(mNv12BaseFd[buf]); mNv12BaseFd[buf] = -1; } if (mNv12Image[buf]) { mDestroyImage(mDevice, mNv12Image[buf], nullptr); mNv12Image[buf] = nullptr; } if (mNv12Mem[buf]) { mFreeMemory(mDevice, mNv12Mem[buf], nullptr); mNv12Mem[buf] = nullptr; } } VkExternalMemoryImageCreateInfo extImgInfo = {}; extImgInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; extImgInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; VkExportMemoryAllocateInfo exportInfo = {}; exportInfo.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO; exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; bool useP010 = (aSwFormat == AV_PIX_FMT_P010) || (aSwFormat == AV_PIX_FMT_P016); # if LIBAVCODEC_VERSION_MAJOR >= 60 useP010 = useP010 || (aSwFormat == AV_PIX_FMT_P012); # endif const VkFormat vkFormat = useP010 ? VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 : VK_FORMAT_G8_B8R8_2PLANE_420_UNORM; VkImageDrmFormatModifierListCreateInfoEXT drmModInfo = {}; drmModInfo.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT; drmModInfo.drmFormatModifierCount = mDrmModifiers.size(); drmModInfo.pDrmFormatModifiers = mDrmModifiers.data(); extImgInfo.pNext = &drmModInfo; VkPhysicalDeviceMemoryProperties memProps; mGetPhysicalDeviceMemoryProperties(physDev, &memProps); int buf = 0; // Clean up any partially-allocated buffers on failure. auto cleanup = mozilla::MakeScopeExit([&] { for (int b = 0; b <= buf; b++) { if (mNv12BaseFd[b] >= 0) { close(mNv12BaseFd[b]); mNv12BaseFd[b] = -1; } if (mNv12Mem[b]) { mFreeMemory(mDevice, mNv12Mem[b], nullptr); mNv12Mem[b] = nullptr; } if (mNv12Image[b]) { mDestroyImage(mDevice, mNv12Image[b], nullptr); mNv12Image[b] = nullptr; } } }); VkMemoryRequirements memReqs; bool useDedicated = true; for (buf = 0; buf < kNumBuffers; buf++) { VkImageCreateInfo imgInfo = {}; imgInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; imgInfo.pNext = &extImgInfo; imgInfo.imageType = VK_IMAGE_TYPE_2D; imgInfo.format = vkFormat; imgInfo.extent = {aWidth, aHeight, 1}; imgInfo.mipLevels = 1; imgInfo.arrayLayers = 1; imgInfo.samples = VK_SAMPLE_COUNT_1_BIT; imgInfo.tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; imgInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; imgInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; imgInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; VkResult res = mCreateImage(mDevice, &imgInfo, nullptr, &mNv12Image[buf]); if (res != VK_SUCCESS) { NS_WARNING( nsPrintfCString( "[VULKAN] ERROR: Failed to create NV12 image[%d]: %d\n", buf, res) .get()); return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Failed to create NV12 image")); } if (buf == 0 && mGetImageDrmFormatModifierPropertiesEXT) { VkImageDrmFormatModifierPropertiesEXT modProps = {}; modProps.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT; if (mGetImageDrmFormatModifierPropertiesEXT(mDevice, mNv12Image[0], &modProps) == VK_SUCCESS) { mDrmModifier = modProps.drmFormatModifier; NS_WARNING(nsPrintfCString("[VULKAN] DRM modifier: 0x%llx\n", (unsigned long long)mDrmModifier) .get()); } } if (buf == 0) { if (mGetImageMemoryRequirements2) { VkMemoryDedicatedRequirements dedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; VkMemoryRequirements2 memReqs2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedReq}; VkImageMemoryRequirementsInfo2 reqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, nullptr, mNv12Image[buf]}; mGetImageMemoryRequirements2(mDevice, &reqInfo, &memReqs2); memReqs = memReqs2.memoryRequirements; useDedicated = dedReq.prefersDedicatedAllocation || dedReq.requiresDedicatedAllocation; } else { mGetImageMemoryRequirements(mDevice, mNv12Image[buf], &memReqs); } const auto entry = mExportRequiresDedicatedByModifier.Lookup(mDrmModifier); const bool exportRequiresDedicated = entry ? entry.Data() : true; useDedicated = useDedicated || exportRequiresDedicated; } uint32_t memTypeIndex = UINT32_MAX; for (uint32_t i = 0; i < memProps.memoryTypeCount; i++) { if (memReqs.memoryTypeBits & (1 << i)) { memTypeIndex = i; break; } } if (memTypeIndex == UINT32_MAX) { mDestroyImage(mDevice, mNv12Image[buf], nullptr); mNv12Image[buf] = nullptr; return MediaResult( NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("No compatible memory type for NV12 image")); } VkMemoryDedicatedAllocateInfo dedicatedInfo = {}; dedicatedInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; dedicatedInfo.pNext = &exportInfo; dedicatedInfo.image = mNv12Image[buf]; VkMemoryAllocateInfo allocInfo = {}; allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; allocInfo.pNext = useDedicated ? (void*)&dedicatedInfo : (void*)&exportInfo; allocInfo.allocationSize = memReqs.size; allocInfo.memoryTypeIndex = memTypeIndex; res = mAllocateMemory(mDevice, &allocInfo, nullptr, &mNv12Mem[buf]); if (res != VK_SUCCESS) { mDestroyImage(mDevice, mNv12Image[buf], nullptr); mNv12Image[buf] = nullptr; return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Failed to alloc NV12 memory")); } res = mBindImageMemory(mDevice, mNv12Image[buf], mNv12Mem[buf], 0); if (res != VK_SUCCESS) { mFreeMemory(mDevice, mNv12Mem[buf], nullptr); mDestroyImage(mDevice, mNv12Image[buf], nullptr); mNv12Mem[buf] = nullptr; mNv12Image[buf] = nullptr; return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Failed to bind NV12 memory")); } mTotalSize = memReqs.size; VkImageSubresource subresY = {VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT, 0, 0}; VkSubresourceLayout layoutY = {}; mGetImageSubresourceLayout(mDevice, mNv12Image[buf], &subresY, &layoutY); mYPitch = layoutY.rowPitch; VkImageSubresource subresUV = {VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT, 0, 0}; VkSubresourceLayout layoutUV = {}; mGetImageSubresourceLayout(mDevice, mNv12Image[buf], &subresUV, &layoutUV); mUvPitch = layoutUV.rowPitch; mUvOffset = layoutUV.offset; VkMemoryGetFdInfoKHR fdInfo = {}; fdInfo.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR; fdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT; fdInfo.memory = mNv12Mem[buf]; res = mGetMemoryFdKHR(mDevice, &fdInfo, &mNv12BaseFd[buf]); if (res != VK_SUCCESS) { mFreeMemory(mDevice, mNv12Mem[buf], nullptr); mDestroyImage(mDevice, mNv12Image[buf], nullptr); mNv12Mem[buf] = nullptr; mNv12Image[buf] = nullptr; return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Failed to export NV12 FD")); } NS_WARNING(nsPrintfCString("[VULKAN] Created NV12 buffer[%d]: fd=%d", buf, mNv12BaseFd[buf]) .get()); } NS_WARNING(nsPrintfCString( "[VULKAN] NV12 images: %ux%u, Y pitch=%u, UV offset=%zu, UV " "pitch=%u, total=%zu, modifier=0x%llx", aWidth, aHeight, mYPitch, mUvOffset, mUvPitch, mTotalSize, (unsigned long long)mDrmModifier) .get()); mWidth = aWidth; mHeight = aHeight; mCurrentBuffer = 0; cleanup.release(); return MediaResult(NS_OK); } MediaResult FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::InitExternalSemaphores( AVBufferRef* aVulkanDevCtx) { if (!mCreateSemaphore) { return MediaResult(NS_OK); } bool opaqueFdSupported = false; if (mGetPhysicalDeviceExternalSemaphoreProperties) { auto* devCtx = (AVHWDeviceContext*)aVulkanDevCtx->data; auto* vkDevCtx = (AVVulkanDeviceContext*)devCtx->hwctx; VkPhysicalDevice physDev = vkDevCtx->phys_dev; VkPhysicalDeviceExternalSemaphoreInfo extSemInfo = {}; extSemInfo.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO; extSemInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; VkExternalSemaphoreProperties extSemProps = {}; extSemProps.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES; mGetPhysicalDeviceExternalSemaphoreProperties(physDev, &extSemInfo, &extSemProps); opaqueFdSupported = (extSemProps.compatibleHandleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) != 0; } VkExportSemaphoreCreateInfo exportSemInfo = {}; exportSemInfo.sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO; exportSemInfo.handleTypes = (opaqueFdSupported && mDecoderMatchesCompositor) // && // mCompositorSupportsOpaqueFdSemaphore) ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; mSemHandleType = exportSemInfo.handleTypes; VkSemaphoreTypeCreateInfo semTypeInfo = {}; semTypeInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO; semTypeInfo.semaphoreType = VK_SEMAPHORE_TYPE_BINARY; semTypeInfo.pNext = &exportSemInfo; bool created[kNumBuffers] = {}; for (int buf = 0; buf < kNumBuffers; buf++) { if (mCopyDoneSem[buf] != VK_NULL_HANDLE) { continue; } VkSemaphoreCreateInfo semInfo = {}; semInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; semInfo.pNext = &semTypeInfo; VkResult res = mCreateSemaphore(mDevice, &semInfo, nullptr, &mCopyDoneSem[buf]); if (res != VK_SUCCESS) { for (int b = 0; b < kNumBuffers; b++) { if (created[b] && mCopyDoneSem[b] && mDestroySemaphore) { mDestroySemaphore(mDevice, mCopyDoneSem[b], nullptr); mCopyDoneSem[b] = VK_NULL_HANDLE; } } return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Failed to create copyDone semaphore")); } created[buf] = true; } return MediaResult(NS_OK); } MediaResult FFmpegVideoDecoder::FFmpegVulkanVideoDecoder::PrepareImageToDRM( AVFrame* aSrcFrame, int* aOutFd, size_t* aOutSize, uint32_t* aOutYPitch, uint32_t* aOutUVPitch, size_t* aOutUVOffset, AVBufferRef* aVulkanDevCtx, VideoFramePool* aFramePool, int32_t* aOutBufIdx, bool aIsCopy) { uint32_t width = aSrcFrame->width; uint32_t height = aSrcFrame->height; uint32_t uvWidth = (width + 1) / 2; uint32_t uvHeight = (height + 1) / 2; AVVkFrame* srcVkFrame = (AVVkFrame*)aSrcFrame->data[0]; if (!srcVkFrame) { return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Missing source Vulkan frame")); } // Lock the frame while accessing it (FFmpeg is threaded) AVHWFramesContext* framesCtx = (AVHWFramesContext*)aSrcFrame->hw_frames_ctx->data; if (framesCtx->sw_format != AV_PIX_FMT_NV12 && framesCtx->sw_format != AV_PIX_FMT_P010 && # if LIBAVCODEC_VERSION_MAJOR >= 60 framesCtx->sw_format != AV_PIX_FMT_P012 && # endif framesCtx->sw_format != AV_PIX_FMT_P016) { return MediaResult( NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL( "Vulkan copy only supports NV12/P010/P012/P016 decode output")); } AVVulkanFramesContext* vkFramesCtx = (AVVulkanFramesContext*)framesCtx->hwctx; vkFramesCtx->lock_frame(framesCtx, srcVkFrame); auto unlockGuard = MakeScopeExit([&] { vkFramesCtx->unlock_frame(framesCtx, srcVkFrame); }); if (mWidth != width || mHeight != height) { if (aIsCopy) { MediaResult initRes = InitCopyRingBuffer( width, height, framesCtx->sw_format, aVulkanDevCtx); if (NS_FAILED(initRes)) { return initRes; } } MediaResult semRes = InitExternalSemaphores(aVulkanDevCtx); if (NS_FAILED(semRes)) { return semRes; } } int bufIdx = -1; int i = 0; int retries = 0; constexpr int kMaxRetries = 1000; // 1000 × 1 ms ≈ 1 s do { if (!aFramePool->IsVulkanFrameSlotInUseByRenderer(i)) { bufIdx = i; break; } if (++i >= kNumBuffers) { i = 0; if (retries++ >= kMaxRetries) { FFMPEGV_LOG("No free Vulkan frame copy slot after %d retries", kMaxRetries); return NS_ERROR_DOM_MEDIA_DECODE_ERR; } PR_Sleep(PR_MillisecondsToInterval(1)); } } while (bufIdx < 0); const uint32_t copySlot = aIsCopy ? (mCopyQueueRoundRobin++ % mCopyQueueCount) : 0; if (aIsCopy) { const uint64_t kFenceWaitNs = 100 * 1000 * 1000; // 100 ms VkResult waitRes = mWaitForFences(mDevice, 1, &mCopyFence[copySlot], VK_TRUE, kFenceWaitNs); if (waitRes == VK_TIMEOUT) { NS_WARNING( "[VULKAN] Copy fence wait timed out; previous copy may be stuck."); return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Vulkan copy fence wait timed out")); } if (waitRes != VK_SUCCESS) { return MediaResult( NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Vulkan waitForFences failed: %d", waitRes)); } mResetFences(mDevice, 1, &mCopyFence[copySlot]); } if (aIsCopy) { VkCommandBufferBeginInfo beginInfo = {}; beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; mBeginCommandBuffer(mCopyCmdBuf[copySlot], &beginInfo); // Barrier for source and destination (both NV12 multi-planar). // Use one barrier per plane so aspectMask has a single bit (avoids driver // bug with combined PLANE_0|PLANE_1 in layout transition/clear paths). VkImageMemoryBarrier barriers[4] = {}; for (auto& barrier : barriers) { barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; } // Source image: plane 0 barriers[0].image = srcVkFrame->img[0]; barriers[0].oldLayout = (VkImageLayout)srcVkFrame->layout[0]; barriers[0].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; barriers[0].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; barriers[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barriers[0].subresourceRange = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 1, 0, 1}; // Source image: plane 1 barriers[1].image = srcVkFrame->img[0]; barriers[1].oldLayout = (VkImageLayout)srcVkFrame->layout[0]; barriers[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; barriers[1].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barriers[1].subresourceRange = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 1, 0, 1}; // Destination NV12 image: plane 0 barriers[2].image = mNv12Image[bufIdx]; barriers[2].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; barriers[2].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[2].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barriers[2].subresourceRange = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 1, 0, 1}; // Destination NV12 image: plane 1 barriers[3].image = mNv12Image[bufIdx]; barriers[3].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; barriers[3].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[3].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barriers[3].subresourceRange = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 1, 0, 1}; mCmdPipelineBarrier( mCopyCmdBuf[copySlot], VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 4, barriers); // Copy Y plane: src PLANE_0 -> dst PLANE_0 VkImageCopy yRegion = {}; yRegion.srcSubresource = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 0, 1}; yRegion.dstSubresource = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 0, 1}; yRegion.extent = {width, height, 1}; mCmdCopyImage(mCopyCmdBuf[copySlot], srcVkFrame->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, mNv12Image[bufIdx], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &yRegion); // Copy UV plane: src PLANE_1 -> dst PLANE_1 VkImageCopy uvRegion = {}; uvRegion.srcSubresource = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 0, 1}; uvRegion.dstSubresource = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 0, 1}; uvRegion.extent = {uvWidth, uvHeight, 1}; mCmdCopyImage(mCopyCmdBuf[copySlot], srcVkFrame->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, mNv12Image[bufIdx], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &uvRegion); const VkImageLayout srcRestoreLayout = (VkImageLayout)srcVkFrame->layout[0]; barriers[0].image = srcVkFrame->img[0]; barriers[0].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; barriers[0].newLayout = srcRestoreLayout; barriers[0].srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barriers[0].dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; barriers[0].subresourceRange = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 1, 0, 1}; barriers[1].image = srcVkFrame->img[0]; barriers[1].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; barriers[1].newLayout = srcRestoreLayout; barriers[1].srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barriers[1].dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; barriers[1].subresourceRange = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 1, 0, 1}; mCmdPipelineBarrier(mCopyCmdBuf[copySlot], VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); barriers[0].image = mNv12Image[bufIdx]; barriers[0].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[0].newLayout = VK_IMAGE_LAYOUT_GENERAL; barriers[0].srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barriers[0].dstAccessMask = 0; barriers[0].subresourceRange = {VK_IMAGE_ASPECT_PLANE_0_BIT, 0, 1, 0, 1}; barriers[1].image = mNv12Image[bufIdx]; barriers[1].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[1].newLayout = VK_IMAGE_LAYOUT_GENERAL; barriers[1].srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barriers[1].dstAccessMask = 0; barriers[1].subresourceRange = {VK_IMAGE_ASPECT_PLANE_1_BIT, 0, 1, 0, 1}; mCmdPipelineBarrier(mCopyCmdBuf[copySlot], VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); mEndCommandBuffer(mCopyCmdBuf[copySlot]); } const uint64_t decodeWaitValue = srcVkFrame->sem_value[0]; const uint64_t decodeSignalValue = decodeWaitValue + 1; srcVkFrame->sem_value[0] = decodeSignalValue; // copyDoneSem is binary (we only signal it). The compositor waits on it (in // MaybeSemaphoreWait when Lock()ing the texture) before using the buffer; // when the slot is released the compositor is done with it and we can reuse. // We never wait on it here (VUID-vkQueueSubmit-pWaitSemaphores-03238). // We only wait on the decode timeline semaphore when present. VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; VkSemaphore signalSems[2] = {mCopyDoneSem[bufIdx], srcVkFrame->sem[0]}; uint64_t signalValues[2] = {0, decodeSignalValue}; // binary value ignored uint32_t signalCount = 1; uint32_t waitCount = 0; const VkSemaphore* pWaitSemaphores = &srcVkFrame->sem[0]; const uint64_t* pWaitValues = &decodeWaitValue; const VkPipelineStageFlags* pWaitStages = &waitStage; if (srcVkFrame->sem[0] != VK_NULL_HANDLE && srcVkFrame->sem_value[0] > 0) { signalCount = 2; waitCount = 1; } VkTimelineSemaphoreSubmitInfo timelineInfo = {}; timelineInfo.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO; timelineInfo.waitSemaphoreValueCount = waitCount; timelineInfo.pWaitSemaphoreValues = pWaitValues; timelineInfo.signalSemaphoreValueCount = signalCount; timelineInfo.pSignalSemaphoreValues = signalValues; VkSubmitInfo submitInfo = {}; submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submitInfo.pNext = &timelineInfo; submitInfo.waitSemaphoreCount = waitCount; submitInfo.pWaitSemaphores = pWaitSemaphores; submitInfo.pWaitDstStageMask = pWaitStages; submitInfo.signalSemaphoreCount = signalCount; submitInfo.pSignalSemaphores = signalSems; submitInfo.commandBufferCount = aIsCopy ? 1u : 0u; submitInfo.pCommandBuffers = &mCopyCmdBuf[copySlot]; AVHWDeviceContext* devCtx = (AVHWDeviceContext*)aVulkanDevCtx->data; AVVulkanDeviceContext* vkCtx = (AVVulkanDeviceContext*)devCtx->hwctx; # if !defined(FF_API_VULKAN_SYNC_QUEUES) || FF_API_VULKAN_SYNC_QUEUES const uint32_t qf = static_cast(mQueueFamilyIndex); vkCtx->lock_queue(devCtx, qf, copySlot); # endif VkResult submitRes = mQueueSubmit(mCopyQueue[copySlot], 1, &submitInfo, mCopyFence[copySlot]); # if !defined(FF_API_VULKAN_SYNC_QUEUES) || FF_API_VULKAN_SYNC_QUEUES vkCtx->unlock_queue(devCtx, qf, copySlot); # endif if (submitRes != VK_SUCCESS) { NS_WARNING( nsPrintfCString("[VULKAN] ERROR: queueSubmit failed: %d", submitRes) .get()); return MediaResult( NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Vulkan queue submit failed: %d", submitRes)); } // Export with the handle type chosen at semaphore creation (Vulkan physical // mDevice OPAQUE_FD support + same mDevice; no compositor dependency). if (mGetSemaphoreFdKHR) { if (mCopyDoneSemFd[bufIdx] >= 0) { close(mCopyDoneSemFd[bufIdx]); mCopyDoneSemFd[bufIdx] = -1; } VkSemaphoreGetFdInfoKHR fdInfo = {}; fdInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR; fdInfo.semaphore = mCopyDoneSem[bufIdx]; fdInfo.handleType = (VkExternalSemaphoreHandleTypeFlagBits)mSemHandleType; if (mGetSemaphoreFdKHR(mDevice, &fdInfo, &mCopyDoneSemFd[bufIdx]) != VK_SUCCESS) { mCopyDoneSemFd[bufIdx] = -1; } } // Do not block decoder on vkWaitSemaphores. The copy-done semaphore is // exported to an fd and passed to the surface; the compositor waits on it in // MaybeSemaphoreWait when using the buffer. if (aIsCopy) { *aOutFd = mNv12BaseFd[bufIdx]; *aOutSize = mTotalSize; *aOutYPitch = mYPitch; *aOutUVPitch = mUvPitch; *aOutUVOffset = mUvOffset; } if (aOutBufIdx) { *aOutBufIdx = bufIdx; } return NS_OK; } # endif // LIBAVCODEC_VERSION_MAJOR >= 60 && !defined(FFVPX_VERSION) #endif // defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK) } // namespace mozilla