use objc2::{ rc::{autoreleasepool, Retained}, runtime::ProtocolObject, }; use objc2_foundation::{NSRange, NSString, NSUInteger}; use objc2_metal::{ MTLAccelerationStructure, MTLAccelerationStructureCommandEncoder, MTLBlitCommandEncoder, MTLBlitPassDescriptor, MTLBuffer, MTLCommandBuffer, MTLCommandBufferStatus, MTLCommandEncoder, MTLCommandQueue, MTLComputeCommandEncoder, MTLComputePassDescriptor, MTLCounterDontSample, MTLDevice, MTLLoadAction, MTLPrimitiveType, MTLRenderCommandEncoder, MTLRenderPassDescriptor, MTLResidencySet, MTLResidencySetDescriptor, MTLSamplerState, MTLScissorRect, MTLSize, MTLStoreAction, MTLTexture, MTLVertexAmplificationViewMapping, MTLViewport, MTLVisibilityResultMode, }; use super::{ adapter::{self, VERTEX_BUFFER_SLOT_START}, conv, TimestampQuerySupport, }; use crate::CommandEncoder as _; use alloc::{ borrow::{Cow, ToOwned as _}, sync::Arc, vec::Vec, }; use core::{ops::Range, ptr::NonNull, sync::atomic}; use smallvec::SmallVec; // has to match `Temp::binding_sizes` const WORD_SIZE: usize = 4; impl Default for super::CommandState { fn default() -> Self { Self { blit: None, acceleration_structure_builder: None, render: None, compute: None, raw_primitive_type: MTLPrimitiveType::Point, index: None, stage_infos: Default::default(), storage_buffer_length_map: Default::default(), vertex_buffer_size_map: Default::default(), immediates: Vec::new(), pending_timer_queries: Vec::new(), } } } /// Helper for passing encoders to `update_bind_group_state`. /// /// Combines [`naga::ShaderStage`] and an encoder of the appropriate type for /// that stage. enum Encoder<'e> { Vertex(&'e ProtocolObject), Fragment(&'e ProtocolObject), Task(&'e ProtocolObject), Mesh(&'e ProtocolObject), Compute(&'e ProtocolObject), } impl Encoder<'_> { fn stage(&self) -> naga::ShaderStage { match self { Self::Vertex(_) => naga::ShaderStage::Vertex, Self::Fragment(_) => naga::ShaderStage::Fragment, Self::Task(_) => naga::ShaderStage::Task, Self::Mesh(_) => naga::ShaderStage::Mesh, Self::Compute(_) => naga::ShaderStage::Compute, } } fn set_buffer( &self, buffer: Option<&ProtocolObject>, offset: NSUInteger, index: NSUInteger, ) { unsafe { match *self { Self::Vertex(enc) => enc.setVertexBuffer_offset_atIndex(buffer, offset, index), Self::Fragment(enc) => enc.setFragmentBuffer_offset_atIndex(buffer, offset, index), Self::Task(enc) => enc.setObjectBuffer_offset_atIndex(buffer, offset, index), Self::Mesh(enc) => enc.setMeshBuffer_offset_atIndex(buffer, offset, index), Self::Compute(enc) => enc.setBuffer_offset_atIndex(buffer, offset, index), } } } fn set_acceleration_structure( &self, buffer: Option<&ProtocolObject>, index: NSUInteger, ) { unsafe { match *self { Self::Vertex(enc) => { enc.setVertexAccelerationStructure_atBufferIndex(buffer, index) } Self::Fragment(enc) => { enc.setFragmentAccelerationStructure_atBufferIndex(buffer, index) } Self::Task(_) => { unreachable!("Acceleration structures are not allowed in task shaders") } Self::Mesh(_) => { unreachable!("Acceleration structures are not allowed in mesh shaders") } Self::Compute(enc) => enc.setAccelerationStructure_atBufferIndex(buffer, index), } } } fn set_bytes(&self, bytes: NonNull, length: NSUInteger, index: NSUInteger) { unsafe { match *self { Self::Vertex(enc) => enc.setVertexBytes_length_atIndex(bytes, length, index), Self::Fragment(enc) => enc.setFragmentBytes_length_atIndex(bytes, length, index), Self::Task(enc) => enc.setObjectBytes_length_atIndex(bytes, length, index), Self::Mesh(enc) => enc.setMeshBytes_length_atIndex(bytes, length, index), Self::Compute(enc) => enc.setBytes_length_atIndex(bytes, length, index), } } } fn set_sampler_state( &self, state: Option<&ProtocolObject>, index: NSUInteger, ) { unsafe { match *self { Self::Vertex(enc) => enc.setVertexSamplerState_atIndex(state, index), Self::Fragment(enc) => enc.setFragmentSamplerState_atIndex(state, index), Self::Task(enc) => enc.setObjectSamplerState_atIndex(state, index), Self::Mesh(enc) => enc.setMeshSamplerState_atIndex(state, index), Self::Compute(enc) => enc.setSamplerState_atIndex(state, index), } } } fn set_texture(&self, texture: Option<&ProtocolObject>, index: NSUInteger) { unsafe { match *self { Self::Vertex(enc) => enc.setVertexTexture_atIndex(texture, index), Self::Fragment(enc) => enc.setFragmentTexture_atIndex(texture, index), Self::Task(enc) => enc.setObjectTexture_atIndex(texture, index), Self::Mesh(enc) => enc.setMeshTexture_atIndex(texture, index), Self::Compute(enc) => enc.setTexture_atIndex(texture, index), } } } } impl super::CommandEncoder { pub fn raw_command_buffer(&self) -> Option<&ProtocolObject> { self.raw_cmd_buf.as_deref() } fn enter_blit(&mut self) -> Retained> { if self.state.blit.is_none() { self.leave_acceleration_structure_builder(); debug_assert!(self.state.render.is_none() && self.state.compute.is_none()); let cmd_buf = self.raw_cmd_buf.as_ref().unwrap(); // Take care of pending timer queries. // If we can't use `sample_counters_in_buffer` we have to create a dummy blit encoder! // // There is a known bug in Metal where blit encoders won't write timestamps if they don't have a blit operation. // See https://github.com/gpuweb/gpuweb/issues/2046#issuecomment-1205793680 & https://source.chromium.org/chromium/chromium/src/+/006c4eb70c96229834bbaf271290f40418144cd3:third_party/dawn/src/dawn/native/metal/BackendMTL.mm;l=350 // // To make things worse: // * what counts as a blit operation is a bit unclear, experimenting seemed to indicate that resolve_counters doesn't count. // * in some cases (when?) using `set_start_of_encoder_sample_index` doesn't work, so we have to use `set_end_of_encoder_sample_index` instead // // All this means that pretty much the only *reliable* thing as of writing is to: // * create a dummy blit encoder using set_end_of_encoder_sample_index // * do a dummy write that is known to be not optimized out. // * close the encoder since we used set_end_of_encoder_sample_index and don't want to get any extra stuff in there. // * create another encoder for whatever we actually had in mind. let supports_sample_counters_in_buffer = self .shared .private_caps .timestamp_query_support .contains(TimestampQuerySupport::ON_BLIT_ENCODER); if !self.state.pending_timer_queries.is_empty() && !supports_sample_counters_in_buffer { autoreleasepool(|_| { let descriptor = MTLBlitPassDescriptor::new(); let mut last_query = None; for (i, (set, index)) in self.state.pending_timer_queries.drain(..).enumerate() { let sba_descriptor = unsafe { descriptor .sampleBufferAttachments() .objectAtIndexedSubscript(i) }; sba_descriptor .setSampleBuffer(Some(set.counter_sample_buffer.as_ref().unwrap())); // Here be dragons: // As mentioned above, for some reasons using the start of the encoder won't yield any results sometimes! unsafe { sba_descriptor.setStartOfEncoderSampleIndex(MTLCounterDontSample) }; unsafe { sba_descriptor.setEndOfEncoderSampleIndex(index as _) }; last_query = Some((set, index)); } let encoder = cmd_buf .blitCommandEncoderWithDescriptor(&descriptor) .unwrap(); // As explained above, we need to do some write: // Conveniently, we have a buffer with every query set, that we can use for this for a dummy write, // since we know that it is going to be overwritten again on timer resolve and HAL doesn't define its state before that. let raw_range = NSRange { location: last_query.as_ref().unwrap().1 as usize * crate::QUERY_SIZE as usize, length: 1, }; encoder.fillBuffer_range_value( &last_query.as_ref().unwrap().0.raw_buffer, raw_range, 255, // Don't write 0, so it's easier to identify if something went wrong. ); encoder.endEncoding(); }); } autoreleasepool(|_| { self.state.blit = Some(cmd_buf.blitCommandEncoder().unwrap()); }); #[allow(clippy::panicking_unwrap, reason = "false positive (fixed by 1.93.1)")] let encoder = self.state.blit.as_ref().unwrap(); // UNTESTED: // If the above described issue with empty blit encoder applies to `sample_counters_in_buffer` as well, we should use the same workaround instead! for (set, index) in self.state.pending_timer_queries.drain(..) { debug_assert!(supports_sample_counters_in_buffer); unsafe { encoder.sampleCountersInBuffer_atSampleIndex_withBarrier( set.counter_sample_buffer.as_ref().unwrap(), index as _, true, ) }; } } self.state.blit.as_ref().unwrap().clone() } pub(super) fn leave_blit(&mut self) { if let Some(encoder) = self.state.blit.take() { encoder.endEncoding(); } } fn enter_acceleration_structure_builder( &mut self, ) -> Retained> { if self.state.acceleration_structure_builder.is_none() { self.leave_blit(); debug_assert!( self.state.render.is_none() && self.state.compute.is_none() && self.state.blit.is_none() ); let cmd_buf = self.raw_cmd_buf.as_ref().unwrap(); autoreleasepool(|_| { self.state.acceleration_structure_builder = cmd_buf.accelerationStructureCommandEncoder().to_owned(); }); } self.state.acceleration_structure_builder.clone().unwrap() } pub(super) fn leave_acceleration_structure_builder(&mut self) { if let Some(encoder) = self.state.acceleration_structure_builder.take() { encoder.endEncoding(); } } fn active_encoder(&mut self) -> Option<&ProtocolObject> { if let Some(ref encoder) = self.state.render { Some(ProtocolObject::from_ref(&**encoder)) } else if let Some(ref encoder) = self.state.acceleration_structure_builder { Some(ProtocolObject::from_ref(&**encoder)) } else if let Some(ref encoder) = self.state.compute { Some(ProtocolObject::from_ref(&**encoder)) } else if let Some(ref encoder) = self.state.blit { Some(ProtocolObject::from_ref(&**encoder)) } else { None } } fn begin_pass(&mut self) { self.state.reset(); self.leave_blit(); self.leave_acceleration_structure_builder(); } /// Updates the bindings for a single shader stage, called in `set_bind_group`. fn update_bind_group_state( &mut self, encoder: Encoder<'_>, index_base: super::ResourceData, bg_info: &super::BindGroupLayoutInfo, dynamic_offsets: &[wgt::DynamicOffset], group_index: u32, group: &super::BindGroup, ) { use naga::ShaderStage as S; let resource_indices = match encoder.stage() { S::Vertex => &bg_info.base_resource_indices.vs, S::Fragment => &bg_info.base_resource_indices.fs, S::Task => &bg_info.base_resource_indices.ts, S::Mesh => &bg_info.base_resource_indices.ms, S::Compute => &bg_info.base_resource_indices.cs, S::RayGeneration | S::AnyHit | S::ClosestHit | S::Miss => unimplemented!(), }; let buffers = match encoder.stage() { S::Vertex => group.counters.vs.buffers, S::Fragment => group.counters.fs.buffers, S::Task => group.counters.ts.buffers, S::Mesh => group.counters.ms.buffers, S::Compute => group.counters.cs.buffers, S::RayGeneration | S::AnyHit | S::ClosestHit | S::Miss => unimplemented!(), }; let mut changes_sizes_buffer = false; for index in 0..buffers { let res = &group.buffers[(index_base.buffers + index) as usize]; match res { super::BufferLikeResource::Buffer { ptr, mut offset, dynamic_index, binding_size, binding_location, } => { let buffer = Some(unsafe { ptr.as_ref() }); if let Some(dyn_index) = dynamic_index { offset += dynamic_offsets[*dyn_index as usize] as wgt::BufferAddress; } let index = (resource_indices.buffers + index) as usize; encoder.set_buffer(buffer, offset as usize, index); if let Some(size) = binding_size { let br = naga::ResourceBinding { group: group_index, binding: *binding_location, }; self.state.storage_buffer_length_map.insert(br, *size); changes_sizes_buffer = true; } } super::BufferLikeResource::AccelerationStructure(ptr) => { let buffer = Some(unsafe { ptr.as_ref() }); let index = (resource_indices.buffers + index) as usize; encoder.set_acceleration_structure(buffer, index); } } } if changes_sizes_buffer { if let Some((index, sizes)) = self .state .make_sizes_buffer_update(encoder.stage(), &mut self.temp.binding_sizes) { let bytes_ptr = NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(); let length = sizes.len() * WORD_SIZE; let index = index as _; encoder.set_bytes(bytes_ptr, length, index); } } let samplers = match encoder.stage() { S::Vertex => group.counters.vs.samplers, S::Fragment => group.counters.fs.samplers, S::Task => group.counters.ts.samplers, S::Mesh => group.counters.ms.samplers, S::Compute => group.counters.cs.samplers, S::RayGeneration | S::AnyHit | S::ClosestHit | S::Miss => unimplemented!(), }; for index in 0..samplers { let res = group.samplers[(index_base.samplers + index) as usize]; let index = (resource_indices.samplers + index) as usize; let state = Some(unsafe { res.as_ref() }); encoder.set_sampler_state(state, index); } let textures = match encoder.stage() { S::Vertex => group.counters.vs.textures, S::Fragment => group.counters.fs.textures, S::Task => group.counters.ts.textures, S::Mesh => group.counters.ms.textures, S::Compute => group.counters.cs.textures, S::RayGeneration | S::AnyHit | S::ClosestHit | S::Miss => unimplemented!(), }; for index in 0..textures { let res = group.textures[(index_base.textures + index) as usize]; let index = (resource_indices.textures + index) as usize; let texture = Some(unsafe { res.as_ref() }); encoder.set_texture(texture, index); } } } impl super::CommandState { fn reset(&mut self) { self.storage_buffer_length_map.clear(); self.vertex_buffer_size_map.clear(); self.stage_infos.vs.clear(); self.stage_infos.fs.clear(); self.stage_infos.cs.clear(); self.stage_infos.ts.clear(); self.stage_infos.ms.clear(); self.immediates.clear(); } fn make_sizes_buffer_update<'a>( &self, stage: naga::ShaderStage, result_sizes: &'a mut Vec, ) -> Option<(u32, &'a [u32])> { let stage_info = &self.stage_infos[stage]; let slot = stage_info.sizes_slot?; result_sizes.clear(); result_sizes.extend(stage_info.sized_bindings.iter().map(|br| { self.storage_buffer_length_map .get(br) .map(|size| u32::try_from(size.get()).unwrap_or(u32::MAX)) .unwrap_or_default() })); // Extend with the sizes of the mapped vertex buffers, in the order // they were added to the map. result_sizes.extend(stage_info.vertex_buffer_mappings.iter().map(|vbm| { self.vertex_buffer_size_map .get(&vbm.id) .map(|size| u32::try_from(size.get()).unwrap_or(u32::MAX)) .unwrap_or_default() })); if !result_sizes.is_empty() { Some((slot as _, result_sizes)) } else { None } } } impl crate::CommandEncoder for super::CommandEncoder { type A = super::Api; unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { let queue = &self.queue_shared.raw; let retain_references = self.shared.settings.retain_command_buffer_references; // Guard against exhausting Metal's command buffer budget. Use the hard // limit (`MAX_COMMAND_BUFFERS`) so we fail before Metal can hang inside // `new_command_buffer`. let previous = self .queue_shared .command_buffer_created_not_submitted .fetch_add(1, atomic::Ordering::AcqRel); if previous >= adapter::MAX_COMMAND_BUFFERS { let current = previous + 1; log::warn!( "metal: refusing to create new command buffer; {current} outstanding command \ buffers exceeds the limit of {}. Treating this as device lost. \ Ensure command encoders are submitted or dropped rather than kept alive \ to avoid exhausting Metal's command buffer budget.", adapter::MAX_COMMAND_BUFFERS ); return Err(crate::DeviceError::Lost); } let raw = autoreleasepool(move |_| { let cmd_buf_ref = if retain_references { queue.commandBuffer() } else { queue.commandBufferWithUnretainedReferences() } .unwrap(); if let Some(label) = label { cmd_buf_ref.setLabel(Some(&NSString::from_str(label))); } cmd_buf_ref.to_owned() }); self.raw_cmd_buf = Some(raw); Ok(()) } unsafe fn discard_encoding(&mut self) { self.leave_blit(); self.leave_acceleration_structure_builder(); // when discarding, we don't have a guarantee that // everything is in a good state, so check carefully if let Some(encoder) = self.state.render.take() { encoder.endEncoding(); } if let Some(encoder) = self.state.compute.take() { encoder.endEncoding(); } let had_command_buffer = self.raw_cmd_buf.is_some(); // Clear the Option first so the underlying `metal::CommandBuffer` is // dropped before we update the counter. self.raw_cmd_buf = None; if had_command_buffer { self.queue_shared .command_buffer_created_not_submitted .fetch_sub(1, atomic::Ordering::AcqRel); } } unsafe fn end_encoding(&mut self) -> Result { // Handle pending timer query if any. if !self.state.pending_timer_queries.is_empty() { self.leave_blit(); self.enter_blit(); } self.leave_blit(); self.leave_acceleration_structure_builder(); debug_assert!(self.state.render.is_none()); debug_assert!(self.state.compute.is_none()); debug_assert!(self.state.pending_timer_queries.is_empty()); Ok(super::CommandBuffer { raw: self.raw_cmd_buf.take().unwrap(), queue_shared: Arc::clone(&self.queue_shared), }) } unsafe fn reset_all(&mut self, _cmd_bufs: I) where I: Iterator, { //do nothing } unsafe fn transition_buffers<'a, T>(&mut self, _barriers: T) where T: Iterator>, { } unsafe fn transition_textures<'a, T>(&mut self, _barriers: T) where T: Iterator>, { } unsafe fn clear_buffer(&mut self, buffer: &super::Buffer, range: crate::MemoryRange) { let encoder = self.enter_blit(); encoder.fillBuffer_range_value(&buffer.raw, conv::map_range(&range), 0); } unsafe fn copy_buffer_to_buffer( &mut self, src: &super::Buffer, dst: &super::Buffer, regions: T, ) where T: Iterator, { let encoder = self.enter_blit(); for copy in regions { unsafe { encoder.copyFromBuffer_sourceOffset_toBuffer_destinationOffset_size( &src.raw, copy.src_offset as usize, &dst.raw, copy.dst_offset as usize, copy.size.get() as usize, ) }; } } unsafe fn copy_texture_to_texture( &mut self, src: &super::Texture, _src_usage: wgt::TextureUses, dst: &super::Texture, regions: T, ) where T: Iterator, { let dst_texture = if src.format != dst.format { let raw_format = self .shared .private_texture_format_caps .map_format(src.format); Cow::Owned(autoreleasepool(|_| { dst.raw.newTextureViewWithPixelFormat(raw_format).unwrap() })) } else { Cow::Borrowed(&dst.raw) }; let encoder = self.enter_blit(); for copy in regions { let src_origin = conv::map_origin(©.src_base.origin); let dst_origin = conv::map_origin(©.dst_base.origin); // no clamping is done: Metal expects physical sizes here let extent = conv::map_copy_extent(©.size); unsafe { encoder.copyFromTexture_sourceSlice_sourceLevel_sourceOrigin_sourceSize_toTexture_destinationSlice_destinationLevel_destinationOrigin( &src.raw, copy.src_base.array_layer as usize, copy.src_base.mip_level as usize, src_origin, extent, &dst_texture, copy.dst_base.array_layer as usize, copy.dst_base.mip_level as usize, dst_origin, ) }; } } unsafe fn copy_buffer_to_texture( &mut self, src: &super::Buffer, dst: &super::Texture, regions: T, ) where T: Iterator, { let encoder = self.enter_blit(); for copy in regions { let dst_origin = conv::map_origin(©.texture_base.origin); // Metal expects buffer-texture copies in virtual sizes let extent = copy .texture_base .max_copy_size(&dst.copy_size) .min(©.size); let bytes_per_row = copy.buffer_layout.bytes_per_row.unwrap_or(0) as u64; let image_byte_stride = if extent.depth > 1 { copy.buffer_layout .rows_per_image .map_or(0, |v| v as u64 * bytes_per_row) } else { // Don't pass a stride when updating a single layer, otherwise metal validation // fails when updating a subset of the image due to the stride being larger than // the amount of data to copy. 0 }; unsafe { encoder.copyFromBuffer_sourceOffset_sourceBytesPerRow_sourceBytesPerImage_sourceSize_toTexture_destinationSlice_destinationLevel_destinationOrigin_options( &src.raw, copy.buffer_layout.offset as usize, bytes_per_row as usize, image_byte_stride as usize, conv::map_copy_extent(&extent), &dst.raw, copy.texture_base.array_layer as usize, copy.texture_base.mip_level as usize, dst_origin, conv::get_blit_option(dst.format, copy.texture_base.aspect), ) }; } } unsafe fn copy_texture_to_buffer( &mut self, src: &super::Texture, _src_usage: wgt::TextureUses, dst: &super::Buffer, regions: T, ) where T: Iterator, { let encoder = self.enter_blit(); for copy in regions { let src_origin = conv::map_origin(©.texture_base.origin); // Metal expects texture-buffer copies in virtual sizes let extent = copy .texture_base .max_copy_size(&src.copy_size) .min(©.size); let bytes_per_row = copy.buffer_layout.bytes_per_row.unwrap_or(0) as u64; let bytes_per_image = copy .buffer_layout .rows_per_image .map_or(0, |v| v as u64 * bytes_per_row); unsafe { encoder.copyFromTexture_sourceSlice_sourceLevel_sourceOrigin_sourceSize_toBuffer_destinationOffset_destinationBytesPerRow_destinationBytesPerImage_options( &src.raw, copy.texture_base.array_layer as usize, copy.texture_base.mip_level as usize, src_origin, conv::map_copy_extent(&extent), &dst.raw, copy.buffer_layout.offset as usize, bytes_per_row as usize, bytes_per_image as usize, conv::get_blit_option(src.format, copy.texture_base.aspect), ) }; } } unsafe fn copy_acceleration_structure_to_acceleration_structure( &mut self, src: &super::AccelerationStructure, dst: &super::AccelerationStructure, copy: wgt::AccelerationStructureCopy, ) { let command_encoder = self.enter_acceleration_structure_builder(); match copy { wgt::AccelerationStructureCopy::Clone => unsafe { command_encoder .copyAccelerationStructure_toAccelerationStructure(&src.raw, &dst.raw); }, wgt::AccelerationStructureCopy::Compact => { command_encoder.copyAndCompactAccelerationStructure_toAccelerationStructure( &src.raw, &dst.raw, ); } }; } unsafe fn begin_query(&mut self, set: &super::QuerySet, index: u32) { match set.ty { wgt::QueryType::Occlusion => { self.state .render .as_ref() .unwrap() .setVisibilityResultMode_offset( MTLVisibilityResultMode::Boolean, index as usize * crate::QUERY_SIZE as usize, ); } _ => {} } } unsafe fn end_query(&mut self, set: &super::QuerySet, _index: u32) { match set.ty { wgt::QueryType::Occlusion => { self.state .render .as_ref() .unwrap() .setVisibilityResultMode_offset(MTLVisibilityResultMode::Disabled, 0); } _ => {} } } unsafe fn write_timestamp(&mut self, set: &super::QuerySet, index: u32) { let support = self.shared.private_caps.timestamp_query_support; debug_assert!( support.contains(TimestampQuerySupport::STAGE_BOUNDARIES), "Timestamp queries are not supported" ); let sample_buffer = set.counter_sample_buffer.as_ref().unwrap(); let with_barrier = true; // Try to use an existing encoder for timestamp query if possible. // This works only if it's supported for the active encoder. if let (true, Some(encoder)) = ( support.contains(TimestampQuerySupport::ON_BLIT_ENCODER), self.state.blit.as_ref(), ) { unsafe { encoder.sampleCountersInBuffer_atSampleIndex_withBarrier( sample_buffer, index as _, with_barrier, ) }; } else if let (true, Some(encoder)) = ( support.contains(TimestampQuerySupport::ON_RENDER_ENCODER), self.state.render.as_ref(), ) { unsafe { encoder.sampleCountersInBuffer_atSampleIndex_withBarrier( sample_buffer, index as _, with_barrier, ) }; } else if let (true, Some(encoder)) = ( support.contains(TimestampQuerySupport::ON_COMPUTE_ENCODER), self.state.compute.as_ref(), ) { unsafe { encoder.sampleCountersInBuffer_atSampleIndex_withBarrier( sample_buffer, index as _, with_barrier, ) }; } else { // If we're here it means we either have no encoder open, or it's not supported to sample within them. // If this happens with render/compute open, this is an invalid usage! debug_assert!(self.state.render.is_none() && self.state.compute.is_none()); // But otherwise it means we'll put defer this to the next created encoder. self.state.pending_timer_queries.push((set.clone(), index)); // Ensure we didn't already have a blit open. self.leave_blit(); }; } unsafe fn reset_queries(&mut self, set: &super::QuerySet, range: Range) { let encoder = self.enter_blit(); let raw_range = NSRange { location: range.start as usize * crate::QUERY_SIZE as usize, length: (range.end - range.start) as usize * crate::QUERY_SIZE as usize, }; encoder.fillBuffer_range_value(&set.raw_buffer, raw_range, 0); } unsafe fn copy_query_results( &mut self, set: &super::QuerySet, range: Range, buffer: &super::Buffer, offset: wgt::BufferAddress, _: wgt::BufferSize, // Metal doesn't support queries that are bigger than a single element are not supported ) { let encoder = self.enter_blit(); match set.ty { wgt::QueryType::Occlusion => { let size = (range.end - range.start) as u64 * crate::QUERY_SIZE; unsafe { encoder.copyFromBuffer_sourceOffset_toBuffer_destinationOffset_size( &set.raw_buffer, range.start as usize * crate::QUERY_SIZE as usize, &buffer.raw, offset as usize, size as usize, ) }; } wgt::QueryType::Timestamp => { unsafe { encoder.resolveCounters_inRange_destinationBuffer_destinationOffset( set.counter_sample_buffer.as_ref().unwrap(), NSRange::new(range.start as usize, (range.end - range.start) as usize), &buffer.raw, offset as usize, ) }; } wgt::QueryType::PipelineStatistics(_) => todo!(), } } // render unsafe fn begin_render_pass( &mut self, desc: &crate::RenderPassDescriptor, ) -> Result<(), crate::DeviceError> { self.begin_pass(); self.state.index = None; assert!(self.state.blit.is_none()); assert!(self.state.compute.is_none()); assert!(self.state.render.is_none()); autoreleasepool(|_| { let descriptor = MTLRenderPassDescriptor::new(); for (i, at) in desc.color_attachments.iter().enumerate() { if let Some(at) = at.as_ref() { let at_descriptor = unsafe { descriptor.colorAttachments().objectAtIndexedSubscript(i) }; at_descriptor.setTexture(Some(&at.target.view.raw)); if let Some(depth_slice) = at.depth_slice { at_descriptor.setDepthPlane(depth_slice as usize); } if let Some(ref resolve) = at.resolve_target { //Note: the selection of levels and slices is already handled by `TextureView` at_descriptor.setResolveTexture(Some(&resolve.view.raw)); } let load_action = if at.ops.contains(crate::AttachmentOps::LOAD) { MTLLoadAction::Load } else if at.ops.contains(crate::AttachmentOps::LOAD_DONT_CARE) { MTLLoadAction::DontCare } else if at.ops.contains(crate::AttachmentOps::LOAD_CLEAR) { at_descriptor.setClearColor(conv::map_clear_color(&at.clear_value)); MTLLoadAction::Clear } else { unreachable!() }; let store_action = conv::map_store_action( at.ops.contains(crate::AttachmentOps::STORE), at.resolve_target.is_some(), ); at_descriptor.setLoadAction(load_action); at_descriptor.setStoreAction(store_action); } } if let Some(ref at) = desc.depth_stencil_attachment { if at.target.view.aspects.contains(crate::FormatAspects::DEPTH) { let at_descriptor = descriptor.depthAttachment(); at_descriptor.setTexture(Some(&at.target.view.raw)); let load_action = if at.depth_ops.contains(crate::AttachmentOps::LOAD) { MTLLoadAction::Load } else if at.depth_ops.contains(crate::AttachmentOps::LOAD_DONT_CARE) { MTLLoadAction::DontCare } else if at.depth_ops.contains(crate::AttachmentOps::LOAD_CLEAR) { at_descriptor.setClearDepth(at.clear_value.0 as f64); MTLLoadAction::Clear } else { unreachable!(); }; let store_action = if at.depth_ops.contains(crate::AttachmentOps::STORE) { MTLStoreAction::Store } else { MTLStoreAction::DontCare }; at_descriptor.setLoadAction(load_action); at_descriptor.setStoreAction(store_action); } if at .target .view .aspects .contains(crate::FormatAspects::STENCIL) { let at_descriptor = descriptor.stencilAttachment(); at_descriptor.setTexture(Some(&at.target.view.raw)); let load_action = if at.stencil_ops.contains(crate::AttachmentOps::LOAD) { MTLLoadAction::Load } else if at .stencil_ops .contains(crate::AttachmentOps::LOAD_DONT_CARE) { MTLLoadAction::DontCare } else if at.stencil_ops.contains(crate::AttachmentOps::LOAD_CLEAR) { at_descriptor.setClearStencil(at.clear_value.1); MTLLoadAction::Clear } else { unreachable!() }; let store_action = if at.stencil_ops.contains(crate::AttachmentOps::STORE) { MTLStoreAction::Store } else { MTLStoreAction::DontCare }; at_descriptor.setLoadAction(load_action); at_descriptor.setStoreAction(store_action); } } let mut sba_index = 0; let mut next_sba_descriptor = || { let sba_descriptor = unsafe { descriptor .sampleBufferAttachments() .objectAtIndexedSubscript(sba_index) }; unsafe { sba_descriptor.setEndOfVertexSampleIndex(MTLCounterDontSample) }; unsafe { sba_descriptor.setStartOfFragmentSampleIndex(MTLCounterDontSample) }; sba_index += 1; sba_descriptor }; for (set, index) in self.state.pending_timer_queries.drain(..) { let sba_descriptor = next_sba_descriptor(); sba_descriptor.setSampleBuffer(Some(set.counter_sample_buffer.as_ref().unwrap())); unsafe { sba_descriptor.setStartOfVertexSampleIndex(index as _) }; unsafe { sba_descriptor.setEndOfFragmentSampleIndex(MTLCounterDontSample) }; } if let Some(ref timestamp_writes) = desc.timestamp_writes { let sba_descriptor = next_sba_descriptor(); sba_descriptor.setSampleBuffer(Some( timestamp_writes .query_set .counter_sample_buffer .as_ref() .unwrap(), )); unsafe { sba_descriptor.setStartOfVertexSampleIndex( timestamp_writes .beginning_of_pass_write_index .map_or(MTLCounterDontSample, |i| i as _), ) }; unsafe { sba_descriptor.setEndOfFragmentSampleIndex( timestamp_writes .end_of_pass_write_index .map_or(MTLCounterDontSample, |i| i as _), ) }; } if let Some(occlusion_query_set) = desc.occlusion_query_set { descriptor.setVisibilityResultBuffer(Some(occlusion_query_set.raw_buffer.as_ref())) } // This strangely isn't mentioned in https://developer.apple.com/documentation/metal/improving-rendering-performance-with-vertex-amplification. // The docs for [`renderTargetArrayLength`](https://developer.apple.com/documentation/metal/mtlrenderpassdescriptor/rendertargetarraylength) // also say "The number of active layers that all attachments must have for layered rendering," implying it is only for layered rendering. // However, when I don't set this, I get undefined behavior in nonzero layers, and all non-apple examples of vertex amplification set it. // So this is just one of those undocumented requirements. if let Some(mv) = desc.multiview_mask { descriptor.setRenderTargetArrayLength(32 - mv.leading_zeros() as usize); } let raw = self.raw_cmd_buf.as_ref().unwrap(); let encoder = raw.renderCommandEncoderWithDescriptor(&descriptor).unwrap(); if let Some(mv) = desc.multiview_mask { // Most likely the API just wasn't thought about enough. It's not like they ever allow you // to use enough views to overflow a 32-bit bitmask. let mv = mv.get(); let msb = 32 - mv.leading_zeros(); let mut maps: SmallVec<[MTLVertexAmplificationViewMapping; 32]> = SmallVec::new(); for i in 0..msb { if (mv & (1 << i)) != 0 { maps.push(MTLVertexAmplificationViewMapping { renderTargetArrayIndexOffset: i, viewportArrayIndexOffset: i, }); } } unsafe { encoder.setVertexAmplificationCount_viewMappings( mv.count_ones() as usize, maps.as_ptr(), ) }; } if let Some(label) = desc.label { encoder.setLabel(Some(&NSString::from_str(label))); } self.state.render = Some(encoder); }); Ok(()) } unsafe fn end_render_pass(&mut self) { self.state.render.take().unwrap().endEncoding(); } unsafe fn set_bind_group( &mut self, layout: &super::PipelineLayout, group_index: u32, group: &super::BindGroup, dynamic_offsets: &[wgt::DynamicOffset], ) { let bg_info = layout.bind_group_infos[group_index as usize] .as_ref() .unwrap(); let render_encoder = self.state.render.clone(); let compute_encoder = self.state.compute.clone(); if let Some(encoder) = render_encoder { self.update_bind_group_state( Encoder::Vertex(&encoder), // All zeros, as vs comes first super::ResourceData::default(), bg_info, dynamic_offsets, group_index, group, ); self.update_bind_group_state( Encoder::Task(&encoder), // All zeros, as ts comes first super::ResourceData::default(), bg_info, dynamic_offsets, group_index, group, ); self.update_bind_group_state( Encoder::Mesh(&encoder), group.counters.ts.clone(), bg_info, dynamic_offsets, group_index, group, ); self.update_bind_group_state( Encoder::Fragment(&encoder), super::ResourceData { buffers: group.counters.vs.buffers + group.counters.ts.buffers + group.counters.ms.buffers, textures: group.counters.vs.textures + group.counters.ts.textures + group.counters.ms.textures, samplers: group.counters.vs.samplers + group.counters.ts.samplers + group.counters.ms.samplers, }, bg_info, dynamic_offsets, group_index, group, ); // Call useResource on all textures and buffers used indirectly so they are alive for (resource, use_info) in group.resources_to_use.iter() { encoder.useResource_usage_stages( unsafe { resource.as_ref() }, use_info.uses, use_info.stages, ); } } if let Some(encoder) = compute_encoder { self.update_bind_group_state( Encoder::Compute(&encoder), super::ResourceData { buffers: group.counters.vs.buffers + group.counters.ts.buffers + group.counters.ms.buffers + group.counters.fs.buffers, textures: group.counters.vs.textures + group.counters.ts.textures + group.counters.ms.textures + group.counters.fs.textures, samplers: group.counters.vs.samplers + group.counters.ts.samplers + group.counters.ms.samplers + group.counters.fs.samplers, }, bg_info, dynamic_offsets, group_index, group, ); // Call useResource on all textures and buffers used indirectly so they are alive for (resource, use_info) in group.resources_to_use.iter() { if !use_info.visible_in_compute { continue; } encoder.useResource_usage(unsafe { resource.as_ref() }, use_info.uses); } } } unsafe fn set_immediates( &mut self, layout: &super::PipelineLayout, offset_bytes: u32, data: &[u32], ) { let state_pc = &mut self.state.immediates; if state_pc.len() < layout.total_immediates as usize { state_pc.resize(layout.total_immediates as usize, 0); } debug_assert_eq!(offset_bytes as usize % WORD_SIZE, 0); let offset_words = offset_bytes as usize / WORD_SIZE; state_pc[offset_words..offset_words + data.len()].copy_from_slice(data); let bytes = NonNull::new(state_pc.as_ptr().cast_mut().cast()).unwrap(); if let Some(ref compute) = self.state.compute { unsafe { compute.setBytes_length_atIndex( bytes, layout.total_immediates as usize * WORD_SIZE, layout.immediates_infos.cs.unwrap().buffer_index as usize, ) }; } if let Some(ref render) = self.state.render { if let Some(vs) = layout.immediates_infos.vs { unsafe { render.setVertexBytes_length_atIndex( bytes, layout.total_immediates as usize * WORD_SIZE, vs.buffer_index as _, ) } } if let Some(fs) = layout.immediates_infos.fs { unsafe { render.setFragmentBytes_length_atIndex( bytes, layout.total_immediates as usize * WORD_SIZE, fs.buffer_index as _, ) } } if let Some(ts) = layout.immediates_infos.ts { if self.shared.private_caps.mesh_shaders { unsafe { render.setObjectBytes_length_atIndex( bytes, layout.total_immediates as usize * WORD_SIZE, ts.buffer_index as _, ) } } } if let Some(ms) = layout.immediates_infos.ms { if self.shared.private_caps.mesh_shaders { unsafe { render.setMeshBytes_length_atIndex( bytes, layout.total_immediates as usize * WORD_SIZE, ms.buffer_index as _, ) } } } } } unsafe fn insert_debug_marker(&mut self, label: &str) { if let Some(encoder) = self.active_encoder() { encoder.insertDebugSignpost(&NSString::from_str(label)); } } unsafe fn begin_debug_marker(&mut self, group_label: &str) { if let Some(encoder) = self.active_encoder() { encoder.pushDebugGroup(&NSString::from_str(group_label)); } else if let Some(ref buf) = self.raw_cmd_buf { buf.pushDebugGroup(&NSString::from_str(group_label)); } } unsafe fn end_debug_marker(&mut self) { if let Some(encoder) = self.active_encoder() { encoder.popDebugGroup(); } else if let Some(ref buf) = self.raw_cmd_buf { buf.popDebugGroup(); } } unsafe fn set_render_pipeline(&mut self, pipeline: &super::RenderPipeline) { self.state.raw_primitive_type = pipeline.raw_primitive_type; match pipeline.vs_info { Some(ref info) => self.state.stage_infos.vs.assign_from(info), None => self.state.stage_infos.vs.clear(), } match pipeline.fs_info { Some(ref info) => self.state.stage_infos.fs.assign_from(info), None => self.state.stage_infos.fs.clear(), } match pipeline.ts_info { Some(ref info) => self.state.stage_infos.ts.assign_from(info), None => self.state.stage_infos.ts.clear(), } match pipeline.ms_info { Some(ref info) => self.state.stage_infos.ms.assign_from(info), None => self.state.stage_infos.ms.clear(), } let encoder = self.state.render.as_ref().unwrap(); encoder.setRenderPipelineState(&pipeline.raw); encoder.setFrontFacingWinding(pipeline.raw_front_winding); encoder.setCullMode(pipeline.raw_cull_mode); encoder.setTriangleFillMode(pipeline.raw_triangle_fill_mode); if let Some(depth_clip) = pipeline.raw_depth_clip_mode { encoder.setDepthClipMode(depth_clip); } if let Some((ref state, bias)) = pipeline.depth_stencil { encoder.setDepthStencilState(Some(state)); encoder.setDepthBias_slopeScale_clamp( bias.constant as f32, bias.slope_scale, bias.clamp, ); } if pipeline.vs_info.is_some() { if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Vertex, &mut self.temp.binding_sizes) { unsafe { encoder.setVertexBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } } if pipeline.fs_info.is_some() { if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Fragment, &mut self.temp.binding_sizes) { unsafe { encoder.setFragmentBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } } if let Some(ts_info) = &pipeline.ts_info { // update the threadgroup memory sizes while self.state.stage_infos.ms.work_group_memory_sizes.len() < ts_info.work_group_memory_sizes.len() { self.state.stage_infos.ms.work_group_memory_sizes.push(0); } for (index, (cur_size, pipeline_size)) in self .state .stage_infos .ms .work_group_memory_sizes .iter_mut() .zip(ts_info.work_group_memory_sizes.iter()) .enumerate() { let size = pipeline_size.next_multiple_of(16); if *cur_size != size { *cur_size = size; unsafe { encoder.setObjectThreadgroupMemoryLength_atIndex(size as _, index) }; } } if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Task, &mut self.temp.binding_sizes) { unsafe { encoder.setObjectBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } } if let Some(_ms_info) = &pipeline.ms_info { // So there isn't an equivalent to // https://developer.apple.com/documentation/metal/mtlrendercommandencoder/setthreadgroupmemorylength(_:offset:index:) // for mesh shaders. This is probably because the CPU has less control over the dispatch sizes and such. Interestingly // it also affects mesh shaders without task/object shaders, even though none of compute, task or fragment shaders // behave this way. if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Mesh, &mut self.temp.binding_sizes) { unsafe { encoder.setMeshBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } } } unsafe fn set_index_buffer<'a>( &mut self, binding: crate::BufferBinding<'a, super::Buffer>, format: wgt::IndexFormat, ) { let (stride, raw_type) = conv::map_index_format(format); self.state.index = Some(super::IndexState { buffer_ptr: NonNull::from(&*binding.buffer.raw), offset: binding.offset, stride, raw_type, }); } unsafe fn set_vertex_buffer<'a>( &mut self, index: u32, binding: crate::BufferBinding<'a, super::Buffer>, ) { let buffer_index = VERTEX_BUFFER_SLOT_START + index; let encoder = self.state.render.as_ref().unwrap(); unsafe { encoder.setVertexBuffer_offset_atIndex( Some(&binding.buffer.raw), binding.offset as usize, buffer_index as usize, ) }; let buffer_size = binding.resolve_size(); if buffer_size > 0 { self.state.vertex_buffer_size_map.insert( buffer_index, core::num::NonZeroU64::new(buffer_size).unwrap(), ); } else { self.state.vertex_buffer_size_map.remove(&buffer_index); } if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Vertex, &mut self.temp.binding_sizes) { unsafe { encoder.setVertexBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } } unsafe fn set_viewport(&mut self, rect: &crate::Rect, depth_range: Range) { let zfar = if self.shared.disabilities.broken_viewport_near_depth { depth_range.end - depth_range.start } else { depth_range.end }; let encoder = self.state.render.as_ref().unwrap(); encoder.setViewport(MTLViewport { originX: rect.x as _, originY: rect.y as _, width: rect.w as _, height: rect.h as _, znear: depth_range.start as _, zfar: zfar as _, }); } unsafe fn set_scissor_rect(&mut self, rect: &crate::Rect) { //TODO: support empty scissors by modifying the viewport let scissor = MTLScissorRect { x: rect.x as _, y: rect.y as _, width: rect.w as _, height: rect.h as _, }; let encoder = self.state.render.as_ref().unwrap(); encoder.setScissorRect(scissor); } unsafe fn set_stencil_reference(&mut self, value: u32) { let encoder = self.state.render.as_ref().unwrap(); encoder.setStencilFrontReferenceValue_backReferenceValue(value, value); } unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) { let encoder = self.state.render.as_ref().unwrap(); encoder.setBlendColorRed_green_blue_alpha(color[0], color[1], color[2], color[3]); } unsafe fn draw( &mut self, first_vertex: u32, vertex_count: u32, first_instance: u32, instance_count: u32, ) { let encoder = self.state.render.as_ref().unwrap(); if first_instance != 0 { unsafe { encoder.drawPrimitives_vertexStart_vertexCount_instanceCount_baseInstance( self.state.raw_primitive_type, first_vertex as _, vertex_count as _, instance_count as _, first_instance as _, ) }; } else if instance_count != 1 { unsafe { encoder.drawPrimitives_vertexStart_vertexCount_instanceCount( self.state.raw_primitive_type, first_vertex as _, vertex_count as _, instance_count as _, ) }; } else { unsafe { encoder.drawPrimitives_vertexStart_vertexCount( self.state.raw_primitive_type, first_vertex as _, vertex_count as _, ) }; } } unsafe fn draw_indexed( &mut self, first_index: u32, index_count: u32, base_vertex: i32, first_instance: u32, instance_count: u32, ) { let encoder = self.state.render.as_ref().unwrap(); let index = self.state.index.as_ref().unwrap(); let offset = (index.offset + index.stride * first_index as wgt::BufferAddress) as usize; if base_vertex != 0 || first_instance != 0 { unsafe { encoder.drawIndexedPrimitives_indexCount_indexType_indexBuffer_indexBufferOffset_instanceCount_baseVertex_baseInstance( self.state.raw_primitive_type, index_count as _, index.raw_type, index.buffer_ptr.as_ref(), offset, instance_count as _, base_vertex as _, first_instance as _, ) }; } else if instance_count != 1 { unsafe { encoder.drawIndexedPrimitives_indexCount_indexType_indexBuffer_indexBufferOffset_instanceCount( self.state.raw_primitive_type, index_count as _, index.raw_type, index.buffer_ptr.as_ref(), offset, instance_count as _, ) }; } else { unsafe { encoder.drawIndexedPrimitives_indexCount_indexType_indexBuffer_indexBufferOffset( self.state.raw_primitive_type, index_count as _, index.raw_type, index.buffer_ptr.as_ref(), offset, ) }; } } unsafe fn draw_mesh_tasks( &mut self, group_count_x: u32, group_count_y: u32, group_count_z: u32, ) { let encoder = self.state.render.as_ref().unwrap(); let size = MTLSize { width: group_count_x as usize, height: group_count_y as usize, depth: group_count_z as usize, }; encoder.drawMeshThreadgroups_threadsPerObjectThreadgroup_threadsPerMeshThreadgroup( size, self.state.stage_infos.ts.raw_wg_size, self.state.stage_infos.ms.raw_wg_size, ); } unsafe fn draw_indirect( &mut self, buffer: &super::Buffer, mut offset: wgt::BufferAddress, draw_count: u32, ) { let encoder = self.state.render.as_ref().unwrap(); for _ in 0..draw_count { unsafe { encoder.drawPrimitives_indirectBuffer_indirectBufferOffset( self.state.raw_primitive_type, &buffer.raw, offset as usize, ) }; offset += size_of::() as wgt::BufferAddress; } } unsafe fn draw_indexed_indirect( &mut self, buffer: &super::Buffer, mut offset: wgt::BufferAddress, draw_count: u32, ) { let encoder = self.state.render.as_ref().unwrap(); let index = self.state.index.as_ref().unwrap(); for _ in 0..draw_count { unsafe { encoder.drawIndexedPrimitives_indexType_indexBuffer_indexBufferOffset_indirectBuffer_indirectBufferOffset( self.state.raw_primitive_type, index.raw_type, index.buffer_ptr.as_ref(), index.offset as usize, &buffer.raw, offset as usize, ) }; offset += size_of::() as wgt::BufferAddress; } } unsafe fn draw_mesh_tasks_indirect( &mut self, buffer: &::Buffer, mut offset: wgt::BufferAddress, draw_count: u32, ) { let encoder = self.state.render.as_ref().unwrap(); for _ in 0..draw_count { unsafe { encoder.drawMeshThreadgroupsWithIndirectBuffer_indirectBufferOffset_threadsPerObjectThreadgroup_threadsPerMeshThreadgroup( &buffer.raw, offset as usize, self.state.stage_infos.ts.raw_wg_size, self.state.stage_infos.ms.raw_wg_size, ) }; offset += size_of::() as wgt::BufferAddress; } } unsafe fn draw_indirect_count( &mut self, _buffer: &super::Buffer, _offset: wgt::BufferAddress, _count_buffer: &super::Buffer, _count_offset: wgt::BufferAddress, _max_count: u32, ) { //TODO } unsafe fn draw_indexed_indirect_count( &mut self, _buffer: &super::Buffer, _offset: wgt::BufferAddress, _count_buffer: &super::Buffer, _count_offset: wgt::BufferAddress, _max_count: u32, ) { //TODO } unsafe fn draw_mesh_tasks_indirect_count( &mut self, _buffer: &::Buffer, _offset: wgt::BufferAddress, _count_buffer: &::Buffer, _count_offset: wgt::BufferAddress, _max_count: u32, ) { unreachable!() } // compute unsafe fn begin_compute_pass(&mut self, desc: &crate::ComputePassDescriptor) { self.begin_pass(); debug_assert!(self.state.blit.is_none()); debug_assert!(self.state.compute.is_none()); debug_assert!(self.state.render.is_none()); let raw = self.raw_cmd_buf.as_ref().unwrap(); autoreleasepool(|_| { // TimeStamp Queries and ComputePassDescriptor were both introduced in Metal 2.3 (macOS 11, iOS 14) // and we currently only need ComputePassDescriptor for timestamp queries let encoder = if self.shared.private_caps.timestamp_query_support.is_empty() { raw.computeCommandEncoder().unwrap() } else { let descriptor = MTLComputePassDescriptor::new(); let mut sba_index = 0; let mut next_sba_descriptor = || { let sba_descriptor = unsafe { descriptor .sampleBufferAttachments() .objectAtIndexedSubscript(sba_index) }; sba_index += 1; sba_descriptor }; for (set, index) in self.state.pending_timer_queries.drain(..) { let sba_descriptor = next_sba_descriptor(); sba_descriptor .setSampleBuffer(Some(set.counter_sample_buffer.as_ref().unwrap())); unsafe { sba_descriptor.setStartOfEncoderSampleIndex(index as _) }; unsafe { sba_descriptor.setEndOfEncoderSampleIndex(MTLCounterDontSample) }; } if let Some(timestamp_writes) = desc.timestamp_writes.as_ref() { let sba_descriptor = next_sba_descriptor(); sba_descriptor.setSampleBuffer(Some( timestamp_writes .query_set .counter_sample_buffer .as_ref() .unwrap(), )); unsafe { sba_descriptor.setStartOfEncoderSampleIndex( timestamp_writes .beginning_of_pass_write_index .map_or(MTLCounterDontSample, |i| i as _), ) }; unsafe { sba_descriptor.setEndOfEncoderSampleIndex( timestamp_writes .end_of_pass_write_index .map_or(MTLCounterDontSample, |i| i as _), ) }; } raw.computeCommandEncoderWithDescriptor(&descriptor) .unwrap() }; if let Some(label) = desc.label { encoder.setLabel(Some(&NSString::from_str(label))); } self.state.compute = Some(encoder.to_owned()); }); } unsafe fn end_compute_pass(&mut self) { self.state.compute.take().unwrap().endEncoding(); } unsafe fn set_compute_pipeline(&mut self, pipeline: &super::ComputePipeline) { let previous_sizes = core::mem::take(&mut self.state.stage_infos.cs.work_group_memory_sizes); self.state.stage_infos.cs.assign_from(&pipeline.cs_info); let encoder = self.state.compute.as_ref().unwrap(); encoder.setComputePipelineState(&pipeline.raw); if let Some((index, sizes)) = self .state .make_sizes_buffer_update(naga::ShaderStage::Compute, &mut self.temp.binding_sizes) { unsafe { encoder.setBytes_length_atIndex( NonNull::new(sizes.as_ptr().cast_mut().cast()).unwrap(), sizes.len() * WORD_SIZE, index as _, ) }; } // update the threadgroup memory sizes for (i, current_size) in self .state .stage_infos .cs .work_group_memory_sizes .iter_mut() .enumerate() { let prev_size = if i < previous_sizes.len() { previous_sizes[i] } else { u32::MAX }; let size: u32 = current_size.next_multiple_of(16); *current_size = size; if size != prev_size { unsafe { encoder.setThreadgroupMemoryLength_atIndex(size as _, i) }; } } } unsafe fn dispatch(&mut self, count: [u32; 3]) { if count[0] > 0 && count[1] > 0 && count[2] > 0 { let encoder = self.state.compute.as_ref().unwrap(); let raw_count = MTLSize { width: count[0] as usize, height: count[1] as usize, depth: count[2] as usize, }; encoder.dispatchThreadgroups_threadsPerThreadgroup( raw_count, self.state.stage_infos.cs.raw_wg_size, ); } } unsafe fn dispatch_indirect(&mut self, buffer: &super::Buffer, offset: wgt::BufferAddress) { let encoder = self.state.compute.as_ref().unwrap(); unsafe { encoder .dispatchThreadgroupsWithIndirectBuffer_indirectBufferOffset_threadsPerThreadgroup( &buffer.raw, offset as usize, self.state.stage_infos.cs.raw_wg_size, ) }; } unsafe fn build_acceleration_structures<'a, T>( &mut self, _descriptor_count: u32, descriptors: T, ) where super::Api: 'a, T: IntoIterator< Item = crate::BuildAccelerationStructureDescriptor< 'a, super::Buffer, super::AccelerationStructure, >, >, { let command_encoder = self.enter_acceleration_structure_builder(); for descriptor in descriptors { let acceleration_structure_descriptor = conv::map_acceleration_structure_descriptor(descriptor.entries, descriptor.flags); match descriptor.mode { crate::AccelerationStructureBuildMode::Build => { command_encoder .buildAccelerationStructure_descriptor_scratchBuffer_scratchBufferOffset( &descriptor.destination_acceleration_structure.raw, &acceleration_structure_descriptor, &descriptor.scratch_buffer.raw, descriptor.scratch_buffer_offset as usize, ); } crate::AccelerationStructureBuildMode::Update => unsafe { command_encoder.refitAccelerationStructure_descriptor_destination_scratchBuffer_scratchBufferOffset( &descriptor.source_acceleration_structure.unwrap().raw, &acceleration_structure_descriptor, Some(&descriptor.destination_acceleration_structure.raw), Some(&descriptor.scratch_buffer.raw), descriptor.scratch_buffer_offset as usize, ); }, } } } unsafe fn place_acceleration_structure_barrier( &mut self, _barriers: crate::AccelerationStructureBarrier, ) { } unsafe fn read_acceleration_structure_compact_size( &mut self, acceleration_structure: &super::AccelerationStructure, buffer: &super::Buffer, ) { let command_encoder = self.enter_acceleration_structure_builder(); command_encoder.writeCompactedAccelerationStructureSize_toBuffer_offset( &acceleration_structure.raw, &buffer.raw, 0, ); } unsafe fn set_acceleration_structure_dependencies( command_buffers: &[&super::CommandBuffer], dependencies: &[&super::AccelerationStructure], ) { let Some(first_command_buffer) = command_buffers.first() else { return; }; let desc = MTLResidencySetDescriptor::new(); desc.setLabel(first_command_buffer.raw.label().as_deref()); let residency_set = first_command_buffer .raw .device() .newResidencySetWithDescriptor_error(&desc) .unwrap(); for command_buffer in command_buffers { command_buffer.raw.useResidencySet(&residency_set); } for dependency in dependencies { residency_set.addAllocation(ProtocolObject::from_ref(&*dependency.raw)); } residency_set.commit(); } } impl Drop for super::CommandEncoder { fn drop(&mut self) { // Metal raises an assert when a MTLCommandEncoder is deallocated without a call // to endEncoding. This isn't documented in the general case at // https://developer.apple.com/documentation/metal/mtlcommandencoder, but for the // more-specific MTLComputeCommandEncoder it is stated as a requirement at // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder. It // appears to be a requirement for all MTLCommandEncoder objects. Failing to call // endEncoding causes a crash with the message 'Command encoder released without // endEncoding'. To prevent this, we explicitiy call discard_encoding, which // calls endEncoding on any still-held MTLCommandEncoders. unsafe { self.discard_encoding(); } self.counters.command_encoders.sub(1); } } impl Drop for super::CommandBuffer { fn drop(&mut self) { // `command_buffer_created_not_submitted` is usually decremented when the command // buffer is submitted. But if we're dropping a command buffer that was never // submitted, we need to decrement the count here. let status = self.raw.status(); if status == MTLCommandBufferStatus::NotEnqueued || status == MTLCommandBufferStatus::Enqueued { let previous = self .queue_shared .command_buffer_created_not_submitted .fetch_sub(1, atomic::Ordering::AcqRel); debug_assert!(previous > 0); } } }