//! The `shred` module defines data structures and methods to pull MTU sized data frames from the //! network. There are two types of shreds: data and coding. Data shreds contain entry information //! while coding shreds provide redundancy to protect against dropped network packets (erasures). //! //! +---------------------------------------------------------------------------------------------+ //! | Data Shred | //! +---------------------------------------------------------------------------------------------+ //! | common | data | payload | //! | header | header | | //! |+---+---+--- |+---+---+---|+----------------------------------------------------------+----+| //! || s | s | . || p | f | s || data (ie ledger entries) | r || //! || i | h | . || a | l | i || | e || //! || g | r | . || r | a | z || See notes immediately after shred diagrams for an | s || //! || n | e | || e | g | e || explanation of the "restricted" section in this payload | t || //! || a | d | || n | s | || | r || //! || t | | || t | | || | i || //! || u | t | || | | || | c || //! || r | y | || o | | || | t || //! || e | p | || f | | || | e || //! || | e | || f | | || | d || //! |+---+---+--- |+---+---+---+|----------------------------------------------------------+----+| //! +---------------------------------------------------------------------------------------------+ //! //! +---------------------------------------------------------------------------------------------+ //! | Coding Shred | //! +---------------------------------------------------------------------------------------------+ //! | common | coding | payload | //! | header | header | | //! |+---+---+--- |+---+---+---+----------------------------------------------------------------+| //! || s | s | . || n | n | p || data (encoded data shred data) || //! || i | h | . || u | u | o || || //! || g | r | . || m | m | s || || //! || n | e | || | | i || || //! || a | d | || d | c | t || || //! || t | | || | | i || || //! || u | t | || s | s | o || || //! || r | y | || h | h | n || || //! || e | p | || r | r | || || //! || | e | || e | e | || || //! || | | || d | d | || || //! |+---+---+--- |+---+---+---+|+--------------------------------------------------------------+| //! +---------------------------------------------------------------------------------------------+ //! //! Notes: //! a) Coding shreds encode entire data shreds: both of the headers AND the payload. //! b) Coding shreds require their own headers for identification and etc. //! c) The erasure algorithm requires data shred and coding shred bytestreams to be equal in length. //! //! So, given a) - c), we must restrict data shred's payload length such that the entire coding //! payload can fit into one coding shred / packet. pub(crate) use self::merkle_tree::PROOF_ENTRIES_FOR_32_32_BATCH; #[cfg(test)] use rand::Rng; #[cfg(feature = "stable-abi")] use solana_frozen_abi_macro::{StableAbi, StableAbiSample, frozen_abi}; use { self::traits::{Shred as _, ShredData as _}, crate::shred::{merkle_tree::MerkleProofEntry, payload::PayloadMutGuard}, bitflags::bitflags, itertools::Either, num_enum::{IntoPrimitive, TryFromPrimitive}, serde::{Deserialize, Serialize}, solana_clock::Slot, solana_cost_model::shred_limit::{ DEFAULT_MAX_CODE_SHREDS_PER_SLOT, DEFAULT_MAX_DATA_SHREDS_PER_SLOT, }, solana_entry::entry::{Entry, create_ticks}, solana_hash::Hash, solana_pubkey::Pubkey, solana_sha256_hasher::hashv, solana_signature::{SIGNATURE_BYTES, Signature}, static_assertions::const_assert_eq, std::{fmt::Debug, mem::MaybeUninit, ops::Range}, thiserror::Error, wincode::{ SchemaRead, SchemaWrite, TypeMeta, io::{Reader, Writer}, pod_wrapper, }, }; pub use { self::{ merkle::{ShredCode, ShredData}, payload::Payload, stats::{ProcessShredsStats, ShredFetchStats}, }, crate::shredder::Shredder, }; #[cfg(any(test, feature = "dev-context-only-utils"))] use {solana_keypair::Keypair, solana_signer::Signer}; mod common; pub mod filter; pub mod merkle; pub mod merkle_tree; mod payload; mod shred_code; pub(crate) mod shred_data; mod stats; mod traits; pub mod wire; // Alias for shred::wire::* for the old code. // New code should use shred::wire::*. pub mod layout { pub use super::wire::*; } pub type Nonce = u32; const_assert_eq!(SIZE_OF_NONCE, 4); pub const SIZE_OF_NONCE: usize = std::mem::size_of::(); /// The following constants are computed by hand, and hardcoded. /// `test_shred_constants` ensures that the values are correct. const SIZE_OF_COMMON_SHRED_HEADER: usize = 83; pub const SIZE_OF_DATA_SHRED_HEADERS: usize = 88; const SIZE_OF_CODING_SHRED_HEADERS: usize = 89; const SIZE_OF_SIGNATURE: usize = SIGNATURE_BYTES; // Shreds are uniformly split into erasure batches with a "target" number of // data shreds per each batch as below. The actual number of data shreds in // each erasure batch depends on the number of shreds obtained from serializing // a &[Entry]. pub const DATA_SHREDS_PER_FEC_BLOCK: usize = 32; pub const CODING_SHREDS_PER_FEC_BLOCK: usize = 32; pub const SHREDS_PER_FEC_BLOCK: usize = DATA_SHREDS_PER_FEC_BLOCK + CODING_SHREDS_PER_FEC_BLOCK; /// An upper bound on maximum number of data shreds we can handle in a slot /// 32K shreds would allow ~320K peak TPS /// (32K shreds per slot * 4 TX per shred * 2.5 slots per sec) pub const MAX_DATA_SHREDS_PER_SLOT: usize = DEFAULT_MAX_DATA_SHREDS_PER_SLOT as usize; pub const MAX_CODE_SHREDS_PER_SLOT: usize = DEFAULT_MAX_CODE_SHREDS_PER_SLOT as usize; pub const MAX_FEC_SETS_PER_SLOT: u32 = MAX_DATA_SHREDS_PER_SLOT as u32 / DATA_SHREDS_PER_FEC_BLOCK as u32; #[cfg(any(test, feature = "dev-context-only-utils"))] pub(crate) const OFFSET_OF_SHRED_VARIANT: usize = SIZE_OF_SIGNATURE; /// Rewrites the Merkle proof height of the serialized shred in `payload`, /// leaving its signature stale; the caller re-signs if the shred has to verify. #[cfg(any(test, feature = "dev-context-only-utils"))] pub fn override_proof_size(payload: &mut [u8], proof_size: u8) { let byte = &mut payload[OFFSET_OF_SHRED_VARIANT]; let shred_variant = ShredVariant::try_from(*byte).expect("payload holds a merkle shred"); *byte = u8::from(match shred_variant { ShredVariant::MerkleCode { resigned, .. } => ShredVariant::MerkleCode { proof_size, resigned, }, ShredVariant::MerkleData { resigned, .. } => ShredVariant::MerkleData { proof_size, resigned, }, }); assert_eq!( ShredVariant::try_from(*byte) .map(ShredVariant::proof_size) .ok(), Some(proof_size), "proof height {proof_size} does not fit the shred_variant encoding" ); } // Statically compute the typical data batch size assuming: // 1. 32:32 erasure coding batch // 2. Merkles are chained // 3. No retransmit signature (only included for last batch) pub const fn get_data_shred_bytes_per_batch_typical() -> u64 { let capacity = match merkle::ShredData::const_capacity(PROOF_ENTRIES_FOR_32_32_BATCH, false) { Ok(v) => v, Err(_proof_size) => { panic!("this is unreachable"); } }; (DATA_SHREDS_PER_FEC_BLOCK * capacity) as u64 } bitflags! { /// Flags carried by data shreds in [`DataShredHeader::flags`]. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct ShredFlags:u8 { /// Mask selecting the six bits that hold the *reference tick*: how many /// ticks had already elapsed in the slot when the leader produced this /// shred. The leader's value ranges over `[0, ticks_per_slot]` but is /// clamped to 63 on the way in, see /// [`ShredFlags::from_reference_tick`]. const SHRED_TICK_REFERENCE_MASK = 0b0011_1111; /// Flag marking the end of a set of shreds whose concatenated data /// deserializes into a /// [`BlockComponent`](solana_entry::block_component::BlockComponent). const DATA_COMPLETE_SHRED = 0b0100_0000; /// Flag marking the last data shred of the slot; nothing follows it. /// Sets the `DATA_COMPLETE_SHRED` bit as well. const LAST_SHRED_IN_SLOT = 0b1100_0000; } } pod_wrapper! { unsafe struct PodShredFlags(ShredFlags); } impl ShredFlags { /// Creates a new ShredFlags from the given reference_tick /// /// SHRED_TICK_REFERENCE_MASK is comprised of only six bits whereas the /// reference_tick has 8 bits (u8). The reference_tick bits will saturate /// in the event that reference_tick > SHRED_TICK_REFERENCE_MASK pub(crate) fn from_reference_tick(reference_tick: u8) -> Self { Self::from_bits_retain(Self::SHRED_TICK_REFERENCE_MASK.bits().min(reference_tick)) } } #[derive(Debug, Error)] pub enum Error { #[error(transparent)] WincodeRead(#[from] wincode::ReadError), #[error(transparent)] WincodeWrite(#[from] wincode::WriteError), #[error(transparent)] Erasure(#[from] reed_solomon_erasure::Error), #[error("Invalid data size: {size}, payload: {payload}")] InvalidDataSize { size: u16, payload: usize }, #[error("Invalid deshred set")] InvalidDeshredSet, #[error("Invalid erasure config")] InvalidErasureConfig, #[error("Invalid erasure shard index: {0:?}")] InvalidErasureShardIndex(/*headers:*/ Box), #[error("Invalid merkle proof")] InvalidMerkleProof, #[error("Invalid Merkle root")] InvalidMerkleRoot, #[error("Invalid num coding shreds: {0}")] InvalidNumCodingShreds(u16), #[error("Invalid parent_offset: {parent_offset}, slot: {slot}")] InvalidParentOffset { slot: Slot, parent_offset: u16 }, #[error("Invalid parent slot: {parent_slot}, slot: {slot}")] InvalidParentSlot { slot: Slot, parent_slot: Slot }, #[error("Invalid payload size: {0}")] InvalidPayloadSize(/*payload size:*/ usize), #[error("Invalid proof size: {0}")] InvalidProofSize(/*proof_size:*/ u8), #[error("Invalid recovered shred")] InvalidRecoveredShred, #[error("Invalid shard size: {0}")] InvalidShardSize(/*shard_size:*/ usize), #[error("Invalid shred flags: {0}")] InvalidShredFlags(u8), #[error("Invalid {0:?} shred index: {1}")] InvalidShredIndex(ShredType, /*shred index:*/ u32), #[error("Invalid shred type")] InvalidShredType, #[error("Invalid shred variant")] InvalidShredVariant, #[error("Invalid packet size, could not get the shred")] InvalidPacketSize, #[error(transparent)] Io(#[from] std::io::Error), #[error("Unknown proof size")] UnknownProofSize, #[error("Empty shreds list")] EmptyIterator, } #[repr(u8)] #[cfg_attr( feature = "stable-abi", derive(StableAbi, StableAbiSample), frozen_abi( abi_digest = "7R7R5DNkXYiSA35A6p5Ej79TmHSAdLicCo5HUkRAMD9Z", abi_serializer = "wincode", test_roundtrip = "eq_and_wire", ) )] #[derive( Clone, Copy, Debug, Eq, Hash, PartialEq, IntoPrimitive, Serialize, TryFromPrimitive, SchemaWrite, SchemaRead, )] #[wincode(tag_encoding = "u8")] pub enum ShredType { /// Shreds containing serialized block data. #[wincode(tag = 0b1010_0101)] Data = 0b1010_0101, /// Shreds containing Reed-Solomon parity over the data shreds of one /// erasure batch. #[wincode(tag = 0b0101_1010)] Code = 0b0101_1010, } /// Shred type plus the two layout parameters needed to parse everything that /// follows the data buffer, packed into the single byte at /// [`ShredCommonHeader::shred_variant`]. /// /// The high 4 bits identifies the variant and the low 4 bits holds /// `proof_size`: /// ```text /// 0b0110_???? MerkleCode /// 0b0111_???? MerkleCode resigned /// 0b1001_???? MerkleData /// 0b1011_???? MerkleData resigned /// ``` /// Every variant in use today is chained, i.e. the payload embeds the Merkle /// root of the previous erasure batch. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] enum ShredVariant { // 0b01??_???? MerkleCode { /// Number of entries in this shred's Merkle proof, i.e. the height of /// the erasure batch's Merkle tree, `ceil(log2(batch size))`. 6 for the /// standard 32:32 batch, see [`PROOF_ENTRIES_FOR_32_32_BATCH`]. Each /// entry occupies `SIZE_OF_MERKLE_PROOF_ENTRY` bytes of the payload, so /// this directly determines how much room is left for data. Encoded in /// the low 4 bits and therefore capped at 15. proof_size: u8, /// Whether the payload reserves a trailing 64 bytes for the /// retransmitter's signature. Set only on the last erasure batch of a /// slot; those shreds are re-signed by each node as they are forwarded /// down the Turbine tree (see [`wire::resign_packet`]), on top of the /// leader's signature which is left intact. resigned: bool, }, // 0b10??_???? MerkleData { /// See [`ShredVariant::MerkleCode::proof_size`]. proof_size: u8, /// See [`ShredVariant::MerkleCode::resigned`]. resigned: bool, }, } /// The header at the front of every shred, data and coding alike. #[derive(Clone, Copy, Debug, PartialEq, Eq, SchemaRead, SchemaWrite)] struct ShredCommonHeader { /// The leader's signature over the *Merkle root of the erasure batch*, not /// over this shred alone. /// /// Every shred in an erasure batch therefore carries the identical /// signature, which is what lets erasure-recovered shreds simply inherit /// the signature of the shreds they were reconstructed from: rebuilding the /// tree yields the same root, so the existing signature still verifies. /// Occupies the first `SIZE_OF_SIGNATURE` bytes of the payload; everything /// it commits to lies past those bytes. signature: Signature, /// Shred type and the layout parameters (`proof_size`, `resigned`) that /// determine where the Merkle root, proof and retransmitter signature sit /// within the payload. shred_variant: ShredVariant, /// The slot this shred belongs to. slot: Slot, /// Position of this shred within its slot, counting from zero. /// /// Data and coding shreds are numbered by two *independent* per-slot /// counters, both of which restart at zero on every slot: /// /// * On a data shred, `index` is its position among the slot's data shreds, /// bounded by [`MAX_DATA_SHREDS_PER_SLOT`]. /// * On a coding shred, `index` is its position among the slot's coding /// shreds, bounded by [`MAX_CODE_SHREDS_PER_SLOT`]. /// /// `sanitize` rejects shreds at or above the respective bound. Because the /// counters are independent, a data shred and a coding shred of the same /// slot routinely share an `index`; only `(slot, index, shred_type)` — i.e. /// [`ShredId`] — is unique. index: u32, /// The cluster's shred version, derived from the genesis hash and the list /// of hard forks. Nodes discard shreds whose version does not match their /// own. version: u16, /// The `index` of the *first data shred* of this shred's erasure batch. /// /// All shreds of a batch, coding shreds included, carry the same value, so /// `(slot, fec_set_index)` identifies the batch — see [`ErasureSetId`]. /// /// It is slot-relative and restarts on every slot: the first erasure batch /// of a slot has `fec_set_index == 0`, the next `32`, and so on in steps /// of [`DATA_SHREDS_PER_FEC_BLOCK`]. fec_set_index: u32, } /// The header that follows [`ShredCommonHeader`] in a data shred. #[derive(Clone, Copy, Debug, PartialEq, Eq, SchemaRead, SchemaWrite)] struct DataShredHeader { /// How many slots back this slot's parent is: `slot - parent_slot`. /// /// Stored as an offset rather than an absolute slot to save space, which /// caps the gap to a parent at `u16::MAX` slots; `Shredder::new` rejects /// anything longer. Must be non-zero except in slot 0 — a zero offset /// elsewhere would make a slot its own parent, and `parent()` returns /// [`Error::InvalidParentOffset`] for it. parent_offset: u16, /// Reference tick and the batch/slot completion markers. See [`ShredFlags`]. #[wincode(with = "PodShredFlags")] flags: ShredFlags, /// Number of bytes in the payload taken by the headers plus the entry data, /// i.e. `SIZE_OF_DATA_SHRED_HEADERS + `. /// /// A slot's entries are serialized into one long byte stream, which is then /// cut into contiguous chunks, one per data shred. Each chunk is written /// into the shred's *data buffer*: the `ShredData::capacity(proof_size, /// resigned)` bytes of the payload that follow the headers. The buffer is /// fixed-size but a chunk need not fill it, so `size` records where the /// chunk stops and the zero padding begins. `data()` returns exactly /// `payload[SIZE_OF_DATA_SHRED_HEADERS..size]`. /// /// Note this is an offset from the start of the payload, not a length: it /// includes the header bytes, which is why it is never below /// `SIZE_OF_DATA_SHRED_HEADERS`. That lower bound is a shred carrying no /// entry data at all, which is emitted to pad a short erasure batch out to /// [`DATA_SHREDS_PER_FEC_BLOCK`] shreds. The upper bound is /// `SIZE_OF_DATA_SHRED_HEADERS + capacity`; outside that range `data()` /// fails with [`Error::InvalidDataSize`]. Padding past `size` is erasure /// coded along with the rest of the buffer. size: u16, } /// The header that follows [`ShredCommonHeader`] in a coding shred, describing /// the Reed-Solomon configuration of the erasure batch. #[derive(Clone, Copy, Debug, PartialEq, Eq, SchemaRead, SchemaWrite)] struct CodingShredHeader { /// Number of data shreds in this erasure batch — the number of shards that /// must be present, in any combination of data and coding, to recover the /// batch. Currently always [`DATA_SHREDS_PER_FEC_BLOCK`]. num_data_shreds: u16, /// Number of coding shreds in this erasure batch, currently always /// [`CODING_SHREDS_PER_FEC_BLOCK`]. num_coding_shreds: u16, /// This shred's position among the batch's coding shreds, in /// `[0, num_coding_shreds)`. Add `num_data_shreds` to get the shred's index /// into the erasure batch as a whole, since data shreds occupy the lower /// range. position: u16, } /// A single shred: one MTU-sized frame of a slot's ledger data, or the /// redundancy that protects such frames against packet loss. /// /// Both variants hold the fully deserialized headers alongside the wire bytes /// they were parsed from. See the [module docs](self) for the on-wire diagrams, /// and [`merkle::ShredData`] / [`merkle::ShredCode`] for how the Merkle root, /// proof and retransmitter signature are laid out after them. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Shred { /// A redundancy shred, computed by Reed-Solomon coding over the data shreds /// of one erasure batch. /// /// It carries no ledger data of its own. A batch of `num_data_shreds` data /// shreds is extended with `num_coding_shreds` of these (both recorded in /// the coding shred's header), and *any* `num_data_shreds` of the resulting /// total suffice to reconstruct the whole batch. A receiver that loses some /// shreds to the network can therefore rebuild them locally instead of /// asking a peer to repair them. /// /// The erasure crate — and so the recovery code and its errors, e.g. /// `reed_solomon_erasure::Error::TooFewParityShards` — calls these *parity /// shards*. ShredCode(merkle::ShredCode), /// A shred carrying one contiguous chunk of the slot's serialized /// [`Entry`]s. ShredData(merkle::ShredData), } /// Tuple which uniquely identifies a shred should it exists. #[derive(Clone, Copy, Eq, Debug, Hash, PartialEq)] pub struct ShredId(Slot, /*shred index:*/ u32, ShredType); impl ShredId { #[inline] pub fn new(slot: Slot, index: u32, shred_type: ShredType) -> ShredId { ShredId(slot, index, shred_type) } #[inline] pub fn slot(&self) -> Slot { self.0 } #[inline] pub fn index(&self) -> u32 { self.1 } #[inline] pub fn shred_type(&self) -> ShredType { self.2 } #[inline] pub(crate) fn unpack(&self) -> (Slot, /*shred index:*/ u32, ShredType) { (self.0, self.1, self.2) } pub fn seed(&self, leader: &Pubkey) -> [u8; 32] { let ShredId(slot, index, shred_type) = self; hashv(&[ &slot.to_le_bytes(), &u8::from(*shred_type).to_le_bytes(), &index.to_le_bytes(), AsRef::<[u8]>::as_ref(leader), ]) .to_bytes() } } /// Tuple which identifies erasure coding set that the shred belongs to. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct ErasureSetId(Slot, /*fec_set_index:*/ u32); impl ErasureSetId { pub fn new(slot: Slot, fec_set_index: u32) -> Self { Self(slot, fec_set_index) } pub(crate) fn slot(&self) -> Slot { self.0 } pub(crate) fn fec_set_index(&self) -> u32 { self.1 } // Storage key for ErasureMeta and MerkleRootMeta in blockstore db. // Note: ErasureMeta column uses u64 so this will need to be typecast pub(crate) fn store_key(&self) -> (Slot, /*fec_set_index:*/ u32) { (self.0, self.1) } } /// To be used with the [`Shred`] enum. /// /// Writes a function implementation that forwards the invocation to an identically defined function /// in one of the two enum branches. /// /// Due to an inability of a macro to match on the `self` shorthand syntax, this macro has 3 /// branches. But they are only different in the `self` argument matching. Make sure to keep the /// identical otherwise. macro_rules! dispatch { ($vis:vis fn $name:ident(&self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => { #[inline] $vis fn $name(&self $(, $arg:$ty)?) $(-> $out)? { match self { Self::ShredCode(shred) => shred.$name($($arg, )?), Self::ShredData(shred) => shred.$name($($arg, )?), } } }; ($vis:vis fn $name:ident(self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => { #[inline] $vis fn $name(self $(, $arg:$ty)?) $(-> $out)? { match self { Self::ShredCode(shred) => shred.$name($($arg, )?), Self::ShredData(shred) => shred.$name($($arg, )?), } } }; ($vis:vis fn $name:ident(&mut self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => { #[inline] $vis fn $name(&mut self $(, $arg:$ty)?) $(-> $out)? { match self { Self::ShredCode(shred) => shred.$name($($arg, )?), Self::ShredData(shred) => shred.$name($($arg, )?), } } } } use wincode::config::ConfigCore; impl Shred { dispatch!(fn common_header(&self) -> &ShredCommonHeader); dispatch!(fn set_signature(&mut self, signature: Signature)); dispatch!(fn signed_data(&self) -> Result); dispatch!(pub fn sanitize(&self) -> Result<(), Error>); dispatch!(pub fn chained_merkle_root(&self) -> Result); dispatch!(fn set_chained_merkle_root(&mut self, chained_merkle_root: &Hash) -> Result<(), Error>); dispatch!(pub fn retransmitter_signature_offset(&self) -> Result); dispatch!(pub(crate) fn retransmitter_signature(&self) -> Result); #[cfg(test)] dispatch!(fn set_retransmitter_signature(&mut self, signature: &Signature) -> Result<(), Error>); dispatch!(pub fn payload(&self) -> &Payload); dispatch!(pub fn into_payload(self) -> Payload); dispatch!(pub fn merkle_root(&self) -> Result); dispatch!(fn merkle_node(&self) -> Result); dispatch!(pub fn proof_size(&self) -> Result); /// Returns true if the Merkle proof size is consistent with a fixed 32:32 /// erasure set, whose 64 leaves fix the proof height. #[inline] pub fn has_correct_proof_size(&self) -> bool { self.common_header().shred_variant.has_correct_proof_size() } dispatch!(fn erasure_shard_mut(&mut self) -> Result>, Error>); dispatch!(fn erasure_shard_index(&self) -> Result); #[cfg(test)] dispatch!(pub(crate) fn erasure_shard(&self) -> Result<&[u8], Error>); pub(super) fn from_payload>(shred: T) -> Result where Payload: From, { match layout::get_shred_variant(shred.as_ref())? { ShredVariant::MerkleCode { .. } => Ok(Self::ShredCode(ShredCode::from_payload(shred)?)), ShredVariant::MerkleData { .. } => Ok(Self::ShredData(ShredData::from_payload(shred)?)), } } pub fn new_from_serialized_shred(shred: T) -> Result where T: AsRef<[u8]> + Into, Payload: From, { Self::from_payload(shred) } /// Unique identifier for each shred. pub fn id(&self) -> ShredId { ShredId(self.slot(), self.index(), self.shred_type()) } pub fn slot(&self) -> Slot { self.common_header().slot } pub fn parent(&self) -> Result { match self { Self::ShredCode(_) => Err(Error::InvalidShredType), Self::ShredData(shred) => shred.parent(), } } /// Returns the application data carried by a data shred. pub fn data(&self) -> Result<&[u8], Error> { match self { Self::ShredCode(_) => Err(Error::InvalidShredType), Self::ShredData(shred) => shred.data(), } } pub fn index(&self) -> u32 { self.common_header().index } pub fn fec_set_index(&self) -> u32 { self.common_header().fec_set_index } pub(crate) fn first_coding_index(&self) -> Option { match self { Self::ShredCode(shred) => shred.first_coding_index(), Self::ShredData(_) => None, } } pub fn version(&self) -> u16 { self.common_header().version } // Identifier for the erasure coding set that the shred belongs to. pub(crate) fn erasure_set(&self) -> ErasureSetId { ErasureSetId(self.slot(), self.fec_set_index()) } pub fn signature(&self) -> &Signature { &self.common_header().signature } #[cfg(feature = "dev-context-only-utils")] pub fn sign(&mut self, keypair: &Keypair) { let data = self.signed_data().unwrap(); let signature = keypair.sign_message(data.as_ref()); self.set_signature(signature); } #[inline] pub fn shred_type(&self) -> ShredType { ShredType::from(self.common_header().shred_variant) } #[inline] pub fn is_data(&self) -> bool { self.shred_type() == ShredType::Data } #[inline] pub fn is_code(&self) -> bool { self.shred_type() == ShredType::Code } pub fn last_in_slot(&self) -> bool { match self { Self::ShredCode(_) => false, Self::ShredData(shred) => shred.last_in_slot(), } } pub fn data_complete(&self) -> bool { match self { Self::ShredCode(_) => false, Self::ShredData(shred) => shred.data_complete(), } } #[cfg(test)] pub(crate) fn reference_tick(&self) -> u8 { match self { Self::ShredCode(_) => ShredFlags::SHRED_TICK_REFERENCE_MASK.bits(), Self::ShredData(shred) => shred.reference_tick(), } } #[must_use] pub fn verify(&self, pubkey: &Pubkey) -> bool { match self.signed_data() { Ok(data) => self.signature().verify(pubkey.as_ref(), data.as_ref()), Err(_) => false, } } // Returns true if the erasure coding of the two shreds mismatch. pub(crate) fn erasure_mismatch(&self, other: &Self) -> Result { match (self, other) { (Self::ShredCode(shred), Self::ShredCode(other)) => Ok(shred.erasure_mismatch(other)), _ => Err(Error::InvalidShredType), } } pub(crate) fn num_data_shreds(&self) -> Result { match self { Self::ShredCode(shred) => Ok(shred.num_data_shreds()), Self::ShredData(_) => Err(Error::InvalidShredType), } } pub(crate) fn num_coding_shreds(&self) -> Result { match self { Self::ShredCode(shred) => Ok(shred.num_coding_shreds()), Self::ShredData(_) => Err(Error::InvalidShredType), } } /// Returns true if the other shred has the same ShredId, i.e. (slot, index, /// shred-type), but different payload. /// Retransmitter's signature is ignored when comparing payloads. pub fn is_shred_duplicate(&self, other: &Shred) -> bool { if self.id() != other.id() { return false; } wire::get_payload_without_retransmitter_signature(self.payload()) != wire::get_payload_without_retransmitter_signature(other.payload()) } #[inline] fn set_merkle_proof<'a, I>(&mut self, proof: I) -> Result<(), Error> where I: IntoIterator>, { match self { Self::ShredCode(shred) => shred.set_merkle_proof(proof), Self::ShredData(shred) => shred.set_merkle_proof(proof), } } #[inline] fn merkle_proof(&self) -> Result, Error> { match self { Self::ShredCode(shred) => shred.merkle_proof().map(Either::Left), Self::ShredData(shred) => shred.merkle_proof().map(Either::Right), } } } impl ShredVariant { #[inline] fn proof_size(self) -> u8 { match self { ShredVariant::MerkleCode { proof_size, .. } | ShredVariant::MerkleData { proof_size, .. } => proof_size, } } /// Returns true if the Merkle proof size is consistent with a fixed 32:32 /// erasure set, whose 64 leaves fix the proof height. #[inline] fn has_correct_proof_size(self) -> bool { self.proof_size() == PROOF_ENTRIES_FOR_32_32_BATCH } } impl From for ShredType { #[inline] fn from(shred_variant: ShredVariant) -> Self { match shred_variant { ShredVariant::MerkleCode { .. } => ShredType::Code, ShredVariant::MerkleData { .. } => ShredType::Data, } } } impl From for u8 { #[inline] fn from(shred_variant: ShredVariant) -> u8 { match shred_variant { ShredVariant::MerkleCode { proof_size, resigned: false, } => proof_size | 0x60, ShredVariant::MerkleCode { proof_size, resigned: true, } => proof_size | 0x70, ShredVariant::MerkleData { proof_size, resigned: false, } => proof_size | 0x90, ShredVariant::MerkleData { proof_size, resigned: true, } => proof_size | 0xb0, } } } impl TryFrom for ShredVariant { type Error = Error; #[inline] fn try_from(shred_variant: u8) -> Result { if shred_variant == u8::from(ShredType::Code) || shred_variant == u8::from(ShredType::Data) { Err(Error::InvalidShredVariant) } else { let proof_size = shred_variant & 0x0F; match shred_variant & 0xF0 { 0x60 => Ok(ShredVariant::MerkleCode { proof_size, resigned: false, }), 0x70 => Ok(ShredVariant::MerkleCode { proof_size, resigned: true, }), 0x90 => Ok(ShredVariant::MerkleData { proof_size, resigned: false, }), 0xb0 => Ok(ShredVariant::MerkleData { proof_size, resigned: true, }), _ => Err(Error::InvalidShredVariant), } } } } unsafe impl SchemaWrite for ShredVariant { type Src = Self; const TYPE_META: TypeMeta = TypeMeta::Static { size: 1, zero_copy: false, }; fn size_of(_src: &Self::Src) -> wincode::WriteResult { Ok(1) } fn write(writer: impl Writer, src: &Self::Src) -> wincode::WriteResult<()> { let repr: u8 = (*src).into(); >::write(writer, &repr) } } unsafe impl<'a, C: ConfigCore> SchemaRead<'a, C> for ShredVariant { type Dst = Self; const TYPE_META: TypeMeta = TypeMeta::Static { size: 1, zero_copy: false, }; fn read(reader: impl Reader<'a>, dst: &mut MaybeUninit) -> wincode::ReadResult<()> { let repr = >::get(reader)?; let value = Self::try_from(repr) .map_err(|_| wincode::ReadError::InvalidTagEncoding(repr as usize))?; dst.write(value); Ok(()) } } pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option) -> u64 { let ticks = create_ticks(1, 0, Hash::default()); max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size) } pub fn max_entries_per_n_shred( entry: &Entry, num_shreds: u64, shred_data_size: Option, ) -> u64 { // Default 32:32 erasure batches yields 64 shreds; log2(64) = 6. let data_buffer_size = ShredData::capacity(/*proof_size:*/ 6, /*resigned:*/ false).unwrap(); let shred_data_size = shred_data_size.unwrap_or(data_buffer_size) as u64; let vec_size = wincode::serialized_size(&vec![entry]).unwrap(); let entry_size = wincode::serialized_size(entry).unwrap(); let count_size = vec_size - entry_size; (shred_data_size * num_shreds - count_size) / entry_size } #[cfg(feature = "dev-context-only-utils")] pub fn verify_test_data_shred( shred: &Shred, index: u32, slot: Slot, parent: Slot, pk: &Pubkey, verify: bool, is_last_in_slot: bool, is_last_data: bool, ) { shred.sanitize().unwrap(); assert!(shred.is_data()); assert_eq!(shred.index(), index); assert_eq!(shred.slot(), slot); assert_eq!(shred.parent().unwrap(), parent); assert_eq!(verify, shred.verify(pk)); if is_last_in_slot { assert!(shred.last_in_slot()); } else { assert!(!shred.last_in_slot()); } if is_last_data { assert!(shred.data_complete()); } else { assert!(!shred.data_complete()); } } #[cfg(test)] pub(crate) fn make_merkle_shreds_for_tests( rng: &mut R, slot: Slot, data_size: usize, is_last_in_slot: bool, ) -> Result, Error> { let chained_merkle_root = Hash::new_from_array(rng.random()); let parent_offset = rng.random_range(1..=u16::try_from(slot).unwrap_or(u16::MAX)); let parent_slot = slot.checked_sub(u64::from(parent_offset)).unwrap(); let mut data = vec![0u8; data_size]; let fec_set_index = rng.random_range(0..21) * DATA_SHREDS_PER_FEC_BLOCK as u32; rng.fill(&mut data[..]); // Resigned shreds from older leaders must still be admitted. merkle::resigned_for_tests::make_shreds_from_data( &Keypair::new(), chained_merkle_root, &data[..], slot, parent_slot, rng.random(), // shred_version rng.random_range(1..64), // reference_tick is_last_in_slot, fec_set_index, // next_shred_index fec_set_index, // next_code_index &mut ProcessShredsStats::default(), ) } #[cfg(test)] mod tests { use { super::*, assert_matches::assert_matches, rand_chacha::{ChaChaRng, rand_core::SeedableRng}, solana_keypair::keypair_from_seed, solana_perf::packet::BytesPacket, test_case::test_case, }; pub(super) const SIZE_OF_SHRED_INDEX: usize = 4; pub(super) const SIZE_OF_SHRED_SLOT: usize = 8; pub(super) const SIZE_OF_SHRED_VARIANT: usize = 1; pub(super) const SIZE_OF_VERSION: usize = 2; pub(super) const SIZE_OF_FEC_SET_INDEX: usize = 4; pub(super) const SIZE_OF_PARENT_OFFSET: usize = 2; pub(super) const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_VARIANT; pub(super) const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SLOT; pub(super) const OFFSET_OF_FEC_SET_INDEX: usize = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + SIZE_OF_VERSION; pub(super) const OFFSET_OF_NUM_DATA: usize = OFFSET_OF_FEC_SET_INDEX + SIZE_OF_FEC_SET_INDEX; pub(super) const OFFSET_OF_PARENT_OFFSET: usize = OFFSET_OF_FEC_SET_INDEX + SIZE_OF_FEC_SET_INDEX; pub(super) const OFFSET_OF_SHRED_FLAGS: usize = OFFSET_OF_PARENT_OFFSET + SIZE_OF_PARENT_OFFSET; pub(super) const OFFSET_OF_DATA_SIZE: usize = OFFSET_OF_SHRED_FLAGS + 1; #[test] fn test_shred_constants() { let common_header = ShredCommonHeader { signature: Signature::default(), shred_variant: ShredVariant::MerkleCode { proof_size: 0, resigned: false, }, slot: Slot::MAX, index: u32::MAX, version: u16::MAX, fec_set_index: u32::MAX, }; let data_shred_header = DataShredHeader { parent_offset: u16::MAX, flags: ShredFlags::all(), size: u16::MAX, }; let coding_shred_header = CodingShredHeader { num_data_shreds: u16::MAX, num_coding_shreds: u16::MAX, position: u16::MAX, }; assert_eq!( SIZE_OF_COMMON_SHRED_HEADER, wincode::serialized_size(&common_header).unwrap() as usize ); assert_eq!( SIZE_OF_CODING_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER, wincode::serialized_size(&coding_shred_header).unwrap() as usize ); assert_eq!( SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER, wincode::serialized_size(&data_shred_header).unwrap() as usize ); let data_shred_header_with_size = DataShredHeader { size: 1000, ..data_shred_header }; assert_eq!( SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_COMMON_SHRED_HEADER, wincode::serialized_size(&data_shred_header_with_size).unwrap() as usize ); assert_eq!( SIZE_OF_SIGNATURE, wincode::serialized_size(&Signature::default()).unwrap() as usize ); assert_eq!( SIZE_OF_SHRED_VARIANT, wincode::serialized_size(&ShredVariant::MerkleCode { proof_size: 15, resigned: true }) .unwrap() as usize ); assert_eq!( SIZE_OF_SHRED_SLOT, wincode::serialized_size(&Slot::default()).unwrap() as usize ); assert_eq!( SIZE_OF_SHRED_INDEX, wincode::serialized_size(&common_header.index).unwrap() as usize ); } #[test] fn test_shred_flags_reference_tick_saturates() { const MAX_REFERENCE_TICK: u8 = ShredFlags::SHRED_TICK_REFERENCE_MASK.bits(); for tick in 0..=u8::MAX { let flags = ShredFlags::from_reference_tick(tick); assert_eq!(flags.bits(), tick.min(MAX_REFERENCE_TICK)); } } #[test] fn test_invalid_parent_offset() { let keypair = Keypair::new(); let shred = Shredder::single_shred_for_tests(10, &keypair); assert_matches!(shred.parent(), Ok(9)); let mut payload = shred.payload().to_vec(); wire::corrupt_and_set_parent_offset(&mut payload, 1000); let shred_res = Shred::new_from_serialized_shred(payload); assert_matches!( shred_res, Err(Error::InvalidParentOffset { slot: 10, parent_offset: 1000 }) ); } // Asserts that ShredType is backward compatible with u8. #[test] fn test_shred_type_compat() { assert_eq!(std::mem::size_of::(), std::mem::size_of::()); assert_matches!(ShredType::try_from(0u8), Err(_)); assert_matches!(ShredType::try_from(1u8), Err(_)); assert_matches!(wincode::deserialize::(&[0u8]), Err(_)); assert_matches!(wincode::deserialize::(&[1u8]), Err(_)); // data shred assert_eq!(ShredType::Data as u8, 0b1010_0101); assert_eq!(u8::from(ShredType::Data), 0b1010_0101); assert_eq!(ShredType::try_from(0b1010_0101), Ok(ShredType::Data)); let buf = wincode::serialize(&ShredType::Data).unwrap(); assert_eq!(buf, vec![0b1010_0101]); assert_matches!( wincode::deserialize::(&[0b1010_0101]), Ok(ShredType::Data) ); // coding shred assert_eq!(ShredType::Code as u8, 0b0101_1010); assert_eq!(u8::from(ShredType::Code), 0b0101_1010); assert_eq!(ShredType::try_from(0b0101_1010), Ok(ShredType::Code)); let buf = wincode::serialize(&ShredType::Code).unwrap(); assert_eq!(buf, vec![0b0101_1010]); assert_matches!( wincode::deserialize::(&[0b0101_1010]), Ok(ShredType::Code) ); } #[test] fn test_shred_variant_compat() { assert_matches!(ShredVariant::try_from(0u8), Err(_)); assert_matches!(ShredVariant::try_from(1u8), Err(_)); assert_matches!(ShredVariant::try_from(0b0101_0000), Err(_)); assert_matches!(ShredVariant::try_from(0b1010_0000), Err(_)); assert_matches!(wincode::deserialize::(&[0b0101_0000]), Err(_)); assert_matches!(wincode::deserialize::(&[0b1010_0000]), Err(_)); assert_matches!(ShredVariant::try_from(0b0101_1010), Err(_)); assert_matches!(wincode::deserialize::(&[0b0101_1010]), Err(_)); assert_matches!(ShredVariant::try_from(0b1010_0101), Err(_)); assert_matches!(wincode::deserialize::(&[0b1010_0101]), Err(_)); } #[test_case(false, 0b0110_0000)] #[test_case(true, 0b0111_0000)] fn test_shred_variant_compat_merkle_code(resigned: bool, byte: u8) { for proof_size in 0..=15u8 { let byte = byte | proof_size; assert_eq!( u8::from(ShredVariant::MerkleCode { proof_size, resigned, }), byte ); assert_eq!( ShredType::from(ShredVariant::MerkleCode { proof_size, resigned, }), ShredType::Code ); assert_eq!( ShredVariant::try_from(byte).unwrap(), ShredVariant::MerkleCode { proof_size, resigned, }, ); let wincode_buf = wincode::serialize(&ShredVariant::MerkleCode { proof_size, resigned, }) .unwrap(); assert_eq!(wincode_buf, vec![byte]); assert_eq!( wincode::deserialize::(&[byte]).unwrap(), ShredVariant::MerkleCode { proof_size, resigned, } ); } } #[test_case(false, 0b1001_0000)] #[test_case(true, 0b1011_0000)] fn test_shred_variant_compat_merkle_data(resigned: bool, byte: u8) { for proof_size in 0..=15u8 { let byte = byte | proof_size; assert_eq!( u8::from(ShredVariant::MerkleData { proof_size, resigned, }), byte ); assert_eq!( ShredType::from(ShredVariant::MerkleData { proof_size, resigned, }), ShredType::Data ); assert_eq!( ShredVariant::try_from(byte).unwrap(), ShredVariant::MerkleData { proof_size, resigned } ); let wincode_buf = wincode::serialize(&ShredVariant::MerkleData { proof_size, resigned, }) .unwrap(); assert_eq!(wincode_buf, vec![byte]); assert_eq!( wincode::deserialize::(&[byte]).unwrap(), ShredVariant::MerkleData { proof_size, resigned } ); } } #[test] fn test_shred_seed() { let mut rng = ChaChaRng::from_seed([147u8; 32]); let leader = Pubkey::new_from_array(rng.random()); let key = ShredId( 141939602, // slot 28685, // index ShredType::Data, ); assert_eq!( bs58::encode(key.seed(&leader)).into_string(), "Gp4kUM4ZpWGQN5XSCyM9YHYWEBCAZLa94ZQuSgDE4r56" ); let leader = Pubkey::new_from_array(rng.random()); let key = ShredId( 141945197, // slot 23418, // index ShredType::Code, ); assert_eq!( bs58::encode(key.seed(&leader)).into_string(), "G1gmFe1QUM8nhDApk6BqvPgw3TQV2Qc5bpKppa96qbVb" ); } fn verify_shred_layout(shred: &Shred, packet: &BytesPacket) { let data = layout::get_shred(packet).unwrap(); assert_eq!(data, packet.data(..).unwrap()); assert_eq!(layout::get_slot(data), Some(shred.slot())); assert_eq!(layout::get_index(data), Some(shred.index())); assert_eq!(layout::get_version(data), Some(shred.version())); assert_eq!(layout::get_shred_id(data), Some(shred.id())); assert_eq!(layout::get_signature(data), Some(*shred.signature())); assert_eq!(layout::get_shred_type(data).unwrap(), shred.shred_type()); if shred.shred_type() == ShredType::Data { let parent_offset = layout::get_parent_offset(data).unwrap(); let slot = layout::get_slot(data).unwrap(); let parent = slot.checked_sub(Slot::from(parent_offset)).unwrap(); assert_eq!(parent, shred.parent().unwrap()); } } #[test] fn test_serde_compat_shred_data() { // bytes of a serialized merkle data shred const PAYLOAD: &str = "aX2ovF3sZRfd6HyqMow9kkrtL3MyJd52m7gvuSjcvA4qayXZ\ cVPhjURcs4JX86YQM8wVrKXqdneqdEUJwBWhFrxSkegDSov6NQoK89SzZi9auEXHHr35dmN\ 4zQbxuNdPjKM2K7b7WKRWaHyoMKQfG9jDbJGcWqwVkAxBmUXZQKryHvAqyNdBuRTdWrMtPK\ DiJWhqVWTmokpyGNceL7mqVr3VrLby6dEuiEUCBHCkhbsXBjfpFZk4yRoSKosb7BViTWWdt\ pWd7NrbDSiE97sBppEU1nWTPaVQh3bu91x8dEoYk696k532MxnhRLcKeL4XzG6P2HzypAck\ JdXiRJDn5E3woA8aiPojqdN9ScthJ8yXq1h4HhvzTRWkRxRBpJL8HEYPBcshwuMLDZ9iBsW\ SFZLmj5v1xH3kDnMuNYJg6Dau6PKHnZyD15tTyFtFtMaXaBc35RqYhsM7s8JuQ9tJ1UfFwd\ khHa1wdrmTWGcvq9DDmALuTtejH1ccoW43GiYSs1TmByJWjRtupvLzMRifZZ7meaGbUBgHU\ kA6t1VN3akoZ9BhdX561KpFGABxTU4NxyFqztEy1EB5EJYtTHwtbJQb1NmNMwKFkazXkn1o\ uKK6drH5y19roH3mMo2JykapbvzYPDBSXUwKQWe1RqSvogapwPxm1EzSRDeXNDP6EYUJJjj\ TAnckNatpT5UZDz4EhpaSbUzd9b5ztqsdPp9HxeBTm412GopAXKN5iSXSPS2WvrEdnANFD7\ tRV3a6PM2SfwpF6eFM5J7xXGJSoPm5TWJSPBMbxttxVFUETSRrBubEsd24aymYZZePJtHr7\ Q8S1deygcyXH5WhhYAmR23hNPv3nUUHe8iwJfaFg73Ncjr8fQBVjwePEy9JKT5jNG5sm87q\ e2RrHEWEwkNKnNgUknoVMbL7y3wmGFpP8VoKTgP51EjMDz7JTxnVsZeRsSp29STteGKbq4i\ wiC5EmMS5K86CAJ86FYt1kXXHJBSw4D79wAMgxRDDycp5PgdowdLxAbwySgpmwdfnxnSD4h\ Y8mo4jLGWokP1mGdgjnPmtMbzndiQCLPjpUcbZoVc6SQrTDCufupkJhy1ewo64yA1db6T2T\ ASTWSHJkjzaWt7QtFfnBo8WoXQrNKw5pyKAQsmP7n6r1SVD7tASfcZAjfaFHxkVvMpKwTQF\ dy9WHxREeCPK3yeN7ACT75RgRuRT1shC1PRCuAu4EFGnBmr3nWuDrYNCG5WrWuW6RRoMyB3\ YaXqjYMXRUVuwb5h2PBP9euBb96Ntung8ihWXa2mbKMYMtmaoYCDhYYrFYszYfdgQH68JYz\ AXZvjFH1SxCETfiXAWGD1aYDa33rXZLcLVx637igoydr77qmzo5YozRQnuXUiJ19PScLWic\ 8jWeVmQ6Mm7BLoGhVPyYbJBeyX5HRwh8CNeLK2ekmhFz9MypB1rM2PXUfcnr2MXS9WRK8bh\ sy47awNdApPdN3RxmuyPLnvmN6FsG5fUNqF8rsz9KUiJh9C4ziYf6NSZvVG2c1KFsQRyFrS\ BzyjqqxBrH1xereV9YNr1gNamFjhZTncpGcPQf9oAoA4LQeSAZXR1dMtfktCs1fFWVbA67F\ dQ1GrpZVGTsZCbuw7Tspns8WoL158AdS7"; let mut rng = { let seed = [1u8; 32]; ChaChaRng::from_seed(seed) }; let mut seed = [0u8; Keypair::SECRET_KEY_LENGTH]; rng.fill(&mut seed[..]); let mut data = [0u8; 4096]; rng.fill(&mut data[..]); let keypair = keypair_from_seed(&seed).unwrap(); let slot = 142076266; let shredder = Shredder::new(slot, slot.saturating_sub(1), 0, 42).unwrap(); let mut shred = shredder .make_shreds_from_data_slice( &keypair, &data, false, Hash::default(), 64, 64, &mut ProcessShredsStats::default(), ) .unwrap() .into_iter() .next() .unwrap(); shred.sign(&keypair); assert!(shred.verify(&keypair.pubkey())); assert_matches!(shred.sanitize(), Ok(())); let payload = bs58::decode(PAYLOAD).into_vec().unwrap(); let packet = BytesPacket::from_bytes(None, payload.clone()); assert_eq!( shred, Shred::new_from_serialized_shred(payload.to_vec()).unwrap() ); verify_shred_layout(&shred, &packet); } #[test] fn test_serde_compat_shred_data_empty() { // bytes of a serialized merkle data shred const PAYLOAD: &str = "HV7qJBe3jCM8aRd4HAXJnJzyDvNYDYsPjjaaK2tTFTxJU2Qj\ 7i87e45TzCg2Vv4rrcBznfs8212svH8aXsM2WYDPst43KyDz99FesBZ8aasxhkUgHGg3Smc\ Pa7opSARcYBpQAG2UHRYFmoPsj3hXADsX5C8JBM3jyHLtbQ78CH11J2dh7J4ps8JxCcUsq7\ E7PVs7NgFku54c8gBuhBuAMykvvSyhGRjyXCL17feubvA8WQyMJz27eXk8hE6LGs98ucsV9\ pScMuXVbAL4rT2cW9gN77QP4mBohJ8miWbMYbi7eLxiXJ9nA6i7XZd32GKscf3Ln8PK7NFT\ JpKxinptw2vd3MuVQRQjNuyZLTEjJSiaxoR1mhSKet27PCTkPWRAxPfMCvLNY2mtdixnFfk\ BdKj1nrwHcQYqNnKHjg3axRhx58QQ7VX5LSNGRtLByZRFMq9wCy7zz7HiMKvMZzddqixHcx\ EFeaM6YFh2sfmW3AKz4pS9s6XL3mAD9MtFvRJUXupNK9P3XtA7BGU8Z8AyYnVi4wxfaWoSd\ 22nhDmMuHTRNHgUDxCEPHfE5enuR3PG4q1DGpVyV9rM6678qFG3cUKdTbJw85uFAkedEhAn\ RsU5u7H2DUTMrZ3AsLGuRmRoLZdxcFGe9GJmWY3WYEBJsdrJUVTnQVDsgvcc7HiFCoGBmGG\ KPS19ncA7Ynie1iokXCjwZNaQf3fpUMksGzkqjfYiFfGQDMNTsPUdcHFAae5Nmh3p8bduG6\ TNHn3A4LoSMX8wwM6Sn4XL3fFLHrkWTV8CJfg67AoqpzgGiWQNdeK2HRABGRbamUxXSzWhA\ tJ7yEPrh9tKTX851mpPjrFHxpu68xDL3t5nd18mtALJ5n5gmTsXMwUxpt9GfGrZyHXteZfH\ jaMLmSqvmDAFH1xADU2SZucRDbAWsuaWUMwaPXCNtiozgJ6uRnyxmwBhikChgtxDSZmRALA\ uYwEnCE8uj4NWvryeVpzfy6m9tqYutCkRsbNodaGmBZ2KGtg4iaQjj4iCK9jKKAxpYVnxbr\ n2jtFezUTsR9dqFh94c9Aa4NdPgkr19hcqabqBZyzan3xP3Jvs78Z5uqkSUVtXP3t3b5ozd\ qjYRMkgwsgGpNqcmBLGANiPXrQ8SseoNsCs3Xtv6Vf7oGg1St3teXSrtWMbsouK3uF8DPgn\ S14yUtrs8cyXX53QMCEuY1wKcoQZwSWqL86FTvZUA3vT6SYvjVKYxAXwNP1ouKdwtePfwdH\ wdM37RT9SMht4BLEXutSCcBys1K3pTWWwqGB87A26apuG1TiqeEugv1FrjprEKyt84S3FgM\ 5eJfdN5NDArvw9bBR81UoYmyZgX5pEY6gNg2xw28Gd7gH9TVe5Y31iggni2oJ3GuBj3R7Ma\ umdo8rE1S4tBWVGsXikv9KFDtMmT2sMmeFuAwDbUZEGsBCAfK7EQpKcYSv7KajtgLpqi6JW\ RP8nBR7FkKR9qv5khhiBLuRzfSxwtADXZknFRU6bh8Ba6JmLhjhqkCYETizXmZVrjvy4gLZ\ we5YHZW5uhbthAzfcLRxuTxnPbyW7LehgwkYde64b8PzqYepYUtxqFHuJSwddis1VuoA2Lb\ M3SeyTe262Q7gUiPEjwQdRXKwUAgGrxVu"; let mut rng = { let seed = [1u8; 32]; ChaChaRng::from_seed(seed) }; let mut seed = [0u8; Keypair::SECRET_KEY_LENGTH]; rng.fill(&mut seed[..]); let keypair = keypair_from_seed(&seed).unwrap(); // PAYLOAD was generated by an older leader, which resigned the last FEC set. let slot = 142076266; let shredder = Shredder::new(slot, slot - 1, 0, 42).unwrap(); let is_last_in_slot = true; let (mut shreds, _) = shredder.entries_to_resigned_merkle_shreds_for_tests( &keypair, &[], is_last_in_slot, Hash::default(), 0, 0, ); let mut shred = shreds.pop().unwrap(); shred.sign(&keypair); assert!(shred.verify(&keypair.pubkey())); assert_matches!(shred.sanitize(), Ok(())); let payload = bs58::decode(PAYLOAD).into_vec().unwrap(); let packet = BytesPacket::from_bytes(None, payload.clone()); assert_eq!( shred, Shred::new_from_serialized_shred(payload.to_vec()).unwrap() ); verify_shred_layout(&shred, &packet); } #[test] fn test_shred_flags_serde() { use wincode::{Deserialize as _, Serialize as _}; let flags = PodShredFlags::deserialize(&[0b0001_0101]).unwrap(); assert_eq!(flags, ShredFlags::from_bits(0b0001_0101).unwrap()); assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 21u8); assert_eq!(PodShredFlags::serialize(&flags).unwrap(), [0b0001_0101]); let flags = PodShredFlags::deserialize(&[0b0111_0001]).unwrap(); assert_eq!(flags, ShredFlags::from_bits(0b0111_0001).unwrap()); assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 49u8); assert_eq!(PodShredFlags::serialize(&flags).unwrap(), [0b0111_0001]); let flags = PodShredFlags::deserialize(&[0b1110_0101]).unwrap(); assert_eq!(flags, ShredFlags::from_bits(0b1110_0101).unwrap()); assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 37u8); assert_eq!(PodShredFlags::serialize(&flags).unwrap(), [0b1110_0101]); let flags = PodShredFlags::deserialize(&[0b1011_1101]).unwrap(); assert_eq!(flags, ShredFlags::from_bits(0b1011_1101).unwrap()); assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); assert_eq!((flags & ShredFlags::SHRED_TICK_REFERENCE_MASK).bits(), 61u8); assert_eq!(PodShredFlags::serialize(&flags).unwrap(), [0b1011_1101]); } // Verifies that LAST_SHRED_IN_SLOT also implies DATA_COMPLETE_SHRED. #[test] fn test_shred_flags_data_complete() { use wincode::Deserialize as _; let mut flags = ShredFlags::empty(); assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); flags.insert(ShredFlags::LAST_SHRED_IN_SLOT); assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); let mut flags = ShredFlags::from_bits(0b0011_1111).unwrap(); assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); flags |= ShredFlags::LAST_SHRED_IN_SLOT; assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); let mut flags = PodShredFlags::deserialize(&[0b1011_1111]).unwrap(); assert!(!flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(!flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); flags.insert(ShredFlags::LAST_SHRED_IN_SLOT); assert!(flags.contains(ShredFlags::DATA_COMPLETE_SHRED)); assert!(flags.contains(ShredFlags::LAST_SHRED_IN_SLOT)); } #[test_case(true)] #[test_case(false)] fn test_is_shred_duplicate(is_last_in_slot: bool) { fn fill_retransmitter_signature( rng: &mut R, shred: Shred, is_last_in_slot: bool, ) -> Shred { let mut shred = shred.into_payload(); let mut signature = [0u8; SIGNATURE_BYTES]; rng.fill(&mut signature[..]); let out = layout::set_retransmitter_signature( &mut shred.as_mut(), &Signature::from(signature), ); if is_last_in_slot { assert_matches!(out, Ok(())); } else { assert_matches!(out, Err(Error::InvalidShredVariant)); } Shred::new_from_serialized_shred(shred).unwrap() } let mut rng = rand::rng(); let slot = 285_376_049 + rng.random_range(0..100_000); let shreds: Vec<_> = make_merkle_shreds_for_tests( &mut rng, slot, 1200 * 5, // data_size is_last_in_slot, ) .unwrap() .into_iter() .map(|shred| fill_retransmitter_signature(&mut rng, shred, is_last_in_slot)) .collect(); { let num_data_shreds = shreds.iter().filter(|shred| shred.is_data()).count(); let num_coding_shreds = shreds.iter().filter(|shred| shred.is_code()).count(); assert!(num_data_shreds > if is_last_in_slot { 31 } else { 5 }); assert!(num_coding_shreds > if is_last_in_slot { 31 } else { 20 }); } // Shreds of different (slot, index, shred-type) are not duplicate. // A shred is not a duplicate of itself either. for shred in &shreds { for other in &shreds { assert!(!shred.is_shred_duplicate(other)); } } // Different retransmitter signature does not make shreds duplicate. for shred in &shreds { let other = fill_retransmitter_signature(&mut rng, shred.clone(), is_last_in_slot); if is_last_in_slot { assert_ne!(shred.payload(), other.payload()); } assert!(!shred.is_shred_duplicate(&other)); assert!(!other.is_shred_duplicate(shred)); } // Shreds of the same (slot, index, shred-type) with different payload // (ignoring retransmitter signature) are duplicate. for shred in &shreds { let mut other = shred.payload().clone(); other.as_mut()[90] = other[90].wrapping_add(1); let other = Shred::new_from_serialized_shred(other).unwrap(); assert_ne!(shred.payload(), other.payload()); assert_eq!( layout::get_retransmitter_signature(shred.payload()).ok(), layout::get_retransmitter_signature(other.payload()).ok() ); assert!(shred.is_shred_duplicate(&other)); assert!(other.is_shred_duplicate(shred)); } } }