//! A track is a collection of semi-reliable and semi-ordered streams, split into a [Producer] and [Subscriber] handle. //! //! A [Producer] creates streams with a key and priority. //! The key is used to determine the order of streams, while the priority is used to determine which stream to transmit first. //! This may seem counter-intuitive, but is designed for live streaming where the newest streams may be higher priority. //! A cloned [Producer] can be used to create streams in parallel, but will error if a duplicate key is used. //! //! A [Subscriber] may not receive all streams in order or at all. //! These streams are meant to be transmitted over congested networks and the key to MoQ Transport is to not block on them. //! Streams will be cached for a potentially limited duration added to the unreliable nature. //! A [Consumer] is a cheap, cloneable handle; subscribing it multiple times fans the same //! cached streams out to each independent [Subscriber]. //! //! The track is closed with [Error] when all writers or readers are dropped. use crate::{Error, Result, StreamError, Timescale, Timestamp, coding}; use crate::{broadcast, cache, group, stats}; use std::sync::Mutex; use super::delivered::Delivered; use super::idle; use super::record::{Pending, Record, Report}; use crate::ledger::{self, Ledger}; pub(crate) use super::subscription::Resolved; use super::subscription::{Accounted, Containing, Cover, Holdings, Unplaced, Window, minus}; pub use super::subscription::{Anchor, Delay, Order, Position, Range, Subscription}; use rangemap::{RangeInclusiveSet, RangeMap, RangeSet}; mod demand; pub(crate) use demand::Edit; pub(super) use demand::Preferences; use demand::Subscriptions; mod fill; mod hole; mod miss; pub(super) use fill::Asking; use fill::FillState; pub use fill::{Fill, FillGroups, FillLookup}; pub(crate) use hole::Hole; #[cfg_attr(not(feature = "async"), allow(unused_imports, reason = "async-only"))] pub(crate) use fill::{GroupFill, Readers as FillReaders}; use std::{ cell::Cell, collections::{BTreeMap, BTreeSet, VecDeque, btree_map, vec_deque}, ops::{self, Bound, RangeBounds}, sync::Arc, sync::OnceLock, sync::PoisonError, sync::Weak, sync::atomic::{AtomicBool, Ordering}, task::{Poll, ready}, time::Duration, }; // The unset priority, the midpoint of the signed range. It goes out as byte 128 on moq-lite // and 127 on IETF, which flips priority (lower first): one urgency on both. const DEFAULT_PRIORITY: i8 = 0; /// Slack before the eviction order is rebuilt, so a track holding just a few groups /// doesn't rebuild on every write. const EVICT_SLACK: usize = 64; /// How many live eviction candidates one debt payment examines (Redis-style /// bounded sampling): enough to step over a few protected (recently accessed) /// groups, small enough that a write never scans a long queue. const EVICT_SCAN: usize = 4; /// One pass over the eviction order at a fixed cache time. #[derive(Clone, Copy)] pub(super) struct ExpiryScan { start: usize, // Ceiling on how many entries this pass examines. `EVICT_SCAN` from the write // path, the whole queue from the pool's cleanup pass. width: usize, now: u64, max_ticks: u64, // Cleanup dates pending activity; write-driven scans only check dated entries. gc: bool, } /// How long a track nobody reads stays, with its cache, before it is let go. /// /// Within the window a returning viewer, or the next of a run of back-to-back /// fetches, finds it: a front its verdicts, a session's copy the groups it cached, /// without another round trip upstream. Sized above the fetch cadence of a segmented /// consumer: HLS polls every `TARGETDURATION` seconds, commonly 6 or 10. A lingering /// track holds no upstream subscription, so waiting longer costs cached state, not a /// viewer. pub(crate) const IDLE_LINGER: Duration = Duration::from_secs(30); /// Publisher-side properties of a track. /// /// These are fixed by the publisher when the track is created and don't change /// while the track is alive. A subscriber learns them via /// [`broadcast::Consumer::track`](broadcast::Consumer::track), /// which returns the publisher's [`Info`] once the subscription is accepted. // // Deliberately not `Copy`, even though it's now a plain value: adding `Copy` turns // every existing `info.clone()` in a consumer's code into a `clippy::clone_on_copy` // error under `-D warnings`. #[derive(Clone, Debug)] #[non_exhaustive] pub struct Info { /// Units per second for per-frame timestamps on this track, or `None` for an untimed /// track. /// /// A track is all timed or all untimed: every frame on a timed track /// carries a timestamp, none on an untimed one does, and a write that doesn't match /// is refused with [`Error::TimestampMismatch`]. Defaults to `None` (untimed). On /// Lite05+ it is reported in TRACK_INFO and the publisher zigzag-delta encodes /// per-frame timestamps at this scale on the wire; on moq-transport it is the /// TIMESCALE Track Property. A track received without one (pre-Lite05 moq-lite, or /// moq-transport without TIMESCALE) is untimed. pub timescale: Option, /// How long a superseded group stays fresh. The newest group is never stale. /// /// A group is stale once either clock reaches the budget: the wall-clock time since /// its successor arrived at this hop, or how far its successor's first timestamp /// trails the live edge (see [`Delay::max`]). An untimed track is judged /// by the wall clock alone, and a peer whose timestamps stop advancing cannot pin a /// superseded group. /// /// A freshness bound on what may still be served, the inverse of an HTTP /// `Cache-Control: max-age`. [`Delay::max`] is clamped to this, since a /// group can't be waited for once it is stale. Reported in TRACK_INFO so /// relays re-serve with the same bound. /// /// An authored track defaults to `Some(Duration::ZERO)`: a superseded group is stale at once, /// so only the live edge is served, and a publisher that wants a reader to reach history sets a /// longer age, or `None`, explicitly. `None` sets no publisher limit: requested with /// `with_max_age(None)`, and kept by a track received without one (an older moq-lite, or /// moq-transport without MAX_CACHE_DURATION). A hop's cache still bounds what it keeps with /// its own pool and ceiling. Stale groups are skipped by readers; reclaiming their bytes is /// the cache pool's [`expiry`](crate::cache::Pool::expiry) window. /// /// This is the `Publisher Max Age` on the wire, the publisher-side half of the /// budget [`Delay::max`] sets for a subscriber. /// /// Encoded as milliseconds in a QUIC varint, so a duration of `2^62` milliseconds /// or more cannot be put on lite-07 wires. moq-transport egress never sends it, since /// its `MAX_CACHE_DURATION` would also expire the newest group; a received one still sets it. Lite05/06 map values at or above /// `2^53 - 1` milliseconds to no limit for older JavaScript readers. Sub-millisecond precision is truncated /// (`Duration::as_millis`) at encode time. pub max_age: Option, /// The publisher's priority for this track, higher first, defaulting to 0. /// /// It breaks ties between subscriptions of equal subscriber priority. Reported in /// TRACK_INFO (Lite05+). pub priority: i8, /// How frames are compressed per group, so later frames compress against earlier ones in /// the same group. Defaults to [`Compression::None`]. /// /// Writers and readers see plain frames either way: the producer compresses each /// frame as it is written and the cache stores the compressed form. A moq-lite-07 peer or a /// moq-transport peer negotiating the Flate extension receives the compressed /// bytes and the codec. Other sessions receive one shared inflated copy per group, /// unless a sealed compression declaration makes that fallback unsupported. /// Worth it for small, repetitive payloads like JSON; wasted on media. pub compression: Compression, /// When a reader asks for a new group: on a live join, on a [`Subscriber::refresh`], or /// both. Defaults to never. See [`Refresh`]. /// /// On moq-transport, a track that refreshes at all is advertised as DYNAMIC_GROUPS, which /// arrives as a [`Refresh::ratio`] of `Some(0)`. pub refresh: Refresh, /// The draft a moq-transport publisher spoke and the Track Properties moq-net does not /// interpret, forwarded unchanged. `None` for a track published over moq-lite, which /// therefore carries no moq-transport state. /// /// Relay-only: moq-lite carries them only between relays, with the `ietf` extension. /// Shared, since every group holds a copy of the info. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) properties: Option>, /// Whether this copy's groups skip the rewind check: set on a lite-05/06 copy, whose /// untimed origin stamps send times, so an older group fetched later looks like a rewind. /// A track with moq-transport provenance ([`Self::ietf`]) skips it too. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) unchecked: bool, } // `unchecked` says how this copy arrived, not what it holds, so copies of one track over // different versions still match. impl PartialEq for Info { fn eq(&self, other: &Self) -> bool { let Self { timescale, max_age, priority, compression, refresh, properties, unchecked: _, } = self; *timescale == other.timescale && *max_age == other.max_age && *priority == other.priority && *compression == other.compression && *refresh == other.refresh && *properties == other.properties } } impl Eq for Info {} /// What asks a track for a new group; see [`Refresh`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Ask { /// A reader joined at the live edge, judged by the ratio. Join, /// A reader called [`Subscriber::refresh`], or a peer asked for a group past the largest. Refresh, /// A reader of the newest group reached a frame its copy shed, so it cannot decode on. Shed, } /// When a reader of a track asks its publisher for a new group, which TRACK_INFO carries. /// /// A track opting in starts each group with an independent frame (a snapshot or a keyframe) /// that the frames after it build on, so a new group is something the publisher can make on /// request. A request asks for a group starting past the newest one; every hop judges it /// against its own cache, which holds the same group, and requests made while one is pending /// coalesce into it. The reader still receives the cached group while the new one is pending. /// /// A live join asks when the newest group outweighs [`ratio`](Self::ratio) (any join /// without one). [`Subscriber::refresh`] asks regardless of the ratio on a track with an /// [`interval`](Self::interval), and like a join otherwise. Either request waits out the /// interval: it takes effect only once the newest group arrived at least that long ago, so /// the interval caps how often the publisher makes groups however many readers ask. A /// track with no group asks for its first at once. The default asks for nothing. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[non_exhaustive] pub struct Refresh { /// The ratio of a group's bytes after its first frame to the first frame past which a /// live join asks for a new group: once the deltas outweigh `ratio` times the first /// frame, a fresh group is cheaper for the joiner. `Some(0)` asks on any join once a delta /// follows the first frame. pub ratio: Option, /// How long after the newest group arrived a request may ask for another, measured on /// each hop's own clock from when the group reached it (or, at the publisher, was /// created), as expiry measures it. A request made sooner waits until then. /// /// Encoded as milliseconds like [`Info::max_age`]; sub-millisecond precision is /// truncated. pub interval: Option, } impl Refresh { /// Set the [`ratio`](Self::ratio), returning `self` for chaining. pub fn with_ratio(mut self, ratio: impl Into>) -> Self { self.ratio = ratio.into(); self } /// Set the [`interval`](Self::interval), returning `self` for chaining. pub fn with_interval(mut self, interval: impl Into>) -> Self { self.interval = interval.into(); self } /// Whether any request may ask for a new group. pub(crate) fn enabled(&self) -> bool { self.ratio.is_some() || self.interval.is_some() } } /// How a track's frames are compressed, with the values TRACK_INFO's `Compression` field /// carries. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Compression { /// Frames travel as written. #[default] None = 0, /// Each group is one DEFLATE stream (draft-lcurley-moq-flate). Deflate = 1, } /// An authored track's defaults: untimed, a zero [`max_age`](Info::max_age) (the live edge only), /// priority 0, uncompressed, never asking for a refresh. impl Default for Info { fn default() -> Self { Self { timescale: None, max_age: Some(Duration::ZERO), priority: DEFAULT_PRIORITY, compression: Compression::None, refresh: Refresh::default(), properties: None, unchecked: false, } } } impl Info { /// Set the per-frame timestamp scale, or `None` for an untimed track, returning `self` /// for chaining. /// /// Defaults to `None` (untimed). On Lite05+ this scale is reported in TRACK_INFO /// and used to encode per-frame timestamps on the wire. pub fn with_timescale(mut self, timescale: impl Into>) -> Self { self.timescale = timescale.into(); self } /// Set how long a superseded group stays fresh, or `None` for no publisher limit, /// returning `self` for chaining. See [`max_age`](Self::max_age). pub fn with_max_age(mut self, max_age: impl Into>) -> Self { self.max_age = max_age.into(); self } /// Set the publisher's priority, returning `self` for chaining. pub fn with_priority(mut self, priority: i8) -> Self { self.priority = priority; self } /// Set how frames are compressed per group, returning `self` for chaining. pub fn with_compression(mut self, compression: Compression) -> Self { self.compression = compression; self } /// Whether frames are compressed at all. pub(crate) fn compressed(&self) -> bool { self.compression != Compression::None } /// Set when a reader asks for a new group, returning `self` for chaining. pub fn with_refresh(mut self, refresh: Refresh) -> Self { self.refresh = refresh; self } /// `timestamp` at this track's [`timescale`](Self::timescale). /// /// A track is all timed or all scale-less, so a frame must match it: a scaled timestamp /// on a scale-less track, or an [unscaled](Timestamp::unscaled) one on a timed track, is /// [`Error::TimestampMismatch`], as is a timestamp the track's scale can't hold. A writer /// that changes state before its frame reaches the track, such as signing or encrypting /// it, rescales first so a refused timestamp changes nothing. pub fn rescale(&self, timestamp: Timestamp) -> Result { match (timestamp.scale(), self.timescale) { (Some(_), Some(timescale)) => timestamp.convert(timescale).map_err(|_| Error::TimestampMismatch), (None, None) => Ok(timestamp), _ => Err(Error::TimestampMismatch), } } /// [`Self::rescale`] for a frame of group `key`: a scale-less track's frame carries its /// group's key, so any other is [`Error::TimestampMismatch`]. A writer that changes state /// before its frame reaches the group checks this first, as with [`Self::rescale`]. pub fn rescale_in(&self, timestamp: Timestamp, key: u64) -> Result { let timestamp = self.rescale(timestamp)?; match timestamp.scale() { None if timestamp.value() != key => Err(Error::TimestampMismatch), _ => Ok(timestamp), } } /// The original moq-transport publisher's draft, whose timestamps may rewind, or /// `None` for a native moq-lite track. pub(crate) fn ietf(&self) -> Option { self.properties.as_ref().map(|properties| properties.version) } } #[cfg(test)] impl Info { /// A millisecond track with no publisher max age, for tests that write timestamps: a test /// of staleness sets its max age explicitly. pub(crate) fn timed() -> Self { Self::default().with_timescale(Timescale::MILLI).with_max_age(None) } } /// Whether a track's cache reflects its live feed; see [`TrackState::set_idle`]. Each /// state holds what only that feed knows, so nothing an ended feed set outlives it. #[derive(Clone, Copy, Debug)] enum Feed { /// Readers take from the cache. Live { /// Where a relative start taken before the feed produced anything counts back /// from: the newest group when it went live, or else the first it produced since. /// Kept after the group is evicted. first: Option, /// The group the route answered with, which can be ahead of any group cached: /// relative starts count back from it until a newer group arrives. edge: Option, }, /// The upstream subscription ended with the copy still held, so how stale the cache /// is cannot be told until the route answers again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] Idle { /// The newest group cached when the track went idle: what the route's answer is /// judged against, whatever lands before it. newest: Option, /// The newest object the ended feed left in the cache. Snapshotted, since the next /// feed's groups can land before its answer is judged. held: Option, }, /// The route answered that its feed reaches group `from`, which the cache cannot show /// yet: until it can, the newest group cached would read as the live edge. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] Answered { from: u64 }, } impl Default for Feed { fn default() -> Self { Self::Live { first: None, edge: None, } } } impl Feed { fn is_live(&self) -> bool { matches!(self, Self::Live { .. }) } } /// How far a track got toward a live [`Producer`]; each step replaces the last. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] enum Publication { /// Nobody answered the request yet. #[default] Pending, /// A dynamic handler took the request. Its answer is the handler's call, so ending /// the broadcast leaves it pending rather than rejecting it. Claimed, /// Minted by `Request::stage` and not committed yet: the producer writes, but waiting /// subscribers resolve only at `Staged::commit`, with everything written by then. Staged, /// A live Producer was minted and committed. Published, } /// How far a track got toward its end; each step replaces the last, and only an abort /// follows a seal or a drop. #[derive(Debug, Default)] enum Outcome { /// Producing, with no end declared. #[default] Open, /// The end is declared and a producer remains: groups below it may still arrive. Ending(End), /// The end is declared and the last producer dropped: nothing more arrives. Sealed(End), /// The last producer dropped without declaring an end or aborting. Dropped { /// Where production stopped when the open groups went; see /// [`TrackState::resume_position`]. resume: Option, }, /// Aborted, or refused before anyone published it. Aborted { err: Error, /// The end declared before the abort. Readers reaching it get the clean end only if it /// had settled first ([`End::settled`]); otherwise a group below it was still open, and /// they get `err`. end: Option, /// Where production stopped when the open groups went. resume: Option, /// How long the upstream refusing the request asked to wait before asking again, kept /// so a relay refuses its own readers with the same interval rather than deriving one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] retry: Option, }, } /// A track's declared end. #[derive(Debug)] struct End { /// The exclusive end: no frame at or past it exists. at: Position, /// The error the track ended with at `at`, if it did not end cleanly: a reader reaching /// the end gets it instead of the end of stream, while everything below stays readable. error: Option, /// On a received copy, when its readers began waiting out the grace for group streams /// below the end that QUIC may deliver in any order: when the end or a start was /// declared, or a group last arrived since. Readers wait for that tail (see /// [`TrackState::tail_deadline`]) rather than end at the boundary with groups still to /// arrive. `None` on a local track, which owes no tail. tail: Option, /// Whether the end settled before a write that could reopen it or an abort, or /// `closed()` reported it: every group below it was produced and finished then, so the /// clean end stands for good. Atomic so reporting it latches through a read guard; see /// [`TrackState::latch_settled`]. settled: AtomicBool, } impl End { fn settled(&self) -> bool { self.settled.load(Ordering::Relaxed) } } impl Outcome { /// The declared end, which an abort keeps. fn end(&self) -> Option<&End> { match self { Self::Ending(end) | Self::Sealed(end) | Self::Aborted { end: Some(end), .. } => Some(end), Self::Open | Self::Dropped { .. } | Self::Aborted { end: None, .. } => None, } } fn end_mut(&mut self) -> Option<&mut End> { match self { Self::Ending(end) | Self::Sealed(end) | Self::Aborted { end: Some(end), .. } => Some(end), Self::Open | Self::Dropped { .. } | Self::Aborted { end: None, .. } => None, } } /// The abort's error. fn aborted(&self) -> Option<&Error> { match self { Self::Aborted { err, .. } => Some(err), _ => None, } } /// The abort's error, if it cut the track off before its declared end settled. fn cut_off(&self) -> Option<&Error> { match self { Self::Aborted { err, end, .. } if !end.as_ref().is_some_and(End::settled) => Some(err), _ => None, } } /// How long the upstream refusing the track asked to wait before asking again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn retry(&self) -> Option { match self { Self::Aborted { retry, .. } => *retry, _ => None, } } /// Whether nothing more can arrive: the track was sealed or aborted. fn is_final(&self) -> bool { matches!(self, Self::Sealed(_) | Self::Aborted { .. }) } /// Whether no producer remains (sealed, dropped, or aborted), so nothing protects the /// live edge any longer: the pool's idle expiry reclaims it like every other group, and a /// stale consumer cannot pin the cache. fn is_released(&self) -> bool { !matches!(self, Self::Open | Self::Ending(_)) } } /// Where a track's live feed starts, as its publisher declared it; each declaration /// replaces the last. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] enum Start { /// Nothing declared. #[default] Unknown, /// The floor a subscription asked for (`None` at the live edge), still waiting on the /// serving session to resolve where the live feed begins (a lite-06+ SUBSCRIBE_START). /// Readers that must know the resolved start (see [`Subscriber::poll_start`]) wait on it; /// everything else treats the floor as declared. Pending(Option), /// The first group the live feed serves. Declared(u64), /// The first group the live feed serves, which its publisher resolved from its /// subscriber's budget (a lite-06+ SUBSCRIBE_START): it may sit above history a reader /// can still fill. Resolved(u64), } impl Start { /// The first group the live feed serves, or the floor asked while that is pending. fn key(self) -> Option { match self { Self::Unknown => None, Self::Pending(floor) => floor, Self::Declared(start) | Self::Resolved(start) => Some(start), } } } /// Whether a track is a copy received from a remote publisher, and what its readers' /// relative and time anchors resolve against. Each step replaces the last. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] enum Anchors { /// The track's own publisher's copy: anchors resolve against its own groups. #[default] Local, /// A received copy holding only part of the track, whose declared end waits for groups /// still in flight below it (see [`End::tail`]): its readers' relative and time anchors are /// refused (see [`Holdings::drops_anchors`]). Refused, /// A received copy whose wire tells it what relative anchors count back from, and carries /// time anchors as group starts, so they resolve here as at the publisher. Wire, } impl Anchors { fn received(self) -> bool { self != Self::Local } } /// Where a received copy's publisher started a range counting back, and the latest group it /// counted from. #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct RelativeStart { start: u64, latest: u64, } #[derive(Default)] pub(crate) struct TrackState { // Ingress stats scope, inherited from a tagged [`broadcast::Producer`] or [`Request`]. // Every group written into the track meters against it, whether the publisher wrote // it live or it answers a fill. Empty (no-op) for an untagged broadcast. ingress: stats::Scope, // The publisher's properties, once known; always Some for Subscriber/Producer. // Copied by value into each group it creates. info: Option, // How far the track got toward a live Producer. A reverse fetch may install `info` // before acceptance, so the two are deliberately separate. publication: Publication, // The broadcast this track belongs to. Supplies the cache pool its groups charge // into and the local `cache_duration` ceiling. broadcast: Arc, // This track's account against the shared cache pool, shared with every group it // creates (see `cache::Track`). Holds the gross-write counter `charge_debt` drains, // and the weak link a frame write follows back here to settle its own debt. cache: Arc, // The upstream publisher's Largest Object, retained even on a fetch-only copy, with the // track's largest position once it was reported: anything received past that outdates it. object_edge: Option<((u64, u64), Position)>, // Cached groups by key: the single source of truth for what is cached. The // two orderings below hold bare keys and validate against this map, so a // removed or replaced group turns their entries into discarded-on-pop hints. // Ordered so an in-range subscriber can seek to its cursor instead of scanning // every cached group for each delivery. lookup: BTreeMap, // The keys in `lookup` the live feed holds ([`Slot::visible`]): the searches for // live content walk these, so fetched backfill between them costs nothing. visible: BTreeSet, contents: group::Contents, // Copies that left `lookup`, or never entered it, while a fetch or reader may still // hold them, so a teardown ends those still being written. Pruned once it doubles // past `detached_kept`, the count its last prune left. detached: Vec, detached_kept: usize, // The flights of each group: copies of their own, from a frame a cached copy shed, for // the readers riding them; see [`Self::board`]. Held weakly, so the last rider leaving // frees one, and pruned as they are looked up. flights: BTreeMap>, // The copies of each key still arriving that another received copy waits on, once per // [`Hold`], which a reader of that copy waits out when it fails; see [`successor`]. holds: BTreeMap>, // Publisher-produced groups in arrival order, walked by subscriptions; an entry // only resolves while its stamp matches the slot's. Fetched backfill // (`insert_group_request`) is deliberately absent: it is served by key, never // replayed to arrival-order subscribers. A fetched head of a live group is the // exception, served through that group's entry. Dead entries are dropped anywhere, // not just at the front, so the protected live edge pins none behind it. arrival: VecDeque, // Eviction order under memory pressure as (key, stamp): every cached // group except the protected latest. `pay_debt` scans victims from the front; // groups accessed more recently than the pool-wide average rotate to the back // instead of dying, decoupling eviction order from arrival order. Entries are // hints that only resolve while their stamp matches the slot's, so a re-served // key can't accumulate duplicate hints that alias its replacement. // Eviction is deliberately approximate: a bounded scan per write. evict: VecDeque<(u64, u32)>, // Outstanding eviction debt in bytes, accrued by writes while the shared pool // is over capacity (see `cache::Pool::accrue`) and paid by aborting this // track's own oldest groups. Per track, so eviction lands proportionally to // what each track writes and never touches another track's cache. debt: u64, // The position the next entry of `arrival` takes: positions only grow, so a reader's // cursor stays valid as dead entries leave from anywhere. next_arrival: usize, // The highest key successfully appended to the track, which can run ahead // of any cached group. max_key: Option, // The newest group the publisher produced, which fetched backfill never moves: where // relative starts count back from (see `latest_live`). newest_produced: Option, // The checked starts cached groups hold, by key (see [`group::Origin::Bound`]), so a // first frame finds its neighbors without walking groups not yet stamped. Filled through // the read guard a first frame holds, so it needs its own lock, never contended. bounds: Mutex>, // The key and start of the highest-key group that left the cache holding a // checked start (see [`group::Origin::Bound`]). Eviction is not a key prefix, so a // group above it is checked against it as well as its cached neighbors; one at or below // it may have lost its lower bound. One entry, so nothing grows with the cache. evicted: Option<(u64, Timestamp)>, // The key of the newest cached group: the live edge, protected from // eviction by never entering the eviction order until no producer remains. // Tracked separately from `max_key`, which a reserved key can push past // any cached group. latest_group: Option, // Incarnation counter for `Slot::stamp`. next_stamp: u32, // How far the track got toward its end: declared, sealed, dropped, or aborted, with // what each of those phases knows. Retained independently of the cache, so an evicted // last group does not lose where the track ended. outcome: Outcome, // The keys the live feed delivered, and the drops a reclaim forgot: what a pending // tail is judged by. Fetched copies never count, and eviction never touches it. A gap folds // away a subscription's grace after a later arrival opened it, as the session's tail // does, so this holds only the gaps still worth waiting for. arrivals: Ledger, // What `arrivals` bills to the pool, [`ledger::RUN_OVERHEAD`] per gap. Paying eviction // debt folds its oldest gaps early, so an unbounded grace holds no more gaps than the // pool would. arrivals_bill: cache::Extra, // The first key the live feed serves, as the publisher declared it (the wire's // SUBSCRIBE_START). Lower groups are not waited for, though one may still arrive late // or be fetched. start: Start, // Whether this is a copy received from a remote publisher, and what its readers' // relative and time anchors resolve against there. anchors: Anchors, // Whether readers ask no fills below `start`: its publisher resolved it from the // budget, but neither this copy's timestamps nor a fill can judge an older group against // a reader's budget. A widened budget reaches lower groups through the feed's re-offer. fills_from_start: bool, // Whether readers start a time anchor at the group starting there, held or not: the // wire's publisher starts one at the first group at or after it, and a copy holding part // of the track cannot tell which earlier group contains the time. starts_at_keys: bool, // Where the upstream publisher started each of a lite-07 copy's live wire ranges, by the // time it asked from: a copy holding part of the track cannot tell which group contains // a time it holds no group at, but its publisher reports where it resolved one. Replaced // whole as the ranges change, so it holds at most one entry per live range. time_starts: BTreeMap, // The newest start in `time_starts` past the time its range asked from, kept as it is // replaced: [`Holdings::unplaced`] reads it on every settle of every reader. starts_past: Option, // Where the upstream publisher started each of a lite-07 copy's live wire ranges counting // back, by how many groups each counted, with the latest group it counted from: the copy // holds only part of the track, so it can't count back over its own groups. Replaced whole // as the ranges change, like `time_starts`. relative_starts: BTreeMap, // Counts changes to how a time start is placed other than by groups arriving or drops: // `time_starts`, `anchors`, and `starts_at_keys`. See [`Unplaced::mark`]. placing: u64, // Keys the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or // SUBSCRIBE_DROP, or a producer cut. Kept apart from the cache so reclaiming a slot // cannot erase a disposition a relay must forward, and as ranges so a wide one costs // one entry. drops: RangeMap, // The keys in `drops`, whatever the reason, as coalesced runs, so whether drops cover a // span is one lookup rather than a walk of every run in it. dropped: RangeSet, // What this track's own publisher produced since its last `append_group`, which skips // every key below it that never arrived and was not given up: the producer moved on. // Arrivals cannot say so, since a gap folds into them once nobody waits for it. Only // keys at or above `appended` are kept, so it holds the runs created since then. produced: RangeSet, // One past the last `append_group`: every key below it arrived, was given up, or // was skipped. appended: Option, // Bumped on every insert into `drops`, so a cursor rescans it only after a change. drops_revision: u64, // Bumped whenever a stale run leaves `drops` or another verdict replaces it, so a front's // reader rechecks the received stale runs it queued only after one may be gone. stale_revision: u64, drops_changes: ledger::Changes, // The pool tick past which each run of `drops` outlives the pool's expiry window, as of // when it was last given up, rounded up to a quarter of the window so runs given up // together stay one entry. Empty when the pool has no expiry. drops_born: RangeMap, // At most the lowest tick in `drops_born`, so an expiry scan finds nothing to forget // without walking it. drops_oldest: Option, // What `drops` and `drops_born` bill to the pool: [`RUN_OVERHEAD`] and [`BORN_OVERHEAD`] // per entry. Paying // eviction debt sheds the lowest runs, and the expiry scan forgets those older than the // pool's window; either leaves their keys unknown again. drops_bill: cache::Extra, // The runs reclaimed from `drops`, for readers that judged them to judge again. forgotten: miss::Forgotten, // How many copies the cache took in that the live feed didn't deliver: fetched backfill, // and fetched heads joining a live group, which may give it a start. Neither is in the // arrival ledger, so a time end parked on a gap looks again. fetched: u64, // Every reader's eviction inbox. Its own lock, so a reader registers through the read // guard it judges under. inboxes: Mutex, // Bumped whenever an eviction lands in an inbox, so a reader with nothing new skips // its own lock. evictions: u64, // Bumped as each group is cached, stamped on its slot: a reader judged a slot cached // only if it was cached by the count the reader last judged at. cached: u64, // Whether the cache reflects the live feed: readers get nothing from it while it does // not, since how stale it is cannot be told; fetches still do. A track is live from // creation, and only a session's copy goes idle: when its upstream subscription ends // with the copy still held, until the route's answer shows in the cache. feed: Feed, // No subscription feeds this copy: its session holds none upstream for it, so nothing // reaches it but what fills answer. Set and cleared only by the session, from its own // state, never judged from the cache. A bounded read's holes are then misses up to its // end at once, with nothing to wait for (see [`Self::nothing_arrives`]). unfed: bool, // The route's live feed starts past a gap after everything cached, so the cache // below this group is unjudgeable for readers with a finite delay budget. live_floor: Option, // Active subscriptions, in their own [`kio::Lock`] so a read-only `Consumer` // registers under that lock instead of writing back into the track state. // Kept here (rather than threaded through every handle) so any holder reaches it. subscriptions: kio::Lock, // Demand wakes a write owes, fired once the state guard is released (see // [`Self::deferred`]). A woken waker may read this track, so none fires under its lock. deferred: Vec, // The fills readers queued for groups they miss (see [`Fill`]), same reasoning: a miss // queues here and a `Requests` drains. fills: kio::Shared, // A front's logical track: read straight from its routes' copies rather than this // cache, which stays empty; see [`super::resume`]. routes: Option, } /// A publisher-produced group in [`TrackState::arrival`]. #[derive(Clone, Copy, Debug)] struct Arrived { /// Its position in arrival order, which a reader's cursor counts in. at: usize, key: u64, stamp: u32, } /// The starts a group's first frame must fall between: a timed track never holds a group /// whose start is at or before the one before it. Read under the track lock and checked /// under it, so no two groups stamped concurrently can each accept a rewind. #[derive(Clone, Copy, Debug, Default)] pub(super) struct Neighbors { floor: Option, ceiling: Option, // At or below the eviction watermark, where the floor may have been evicted. late: bool, } impl Neighbors { /// Refuse a `start` that rewinds past a neighbor's, or ties one, with /// [`Error::TimestampMismatch`], or say how the group holds it: starts strictly increase. pub(super) fn admit(&self, start: Timestamp) -> Result { if self.floor.is_some_and(|floor| start <= floor) || self.ceiling.is_some_and(|ceiling| start >= ceiling) { return Err(Error::TimestampMismatch); } Ok(if self.late { group::Origin::Unverified } else { group::Origin::Bound }) } } /// A cached group plus its bookkeeping in the track's `lookup` map. /// /// Access times and the evictable-population sample live in the group's own /// `cache::Charge`, so they share the group's lifecycle exactly: an abort from any /// handle releases the bytes and the sample together. struct Slot { group: group::Handle, // Incarnation stamp, echoed by this slot's arrival entry (if any). A re-served // key (an aborted group re-created by the publisher or re-fetched as // backfill) gets a fresh stamp, so a historical arrival entry can't resolve to // the replacement and deliver it twice or at the wrong position. stamp: u32, // [`TrackState::cached`] once this incarnation was cached. cached: u64, // Whether this incarnation came from the live publisher and can replace older // subscription content. Fetch-only backfill stays cached but never anchors drift. visible: bool, // When the group arrived here, before its first frame: the start of the wall clock for // the group before it (see [`TrackState::arrival`]). Read on the pool's clock. arrived: crate::time::Instant, // A fetched copy of a live `group` that the feed started past the frames a fetch // asked for. One that runs to the end of the group is served in the live group's // place while it lasts; `group` stays the slot, still written by the feed, // so an abandoned fetch leaves the group servable rather than gone. Boxed: rare. head: Option>, // A final drop of this group while a fetched head still filling served it: the head wins // if it finishes, and the drop is why the group is gone if it dies first. Kept here rather // than in the track's drops, so no drop sits beside a group readers are handed. dropped: Option, // A cut on an ended track that outlived its expiry: readers no longer get it, as they // no longer get the group it was. Atomic because an ended track's sweep holds only a // read guard, so it cannot reclaim the slot. retired: AtomicBool, } impl Slot { /// Whether the live feed's group still holds this slot. fn is_live(&self) -> bool { self.visible && !self.group.is_aborted() } /// Whether this slot counts as live content: [`Self::is_live`], and its first frame /// opened or it finished. A frameless group is handed to readers, but no judgment /// counts it until then: not the live edge, a successor, nor the rejoin wait. fn is_shown(&self) -> bool { self.is_live() && self.group.is_started() } /// Whether a group-only reader may treat this slot as reaching the original end. fn serves_end(&self) -> bool { let group = self.serving(); !group.is_aborted() && !group.back_cut() } /// Whether the arrival walk hands this slot out: the live feed's, and still serving, a /// fetched head counting for an aborted live group. fn is_walked(&self) -> bool { self.visible && self.serves_end() } /// Why the publisher gave up this live-feed group readers were shown, once nothing /// serves its end: a reader is handed the group to see that error, never a missing group. fn cut(&self) -> Option<&Error> { match self.visible && !self.serves_end() && !self.retired.load(Ordering::Relaxed) { true => self.verdict(), false => None, } } /// Why the publisher gave this group up, whichever copy serves it: the live copy's cut, /// or a final drop that a fetched head outlived only while it filled, once the head /// failed. A copy that finished won over the drop, even once freed, and a head the cache /// let go ages out like any group, as the slot it goes with does. fn verdict(&self) -> Option<&Error> { self.group.cut_error().or_else(|| { self.dropped.as_ref().filter(|_| { !self.copies().any(group::Handle::is_settled) && self.head.as_deref().is_some_and(|head| head.fate().failed()) }) }) } /// Whether a cursor hands this slot out: walked, or cut. fn is_read(&self) -> bool { self.is_walked() || self.cut().is_some() } /// The copy to hand readers: the fetched head while it lasts, else the live group. fn serving(&self) -> &group::Handle { match &self.head { Some(head) if !head.is_aborted() && !head.back_cut() => head, _ => &self.group, } } /// Whether the copy serving this slot finished, even one a transport bound cut short, so /// nothing it promised a reader is still in flight. A fetched head serving the group to /// its end is judged in place of the live copy beside it, which may still be written or /// may have been cut or evicted. Once no head serves, an aborted live copy holds the end /// open: a session that died mid-group delivered less than the end promised. A failed /// fetch-only copy holds nothing open, since the feed never promised it. fn is_settled(&self) -> bool { (!self.visible && self.is_aborted()) || self.serving().is_settled() } /// Every copy this slot holds, the live group first. fn copies(&self) -> impl Iterator { std::iter::once(&self.group).chain(self.head.as_deref()) } /// Whether every copy is gone, so the slot has nothing left to serve. fn is_aborted(&self) -> bool { self.copies().all(group::Handle::is_aborted) } /// The mean access over copies still held, weighed the way each one samples the /// pool's average: a maximum would sit above the mean forever, so a lone slot with /// two copies could never be evicted. fn cache_accessed(&self) -> u64 { let (sum, count) = self .copies() .filter(|group| !group.is_aborted()) .fold((0u128, 0u128), |(sum, count), group| { (sum + group.cache_accessed() as u128, count + 1) }); (sum / count.max(1)) as u64 } /// The bytes every copy holds, all freed together. fn cache_size(&self) -> u64 { let mut size = self.group.cache_size(); if let Some(head) = &self.head { size += if head.same_content(&self.group) { head.delivery_size() } else { head.cache_size() }; } size } /// Whether every copy still held sat idle past the scan's expiry window. Asks /// each one, so a cleanup scan dates them all. fn is_expired(&self, scan: &ExpiryScan) -> bool { let idle = |group: &group::Handle| { group .cache_accessed_tick(scan.gc.then_some(scan.now)) .is_some_and(|tick| scan.now.saturating_sub(tick) > scan.max_ticks) }; // No copy of a cut serves its end, so it ages as the live group it was: a reader // behind still gets it for as long as it would have got that group. let mut expired = self.cut().is_none() || idle(&self.group); // Not `all`, which stops at the first live copy and leaves the rest undated. for group in self.copies().filter(|group| !group.is_aborted()) { expired &= idle(group); } expired } /// Whether nothing is left to keep the slot for: aborted, and not a cut that ages /// like a group. fn is_spent(&self) -> bool { self.is_aborted() && self.cut().is_none() } /// Enter every copy into the evictable population, so the pool's access average /// samples whichever one readers use. fn cache_demote(&self) { for group in self.copies() { group.cache_demote(); } } /// The checked start of whichever copy began at frame 0. fn bound(&self) -> Option { self.copies().find_map(group::Handle::bound) } /// Abort every copy this slot holds. fn abort(&self, err: Error) { for group in self.copies() { let _ = group.clone().abort(err.clone()); } } } /// Heap the track keeps per cached group, excluding the group itself /// ([`group::CACHE_OVERHEAD`]). /// /// One [`Slot`] under its key in `lookup`, its key in `visible`, its start in /// `starts` once frame 0 arrives, plus a hint in each of `arrival` and `evict`. Doubled /// because the containers run half empty in the worst case: a /// `BTreeMap` node sits between half and fully packed, and a `VecDeque` holds up to /// twice the entries in it. Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is /// derived rather than measured. pub(super) const CACHE_OVERHEAD: u64 = 2 * (2 * size_of::() + size_of::() + size_of::<(u64, Timestamp)>() + size_of::() + size_of::<(u64, u32)>()) as u64; /// Heap the track keeps per run of its drop set: the run and its reason in `drops`, at most /// one coalesced run in `dropped`, and its two entries in `drops_changes`. Doubled for half-full `BTreeMap` nodes, as /// [`CACHE_OVERHEAD`] is. Billed to the pool, so a peer fragmenting its drops into runs costs /// what the cache would hold, never more. pub(super) const RUN_OVERHEAD: u64 = 2 * (size_of::<(ops::Range, Hole)>() + size_of::>() + size_of::<(u64, (u64, u64))>() + size_of::<(u64, u64)>()) as u64; /// Heap the track keeps per entry of its drop set's expiry index, `drops_born`, doubled as /// [`RUN_OVERHEAD`] is. pub(super) const BORN_OVERHEAD: u64 = 2 * size_of::<(ops::Range, u64)>() as u64; /// The groups this hop holds, for resolving relative and time anchors: ordered lookups in /// the cache, so no query walks more than the groups it answers. impl Holdings for TrackState { fn drops_anchors(&self) -> bool { self.anchors == Anchors::Refused } fn looks_up(&self) -> bool { // A received copy's handler is its session, which fetches by group alone. !self.anchors.received() && self.fills.read().has_handlers() } fn containing(&self, time: u64) -> Containing { if self.starts_at_keys { return Containing::Group(time); } if self.anchors == Anchors::Wire { return self.containing_received(time); } match self.lookup.range(..=time).next_back() { Some((&key, _)) => Containing::Group(key), // Every key at or before it is gone, so whichever group held it will never arrive. None if self.dropped_cover(0..time.saturating_add(1)) => Containing::Before, None if self.lookup.is_empty() => Containing::Unknown, None => Containing::Before, } } /// Only times [`Self::containing`] can't place, but those `time_starts` places directly. /// It leaves out the times below the newest start the publisher reported past an earlier /// range's time: `containing_received` places those once no held group precedes them, /// which an eviction brings about without changing the mark. A consistent publisher /// reports its oldest group for each such range, so the run then still starts where its /// unknown times do. fn unplaced(&self) -> Option { let none = Unplaced::NONE; let past = self.starts_past.filter(|_| self.anchors == Anchors::Wire); // The times below every held group that no inference from an earlier range places. let unheld = |run: ops::RangeInclusive| match past { Some(past) => (*run.start()).max(past)..=*run.end(), None => run, }; // The times at or after `from` whose keys from `from` through them drops do not all // cover. As in `dropped_cover`, the last time checks the keys below it only, so a run // from the last key covers nothing and leaves nothing unplaced. let uncovered = |from: u64| match self.dropped.get(&from) { _ if from == u64::MAX => none.clone(), Some(run) if run.end == u64::MAX => none.clone(), Some(run) => run.end..=u64::MAX, None => from..=u64::MAX, }; let runs = if self.starts_at_keys { [none.clone(), none] } else { match (self.lookup.first_key_value(), self.lookup.last_key_value()) { // Before any group, only a time whose keys down to 0 were all dropped is placed. (None, _) | (_, None) => [unheld(uncovered(0)), none], (Some(_), Some(_)) if self.anchors != Anchors::Wire => [none.clone(), none], // A copy places a time below its groups only once the keys down to 0 were // dropped, and one above them only once the keys down to the newest were. (Some((&first, _)), Some((&last, _))) => { let below = uncovered(0); let below = *below.start()..=(*below.end()).min(first.saturating_sub(1)); let below = match first { 0 => none.clone(), _ => unheld(below), }; let above = match last.checked_add(1) { Some(next) => uncovered(next), None => none, }; [below, above] } } }; Some(Unplaced { runs, mark: self.placing, }) } fn next_start(&self, from: u64) -> Option { self.lookup.range(from..).next().map(|(&key, _)| key) } fn back(&self, latest: u64, back: u64) -> Option { let Some(back) = back.checked_sub(1) else { return Some(latest); }; // Bounded by the cache: a huge count walks it once and reaches back to the oldest. let back = usize::try_from(back).ok()?; self.lookup.range(..latest).rev().nth(back).map(|(&key, _)| key) } /// A received copy starts where its publisher did, counting from the same latest group: a /// newer one moved the live edge, which the copy then counts from itself. fn reported_start(&self, latest: u64, back: u64) -> Option { self.relative_starts .get(&back) .filter(|reported| reported.latest == latest) .map(|reported| reported.start) } fn complete(&self) -> bool { self.outcome.is_final() } /// In media time, from the newest frame held, or the newest group still served when it /// starts later, never a group given up; on a scale-less track, by arrival on this hop's clock, as expiry measures it. /// /// Always a group this hop holds: the newest one starting at or before the target, or the /// oldest one when none does, so a copy never asks its upstream for the prefix it lacks. /// Groups keyed by their start are found by key; on a copy whose wire keys groups /// otherwise, by when each one's first frame presented. Pending until a group is held. fn reach_back(&self, _latest: u64, min: Duration) -> Option { let oldest = || self.walked(..).next().map(|(&key, _)| Anchor::Time(key)); let Some(scale) = self.info.as_ref()?.timescale else { let key = self.walked(..).rev().find(|(_, slot)| self.aged(slot.arrived, min)); return key.map(|(&key, _)| Anchor::Time(key)).or_else(oldest); }; let newest = self .live_edge(&Servable::ALL) .and_then(|edge| edge.timestamp.convert(scale).ok()) .map(Timestamp::value); // Rounded up to whole ticks, so the start is never newer than the Min Age allows; one // past every timestamp reaches back to the oldest group. let back = u64::try_from( min.as_nanos() .saturating_mul(u128::from(scale.as_u64())) .div_ceil(1_000_000_000), ) .unwrap_or(u64::MAX); if self.anchors != Anchors::Refused && !self.starts_at_keys { let held = self.walked(..).next_back().map(|(&key, _)| key); let edge = newest.max(held)?; let target = edge.saturating_sub(back); let key = self.walked(..=target).next_back(); return key.map(|(&key, _)| Anchor::Time(key)).or_else(oldest); } let target = newest?.saturating_sub(back); let started = |slot: &Slot| { self.started_at(slot) .and_then(|start| start.convert(scale).ok()) .is_some_and(|start| start.value() <= target) }; let key = self.walked(..).rev().find(|(_, slot)| started(slot)); key.map(|(&key, _)| Anchor::Time(key)).or_else(oldest) } fn never_produced(&self, keys: ops::Range) -> bool { // Skip runs coalesce, so a gap between two held groups is one run unless the cache // evicted a group inside it. keys.is_empty() || self .drops .get_key_value(&keys.start) .is_some_and(|(run, hole)| *hole == Hole::Skip && run.end >= keys.end) } } impl TrackState { /// How many keys just below `key` the publisher declared it never produced: the skip /// run that ends at `key`, as a moq-transport Prior Group ID Gap names it. fn skipped_below(&self, key: u64) -> u64 { let Some(below) = key.checked_sub(1) else { return 0; }; match self.drops.get_key_value(&below) { Some((run, Hole::Skip)) => key - run.start, _ => 0, } } } impl TrackState { /// [`Holdings::containing`] on a lite-07 copy, which holds only part of the track: a group /// held before `time` contains it only once nothing between them is still to arrive, so /// until then it waits for the group starting there, the gap's holes, a group past it, /// or where the publisher reports starting the range it asked from `time`. A group past /// it says the feed went by: one between still in flight, a stream overtaken, would /// only add that group to the reader's range. /// /// With no group held at or before `time`, a range asked from an earlier time, or from /// the oldest group (reported as time 0), that the publisher started past `time` says /// the same: no group starts at or before it. A reader joining a copy that another /// reader already asks from further back adds no range of its own, so that is the only /// report its time start gets. A group the copy still holds there is a group the /// publisher has since expired, so it wins: the time still waits to learn whether it /// contains it. fn containing_received(&self, time: u64) -> Containing { if self.lookup.contains_key(&time) { return Containing::Group(time); } match self.time_starts.get(&time) { Some(&start) if start <= time => return Containing::Group(start), // The publisher serves nothing at or before it, as it does for a time older than // every group it holds. Some(_) => return Containing::Before, None => {} } match self.lookup.range(..=time).next_back() { Some((&key, _)) if self.dropped_cover(key.saturating_add(1)..time.saturating_add(1)) || self.lookup.range(time..).next().is_some() => { Containing::Group(key) } None if self.dropped_cover(0..time.saturating_add(1)) || (self.starts_past.is_some_and(|past| past > time) && self.time_starts.range(..time).any(|(_, &start)| start > time)) => { Containing::Before } _ => Containing::Unknown, } } /// Whether drops, for whatever reason, cover every key in `keys`. An empty run is covered. fn dropped_cover(&self, keys: ops::Range) -> bool { keys.is_empty() || self.dropped.get(&keys.start).is_some_and(|run| run.end >= keys.end) } /// The upstream publisher's Largest Object, unless a frame or group received since its /// report outdates it, even one the cache has since evicted. fn object_edge(&self) -> Option<(u64, u64)> { let (edge, mark) = self.object_edge?; self.cache .largest() .is_none_or(|largest| largest <= mark) .then_some(edge) } /// Ask for a group past every key written, judged by [`Info::refresh`]: a live join /// only when the newest group outweighs the ratio, and either request only once the /// newest group is at least the interval old. A request made sooner is deferred until /// then, and one made while a request is pending coalesces into it. /// /// Takes the read guard to release it before locking the demand, so the two locks /// never nest. pub(crate) fn ask(state: kio::Ref<'_, TrackState>, ask: Ask) { let Some((at, subscriptions)) = Self::admit(state, ask) else { return; }; let mut wakes = subscriptions.lock().refresh(at); wakes.wake(); } /// [`Self::ask`] for a [`Subscriber::refresh`] caller, who waits for the request to be /// sent and withdraws it by dropping the [`Refreshing`]. fn request(state: &kio::ConsumerWeak) -> Refreshing { let Some((at, subscriptions)) = Self::admit(state.read(), Ask::Refresh) else { return Refreshing::done(); }; let (requested, mut wakes) = subscriptions.lock().request(at); wakes.wake(); match requested { demand::Requested::Done => Refreshing::done(), demand::Requested::Waiting(epoch) => Refreshing::waiting(Ticket { state: state.clone(), subscriptions, epoch, }), } } /// When a request as `ask` takes effect, if the track's [`Refresh`] lets it ask at all. fn admit(state: kio::Ref<'_, TrackState>, ask: Ask) -> Option<(crate::time::Instant, kio::Lock)> { let refresh = state.info.as_ref().map(|info| info.refresh).filter(Refresh::enabled)?; let latest = state.latest_group?; let slot = state.lookup.get(&latest); let ratio = match ask { Ask::Join => refresh.ratio, // The interval alone judges a request on a track that declares one. Ask::Refresh => refresh.ratio.filter(|_| refresh.interval.is_none()), Ask::Shed => None, }; if let Some(ratio) = ratio && !slot.is_some_and(|slot| slot.serving().over_refresh(ratio)) { return None; } let now = state.cache.pool().now_wall(); let at = match (refresh.interval, slot) { // `None` when no instant reaches it. (Some(interval), Some(slot)) => slot.arrived.checked_add(interval)?.max(now), _ => now, }; Some((at, state.subscriptions.clone())) } /// A reader of group `key` reached a frame its copy shed: ask for a new group if it is /// the newest, regardless of the ratio. An older group's reader moves on to the groups /// after it instead. pub(super) fn shed_read(state: kio::Ref<'_, TrackState>, key: u64) { if state.latest_group == Some(key) { Self::ask(state, Ask::Shed); } } /// The starts a first frame of group `key` must fall between: the nearest checked /// starts cached on either side, since groups land out of key order, and the /// eviction watermark. A group at or below the watermark is late: its floor may be gone, /// so the watermark caps it instead. /// /// Usually one group lookup each way; only unstarted or partial neighbors are skipped. /// `None` on an [unchecked](Info::unchecked) track, whose timestamps may rewind. pub(super) fn neighbors(&self, key: u64) -> Option { if self .info .as_ref() .is_some_and(|info| info.unchecked || info.ietf().is_some()) { return None; } let bounds = self.bounds.lock().unwrap_or_else(PoisonError::into_inner); let below = bounds.range(..key).next_back().map(|(_, start)| *start); let above = bounds .range((Bound::Excluded(key), Bound::Unbounded)) .next() .map(|(_, start)| *start); drop(bounds); Some(match self.evicted { // The evicted group itself, fetched again, starts where it did. Some((evicted, _)) if key == evicted => Neighbors { floor: below, ceiling: above, late: true, }, Some((evicted, start)) if key < evicted => Neighbors { floor: below, ceiling: Some(above.map_or(start, |above| above.earlier(start))), late: true, }, Some((_, start)) => Neighbors { floor: Some(below.map_or(start, |below| below.later(start))), ceiling: above, late: false, }, None => Neighbors { floor: below, ceiling: above, late: false, }, }) } /// Group `key` wrote its first frame: share a just-checked `bound` with its neighbors. pub(super) fn first_frame(&self, key: u64, bound: Option) { if let Some(bound) = bound { self.share_bound(key, bound); } } /// Record group `key`'s just-checked `start` as a bound for its neighbors. A group /// stamped after it left the cache bounds nothing. fn share_bound(&self, key: u64, start: Timestamp) { if self.lookup.get(&key).and_then(Slot::bound) != Some(start) { return; } self.bounds .lock() .unwrap_or_else(PoisonError::into_inner) .insert(key, start); } /// Take group `key` out of the cache, raising the eviction watermark to its start. fn uncache(&mut self, key: u64) -> Option { let slot = self.remove_slot(key)?; if let Some(start) = slot.bound() && self.evicted.is_none_or(|(evicted, _)| key > evicted) { self.evicted = Some((key, start)); } Some(slot) } /// [`Self::uncache`] a servable group to free the cache and abort it with `err`, recording /// it for the readers that judged it cached to judge again. fn evict_group(&mut self, key: u64, err: Error) { let slot = self.uncache(key).expect("a cached group"); slot.abort(err); let landed = self .inboxes .get_mut() .unwrap_or_else(PoisonError::into_inner) .evicted(key, slot.cached); if landed { self.evictions += 1; } } /// [`Self::uncache`] a group that another copy displaces rather than one that ends, so a /// teardown still ends the copies its readers hold. fn displace(&mut self, key: u64) { if let Some(slot) = self.uncache(key) { for group in slot.copies() { self.detach(group); } } } /// Keep `group` for a teardown to end, though the cache does not hold it. fn detach(&mut self, group: &group::Handle) { self.detached.push(group.detach()); if self.detached.len() > 2 * self.detached_kept { self.detached.retain(|copy| copy.writing().is_some()); self.detached_kept = self.detached.len(); } } /// Record keys the upstream publisher gave up, with why. A skip is final and the /// most authoritative verdict, so a drop never replaces one; see [`Self::skip_groups`]. /// Returns the runs recorded. fn drop_groups(&mut self, groups: ops::Range, hole: Hole) -> RangeSet { let mut runs = RangeSet::new(); if groups.is_empty() { return runs; } runs.insert(groups.clone()); if hole != Hole::Skip { for (run, given) in self.drops.overlapping(&groups) { if *given == Hole::Skip { runs.remove(run.clone()); } } } self.cache.wakes().arrived(groups); for run in runs.iter() { self.change_drops(run.clone(), Some(hole.clone())); } runs } /// Whether the publisher declared `key` skipped: never produced, which is final. fn skipped(&self, key: u64) -> bool { self.drops.get(&key).is_some_and(|hole| *hole == Hole::Skip) } /// The exclusive end of the run at `key` that no group will fill: one the publisher /// declared skipped, or with `dropped`, also one given up for good. A stale drop is not: /// a wider budget takes it back. fn hole_end(&self, key: u64, dropped: bool) -> Option { let (run, hole) = self.drops.get_key_value(&key)?; let gone = match hole { Hole::Skip => true, Hole::Drop { stale, .. } => dropped && stale.is_none(), }; gone.then_some(run.end) } /// The start of the group before `start`, as a lite-07 GROUP's Previous Start names it: /// just below the run of starts the publisher declared skipped directly under `start`, or /// the group just below it, and `None` when neither says. A run skipped down to 0 has no /// group below it. See [`Subscriber::previous_start`]. fn previous_start(&self, start: u64) -> Option { let below = start.checked_sub(1)?; match self.drops.get_key_value(&below) { Some((run, hole)) if *hole == Hole::Skip => run.start.checked_sub(1), _ => self.lookup.contains_key(&below).then_some(below), } } /// Whether the live feed's copy of the group at `key` is cached and still serves it. fn holds_group(&self, key: u64) -> bool { self.lookup .get(&key) .is_some_and(|slot| slot.visible && slot.serves_end()) } /// Skip what of `groups` holds no cached group, replacing any drop of it, final or stale: /// "never produced" is the most authoritative verdict on a key. A reader a drop /// already ended stays ended, and a relay forwards the skip over the drop it reported. /// A group that arrived is never overwritten, since it was produced after all. Returns /// the cached keys left alone. See [`Producer::skip_groups`]. fn skip_groups(&mut self, groups: ops::Range) -> RangeSet { let mut cached = RangeSet::new(); if groups.is_empty() { return cached; } let mut left = RangeSet::new(); left.insert(groups.clone()); for (key, _) in self.lookup.range(groups.clone()) { if let Some(end) = key.checked_add(1) { left.remove(*key..end); cached.insert(*key..end); } } // Already skipped: nothing changes, so no reader rescans. for (run, hole) in self.drops.overlapping(&groups) { if *hole == Hole::Skip { left.remove(run.clone()); } } for run in left.iter().cloned().collect::>() { self.drop_groups(run, Hole::Skip); } cached } /// Settle an append at `key`: skip every key below it that this track's own /// publisher never produced, from just past its previous append (before the first, from /// the lowest it produced), since appending says it will not go back. The skip replaces a /// drop given there, as [`Self::skip_groups`] does. fn appended(&mut self, key: u64) { if let Some(from) = self.appended.or(self.produced.iter().next().map(|run| run.start)) && from < key { let mut left = RangeSet::new(); left.insert(from..key); for run in self.produced.iter() { left.remove(run.clone()); } for run in left.iter().cloned().collect::>() { self.skip_groups(run); } } self.produced.clear(); self.appended = key.checked_add(1); } /// Forget every run held as stale; see [`Producer::supersede_stale`]. fn supersede_stale(&mut self) -> RangeSet { let stale: RangeSet = self .drops .iter() .filter(|(_, hole)| hole.stale_budget().is_some()) .map(|(run, _)| run.clone()) .collect(); for run in stale.iter() { self.change_drops(run.clone(), None); } stale } /// Keep the change index aligned with the range map, including the neighbours an /// insertion coalesces and the two pieces a removal can leave. fn change_drops(&mut self, groups: ops::Range, hole: Option) { if hole.as_ref().is_none_or(|hole| hole.stale_budget().is_none()) && self .drops .overlapping(&groups) .any(|(_, held)| held.stale_budget().is_some()) { self.stale_revision += 1; } let mut affected = groups.start.saturating_sub(1)..groups.end.saturating_add(1); let old: Vec<_> = self.drops.overlapping(&affected).map(|(run, _)| run.clone()).collect(); for run in &old { self.drops_changes.remove(run.start); affected.start = affected.start.min(run.start); affected.end = affected.end.max(run.end); } // Whole-map lengths: a coalesced run can span many expiry entries, which a count over // the affected span would scan on every change. let entries = |state: &Self| state.drops.len() as u64 * RUN_OVERHEAD + state.drops_born.len() as u64 * BORN_OVERHEAD; let before = entries(self); match hole { Some(hole) => { self.drops.insert(groups.clone(), hole); self.dropped.insert(groups.clone()); let max_ticks = self.cache.pool().expiry_ticks(); if max_ticks != u64::MAX { // Rounded up to a quarter of the window, so a run is forgotten no sooner // than the window after it was given up. let quarter = (max_ticks / 4).max(1); let now = self.cache.pool().now(); let expires = (now / quarter) .saturating_add(1) .saturating_mul(quarter) .saturating_add(max_ticks); self.drops_born.insert(groups, expires); self.drops_oldest = Some(self.drops_oldest.map_or(expires, |oldest| oldest.min(expires))); } } None => { self.drops.remove(groups.clone()); self.dropped.remove(groups.clone()); self.drops_born.remove(groups); } } for (run, _) in self.drops.overlapping(&affected) { self.drops_changes.insert(run.clone()); } let after = entries(self); if after > before { self.drops_bill.add(after - before); } else { self.drops_bill.sub(before - after); } self.drops_revision += 1; } /// Whether a run of the drop set may have outlived the pool's expiry window by `now`. fn drops_expired(&self, now: u64) -> bool { self.drops_oldest.is_some_and(|expires| now >= expires) } /// Forget the runs of the drop set given up longer than the pool's expiry window before /// `now`: their keys are unknown again, as a shed run's are. Walks the set only once /// its oldest run is due, which a quarter of the window separates from the next. fn expire_drops(&mut self, now: u64) { if !self.drops_expired(now) { return; } let kept = self.kept_drops(); let expired: Vec<_> = self .drops_born .iter() .filter(|(run, expires)| now >= **expires && run.start < kept) .map(|(run, _)| run.start..run.end.min(kept)) .collect(); for run in expired { self.forget_drops(run); } self.drops_oldest = self .drops_born .iter() .filter(|(run, _)| run.start < kept) .map(|(_, expires)| *expires) .min(); } /// Where the runs a reclaim may forget end: every run on a received copy, whose tail /// decides its end, but only those below a local track's edge, since one above it is what /// lets its declared end be reached (see [`Self::reaches`]). A local producer gives up only /// what it chose to. A closed track keeps them until its last handle frees the state. fn kept_drops(&self) -> u64 { match self.anchors.received() { true => u64::MAX, false => self.edge(), } } /// The runs from `from` up to [`Self::kept_drops`] were kept and are now reclaimable: date /// the oldest of them, which the expiry scan skipped, so the next scan finds it. fn rearm_drops(&mut self, from: u64) { let due = self .drops_born .overlapping(&(from..self.kept_drops())) .map(|(_, expires)| *expires) .min(); if let Some(due) = due { self.drops_oldest = Some(self.drops_oldest.map_or(due, |oldest| oldest.min(due))); } } /// Forget the lowest runs of the drop set until the debt is paid or `cap` bytes are /// freed, returning the bytes freed. Their keys are unknown again: a reader judges /// them by its budget, or fills them by range. fn shed_drops(&mut self, cap: u64) -> u64 { let mut paid = 0u64; let kept = self.kept_drops(); while self.debt > 0 && paid < cap { let Some(run) = self .drops .first_range_value() .map(|(run, _)| run.start..run.end.min(kept)) .filter(|run| !run.is_empty()) else { break; }; let billed = self.drops_bill.size(); self.forget_drops(run); let freed = billed.saturating_sub(self.drops_bill.size()); self.debt -= freed.min(self.debt); paid = paid.saturating_add(freed); } paid } /// Forget `run` of the drop set to reclaim it, accounting it in the arrivals as an evicted /// group stays: the publisher accounted for it, so neither the tail nor a reader waits on /// it again, and a reader that wants it asks a fill or judges it by its budget. /// /// A reclaim never adds a run to the arrivals, so forgetting a flood of fragments costs /// nothing there: a run touching no accounted one takes the gap below it too, given up /// early as a fold would. Nothing below the tail's floor is accounted, since nothing waits /// on it and accounting it would lower the floor onto gaps nobody waited for. fn forget_drops(&mut self, run: ops::Range) { self.change_drops(run.clone(), None); self.forgotten.record(run.clone()); let floor = self.tail_floor(); let start = run.start.max(floor); if start >= run.end { return; } // The highest accounted run starting at or below the run's end touches it when it ends // at or past its start; otherwise it is the run below. let start = match self.arrivals.end_below(run.end) { Some(end) if end >= start => start, below => below.map_or(floor, |end| end.max(floor)), }; let now = self.cache.pool().now_wall(); // Only a first run takes this as the age of the gap below it, which nothing opened yet: // the tail's, so an end already reached stays reached. let since = self .outcome .end() .and_then(|end| end.tail) .map_or(now, |tail| tail.min(now)); self.arrivals.account(start..run.end, since); self.bill_arrivals(); self.cache.wakes().arrived(start..run.end); } /// Remove an aborted slot, keeping a cut as a drop: reclaiming it must not erase a /// disposition a relay still has to forward. fn reclaim_aborted(&mut self, key: u64) { if let Some(slot) = self.uncache(key) && slot.visible && let Some(err) = slot.verdict() && let Some(end) = key.checked_add(1) { self.drop_groups(key..end, Hole::drop(StreamError::from(err))); } } /// Why `key` was given up, if it was: cut while cached, or dropped. fn given_up(&self, key: u64) -> Option { if let Some(err) = self.lookup.get(&key).and_then(Slot::cut) { return Some(err.clone()); } self.drops.get(&key).map(|hole| Error::Stream(hole.error())) } /// Whether a Producer was minted, committed or not. fn minted(&self) -> bool { matches!(self.publication, Publication::Staged | Publication::Published) } /// A dynamic handler took the request. fn claim(&mut self) { self.publication = Publication::Claimed; } /// Mint the Producer, holding waiting subscribers until [`Self::commit`]. fn stage(&mut self, info: Info) { self.publication = Publication::Staged; self.install(info); } /// Resolve waiting subscribers with what the minted Producer wrote. fn commit(&mut self) { debug_assert_eq!(self.publication, Publication::Staged, "only a staged track commits"); self.publication = Publication::Published; } /// Mint the Producer and resolve waiting subscribers at once. fn publish(&mut self, info: Info) { self.publication = Publication::Published; self.install(info); } fn poll_info(&self) -> Poll> { if let Some(info) = self.info.as_ref().filter(|_| self.publication != Publication::Staged) { Poll::Ready(Ok(info.clone())) } else if let Some(err) = self.outcome.aborted() { // Aborted before anyone served it, so the info can never arrive: fail the // waiting subscribes instead of parking them on a track nobody will fill. Poll::Ready(Err(err.clone())) } else { Poll::Pending } } /// Whether readers may take from the cache: it reflects the live feed, or the track /// reached its end, which makes the cache the whole of it. An end declared but not /// reached does not: the groups before it are still to come from the feed. fn readable(&self) -> bool { self.feed.is_live() || self.outcome.is_final() || self.is_complete() } /// Where relative starts count back from: the live feed's newest group, produced or /// answered. `None` while the cache does not reflect the feed, since an idle copy's /// newest group is only what an ended feed left: a start taken then waits, as on an /// empty track. An idle copy that ends counts from its newest cached group. fn latest_live(&self) -> Option { let edge = match self.feed { Feed::Live { edge, .. } => edge, _ => None, }; let floor = self.live_floor.unwrap_or(0); self.newest_produced .max(edge) .filter(|&latest| self.readable() && latest >= floor) } /// Where a relative start that waited on [`Self::latest_live`] counts back from. An idle /// copy that ended without going live counts from its newest group, which its cache now /// holds the whole of, and not from where an earlier feed began. fn first_live(&self) -> Option { match self.feed { Feed::Live { first, .. } => first, _ => self.latest_live(), } } /// The cache reflects the live feed again, which reaches `edge` if the route named it: /// a relative start that waited counts back from its newest group now. fn go_live(&mut self, edge: Option) { self.feed = Feed::Live { first: None, edge }; self.feed = Feed::Live { first: self.latest_live(), edge, }; } /// Find the next live group at or after `index` in arrival order, for a reader reading /// `window`. /// /// Returns the group and its absolute index so the consumer can advance past it. /// The cached producer rather than a consumer: `consume` reads the clock and can /// take the group's own lock, which the caller does once the track guard is gone. /// /// Moves `index` past the leading entries this reader is never handed: dead, or below /// `min_key`, which a lower floor re-offers from the cache instead. So a wake walks /// only what the reader may still be handed, however many stale entries arrived. fn poll_recv_group( &self, index: &Cell, min_key: u64, window: &Window, ) -> Poll>> { if self.serves_bounded(window) { let mut passed = true; for entry in self.arrivals_from(index.get()) { let slot = self.lookup.get(&entry.key).filter(|slot| slot.stamp == entry.stamp); let Some(slot) = slot.filter(|_| entry.key >= min_key) else { if passed { index.set(entry.at + 1); } continue; }; // A cut group is handed out too, so its reader sees why it ended. if slot.is_read() { return Poll::Ready(Ok(Some((slot.serving().clone(), entry.at)))); } passed = false; } } // TODO once we have drop notifications, check if index == final_key. if self.is_complete() { Poll::Ready(Ok(None)) } else if let Some(err) = self.outcome.aborted() { Poll::Ready(Err(err.clone())) } else { Poll::Pending } } /// Find the smallest-key cached group in `window` satisfying /// `next_key <= seq < end_key (if set)`. Used by /// [`Subscriber::next_group`] so the range can be widened (or unset) /// after the fact and previously-skipped cached groups become available /// without scanning past them in arrival order. /// /// Returns `Poll::Pending` when no in-range group is currently cached but /// future groups could still arrive in range; returns `Ok(None)` only when /// the track is finalized and no further in-range group is possible. fn poll_next_in_range( &self, next_key: u64, end_key: Option, window: &Window, ) -> Poll>> { // Nothing more can arrive once the track was sealed (dropped with its end // declared) or aborted. Either closed the channel, so parking is not an option: // the consumer would be handed the abort, or `Dropped`, instead of how it ended. let closed = self.outcome.is_final(); // Once nothing more is in range: an abort before the end settled cut the track // off. One after it is a clean end (see `is_complete`), as is reaching the end. let end = || match self.outcome.cut_off() { Some(err) => Err(err.clone()), None => Ok(None), }; // If the exclusive end is already at or below where we'd resume, no // group can ever satisfy this call until the cap rises or the ranges widen. // Pending (not None) so the consumer is parked rather than told the stream // is over, unless the track declared its end: a finished track ends a reader // with nothing left in range, as the arrival path does. An empty range // (`end == 0`) parks even at the first key. let cap = match (end_key, window.end()) { (Some(cap), Some(end)) => Some(cap.min(end)), (cap, end) => cap.or(end), }; if let Some(cap) = cap && cap <= next_key { if closed || self.final_key().is_some() { return Poll::Ready(end()); } return Poll::Pending; } let best = window .runs(next_key) .filter(|_| self.serves_bounded(window)) .find_map(|run| { // A cut group is handed out too, so its reader sees why it ended. self.lookup .range(run) .map(|(_, slot)| slot) .take_while(|slot| super::subscription::before_end(slot.group.start, end_key)) .find(|slot| slot.serves_end() || slot.cut().is_some()) .map(Slot::serving) }); if let Some(group) = best { // Deliberately no cache refresh here: this is a pure seek, and the caller may // convict the candidate rather than deliver it. The caller stamps what it // delivers; a group walked off must not be shielded from eviction. The caller // consumes it with the track guard released, like `poll_recv_group`. return Poll::Ready(Ok(Some(group.clone()))); } // No in-range group is cached. Decide whether more could ever arrive. // `final_key` is one past the last possible key. If our // floor is already at/past it, nothing else can land in range. // A closed track produces nothing more either, so a gap below its // boundary is the end, not a wait, as is one in a complete track, which // ends the arrival path too. Cached in-range groups were returned above, // so an aborted track drains what finished first. // This is the cursor the ordered reader uses. if closed || self.is_complete() || self.final_key().is_some_and(|fin| next_key >= fin) { return Poll::Ready(end()); } Poll::Pending } /// The cached group for `key`, but only when it holds every frame from /// `frame_start` onward. /// /// A group cached from a frame-bounded subscription starts partway in, and serving /// it to someone who asked for the whole group would silently hand back a tail. It /// is a miss instead, so the fetch goes upstream for the frames that are missing. fn covering_group(&self, key: u64, frame_start: u64) -> Option<&group::Handle> { self.covering_range( key, group::Bounds { start: frame_start, end: None, }, ) } fn covering_range(&self, key: u64, frames: group::Bounds) -> Option<&group::Handle> { self.lookup.get(&key)?.copies().find(|group| group.covers(frames)) } /// The local freshness bound, or `None` when unknown or unlimited: the publisher's max /// age under the local ceiling, how long a reader may wait for a late group, without /// changing the publisher's metadata. fn max_age_bound(&self) -> Option { let published = self.info.as_ref()?.max_age; match published { Some(age) => Some(self.broadcast.cache_duration.map_or(age, |ceiling| age.min(ceiling))), None => self.demand_bound(), } } /// The bound on the aggregate subscription: the local cache ceiling alone, or `None` /// when unlimited. It needs no info, so a track asks upstream for the same demand /// before and after its info lands, and the publisher enforces its own max age. fn demand_bound(&self) -> Option { self.broadcast.cache_duration } /// The live edge a subscription served `servable` measures drift against: the /// highest-key group in it that has presented a frame, or `None` while nothing /// stamped is servable (an unstamped track drives no media-time expiry; the wall /// clock alone judges it). /// /// One scan answers a whole poll. The highest-key edge in range is above /// every candidate below it and above none at or past it. Recomputing per candidate /// would make walking a backlog of N groups off in one poll cost O(N^2). /// /// Bounded because a group can only be late relative to data that would actually be /// served in its place: a reader capped by [`Subscriber::with_range`] can't jump past /// the cap, and one reading disjoint ranges never reads the gap between them, so the /// groups it still wants aren't stale just because the track ran on. /// Fetched backfill is absent from `arrival`, so it cannot age subscription content /// as though it were a live replacement. fn live_edge(&self, servable: &Servable) -> Option { self.find_servable(servable, 0, true, |slot| { if !slot.is_shown() { return None; } // The map is ordered by key, so the first stamped group from the // back is the newest content that exists. let timestamp = self.started_at(slot)?; Some(PresentationEdge { key: slot.group.start, stamp: slot.stamp, timestamp: slot.group.latest().unwrap_or(timestamp), }) }) } /// This track's own edge within `servable`, for measuring drift. A scale-less track's /// edge is its newest shown group's key, so its max age counts bare units. fn drift_edge(&self, servable: &Servable) -> Edge { Edge { presentation: self.live_edge(servable), servable: servable.clone(), } } /// Wake `waiter` once a live group held starts: a group starting wakes no reader parked on /// the track, and a live join's reach back waits on one to measure the live edge by. fn watch_starts(&self, waiter: &kio::Waiter) { for (_, slot) in self.walked(..) { let _ = slot.group.poll_started(waiter); } } /// The live-feed groups in `range` still served: an aborted one cannot start a join, whose /// floor would then pass the usable group before it. fn walked(&self, range: impl ops::RangeBounds) -> impl DoubleEndedIterator { self.live(range).filter(|(_, slot)| slot.is_walked()) } /// The live feed's cached groups within `range`, by key: never fetched backfill. fn live(&self, range: impl ops::RangeBounds) -> impl DoubleEndedIterator { // Without backfill every cached group is the feed's, so the cache is walked as is. match self.visible.len() == self.lookup.len() { true => LiveWalk::Cached(self.lookup.range(range)), false => LiveWalk::Indexed( self.visible .range(range) .map(|key| (key, self.lookup.get(key).expect("an indexed live slot"))), ), } } /// The first live-feed group at or above `from` that `servable` covers for which `f` /// answers, lowest first or, with `rev`, highest first: one lookup for a single run, and /// the reader's index of the live groups among several. fn find_servable<'a, R>( &'a self, servable: &Servable, from: u64, rev: bool, mut f: impl FnMut(&'a Slot) -> Option, ) -> Option { match servable { Servable::Run { start, end } => { let from = from.max(*start); // A range ending before it starts would panic, and holds nothing anyway. if end.is_some_and(|end| end < from) { return None; } let end = end.map_or(Bound::Unbounded, Bound::Excluded); let mut live = self.live((Bound::Included(from), end)); match rev { true => live.rev().find_map(|(_, slot)| f(slot)), false => live.find_map(|(_, slot)| f(slot)), } } Servable::Runs(several) => { let hi = match several.cap { Some(cap) => cap.checked_sub(1)?, None => u64::MAX, }; if from > hi { return None; } // Through the reader's index of the live groups among its runs. several .live .find(self, &several.groups, from, hi, rev, &mut f) .map(|(_, found)| found) } } } /// Whether `key` still holds the servable incarnation `stamp`. fn holds(&self, key: u64, stamp: u32) -> bool { self.lookup .get(&key) .is_some_and(|slot| slot.stamp == stamp && slot.serves_end()) } /// The furthest presentation time the group at `key` could still reach: where /// its immediate successor begins, or `None` when nothing proves where it stops. /// /// An upper bound, deliberately. A frame's duration is not on the wire, so a group's /// own last timestamp says where it *starts* presenting, not where it ends; only its /// successor's start proves it cannot run past it. The bound is the *immediate* /// servable successor's start, so an unstamped successor leaves the reach unbounded /// until it presents its first frame, and so does having no successor in `servable` /// at all. fn reach(&self, key: u64, servable: &Servable) -> Option { if let Some(end) = self.own_end(key) { return Some(end); } self.started_at(self.successor(key, servable)?) } /// A key, or a group's end, in presentation time: in the track's timescale, or unscaled /// on a scale-less track, over the whole key range, whose max age counts bare units as /// milliseconds (see [`PresentationEdge::behind`]). fn presented(&self, value: u64) -> Option { match self.info.as_ref()?.timescale { Some(scale) => Timestamp::new(value, scale).ok(), None => Some(Timestamp::unscaled(value)), } } /// Where the group in `slot` starts presenting: its first frame's timestamp, or its key /// on a scale-less track, whose frames carry none. fn started_at(&self, slot: &Slot) -> Option { match self.info.as_ref()?.timescale { Some(_) => slot.group.timestamp(), None => self.presented(slot.group.start), } } /// Where group `key` ends, once its producer finished it whole: what bounds its reach /// without a successor (see [`group::Producer::finish_at`]). fn own_end(&self, key: u64) -> Option { self.presented(self.lookup.get(&key)?.group.end()?) } /// When group `key`, finished whole, stops presenting on the wall clock: its arrival plus /// its span, which stands in for a successor arriving, so the last group of a paused or /// finished track ages out without one. fn ended_at(&self, key: u64) -> Option { let slot = self.lookup.get(&key)?; let span = self.presented(slot.group.end()?.checked_sub(key)?)?; slot.arrived.checked_add(age(span)?) } /// The group after `key` that bounds it: the first servable one in `servable` whose /// first frame opened or that finished, stamped or not, empty or not. fn successor(&self, key: u64, servable: &Servable) -> Option<&Slot> { self.find_servable(servable, key.checked_add(1)?, false, |slot| { slot.is_shown().then_some(slot) }) } /// The group after `key` that starts its wall clock: like [`Self::successor`], but /// counting a group before its first frame. Its header arrived, so a relay ages the group /// before it as the origin does, even if its first frame never comes. fn arrival(&self, key: u64, servable: &Servable) -> Option<&Slot> { self.find_servable(servable, key.checked_add(1)?, false, |slot| { slot.is_live().then_some(slot) }) } /// The slot `select` picks, with `waiter` parked on its abort. /// /// An abort only closes the group, never touching the track, so one that landed before /// registering re-selects, or the replacement would go unwatched. fn watch_abort<'a>( &'a self, waiter: &kio::Waiter, select: impl Fn(&'a Self) -> Option<&'a Slot>, ) -> Option<&'a Slot> { loop { match select(self) { Some(slot) if slot.group.poll_aborted(waiter).is_ready() => continue, slot => return slot, } } } /// Whether `key`'s successor arrived a full `budget` ago. /// /// The wall-clock twin of [`Self::reach`], re-read the same way rather than latched: /// a group that lands between the two restarts the clock, and an aborted successor /// hands it to the next one. The clock is read only once there is a successor, /// so judging the newest group, the common case, costs no clock read. fn superseded(&self, key: u64, servable: &Servable, budget: Duration) -> bool { match self.arrival(key, servable) { Some(successor) => self.aged(successor.arrived, budget), None => self.ended_at(key).is_some_and(|ended| self.aged(ended, budget)), } } /// Whether a full `budget` passed since a successor `arrived`, or since a group finished /// whole stops presenting, which may still be ahead: a zero budget ages it only once /// that comes. fn aged(&self, arrived: crate::time::Instant, budget: Duration) -> bool { let now = self.cache.pool().now_wall(); now >= arrived && now.saturating_duration_since(arrived) >= budget } /// When a group was superseded, given `arrival`, its cached successor in `servable`: when /// `arrival` arrived, or when the successor `superseded` recorded did, unless a group landed /// between them since. The record outlives the successor's eviction: otherwise an evicted /// successor restarts the clock at whichever newer group the cache still holds, which a /// budget past the cache window never reaches. A recorded successor that failed hands the /// clock on, as a re-read does, whether or not the cache has reclaimed it yet. fn superseded_at( &self, arrival: Option<&Slot>, servable: &Servable, superseded: Option<&Superseded>, ) -> Option { let Some(superseded) = superseded else { return arrival.map(|slot| slot.arrived); }; let mut last = superseded.0.lock().unwrap_or_else(PoisonError::into_inner); if let Some(recorded) = &*last && !recorded.fate.failed() && servable.contains(recorded.key) && arrival.is_none_or(|slot| slot.group.start > recorded.key) { return Some(recorded.arrived); } *last = arrival.map(|slot| Successor { key: slot.group.start, arrived: slot.arrived, fate: slot.group.fate(), }); arrival.map(|slot| slot.arrived) } /// Whether the group at `key` has drifted further behind `edge` than `budget` /// tolerates, so a subscriber should skip it rather than hand it over. /// /// Presentation time measures a group by how far it could still *reach*, not by how far /// behind it started. Being behind is survivable: priority transmits newer groups /// first, so a backlog bursts at whatever rate is left over and closes the gap faster /// than the live edge advances. What is not survivable is having nothing left worth /// delivering, so a group is abandoned only once everything it could still present /// falls outside the budget. /// /// A group's reach is bounded by its own end once finished whole, else by its immediate /// successor (see [`Self::reach`]): it cannot present past where the next group begins. /// The candidate itself needs no timestamp: an empty group is bounded by its stamped /// successor the same way. /// /// The wall clock judges it too: a group is also stale once its successor arrived a /// full `budget` ago (see [`Self::superseded`]), so a peer whose timestamps stop /// advancing or an empty successor that never presents cannot pin a superseded group. /// Media time still convicts first when a burst arrives at once. A group finished whole /// ages from its arrival plus its span instead when no successor arrived, so the last /// group of a paused or finished track ages out; one with neither is never convicted. /// /// The bound is exclusive, so the comparison is `>=` rather than `>`: the freshest frame /// a group could still hold sits just *below* its reach, so an age equal to the budget /// already puts every frame in it strictly past the budget. That also makes a zero /// budget fall out for free instead of needing a special case. /// /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and a group received from past its first frame carries a later /// frame's timestamp on a low key without being an edge at all. /// /// A group below [`Self::cache_floor`] is stale whether or not the cache still holds /// it: an eviction between selecting it and this verdict must not hand it out. fn is_stale(&self, key: u64, edge: &Edge, budget: Duration) -> bool { key < self.cache_floor(budget) || (self.lookup.contains_key(&key) && self.drifted(key, edge, budget)) } /// A finite subscription cannot admit a cache whose reach its resumed feed cannot /// bound. Unlimited history still owes it, and one-shot fetches judge their own budget. fn cache_floor(&self, budget: Duration) -> u64 { self.live_floor.filter(|_| budget != Duration::MAX).unwrap_or(0) } /// Whether group `key`, cached or not, has a successor a full `budget` behind /// the edge, or one that arrived a full `budget` ago. fn drifted(&self, key: u64, edge: &Edge, budget: Duration) -> bool { if self.superseded(key, &edge.servable, budget) { return true; } self.live_above(key, edge) .is_some_and(|live| live.behind(self.reach(key, &edge.servable), budget)) } /// The edge that could convict group `key`: above it, and still holding the /// servable incarnation it was resolved from. It was resolved under an earlier lock, /// and failing safe (delivering) is right, since the next poll resolves a fresh edge. fn live_above(&self, key: u64, edge: &Edge) -> Option { edge.presentation .filter(|live| live.key > key && self.holds(live.key, live.stamp)) } /// [`Self::drifted`] within `servable`, parking `waiter` on exactly what else can move the /// verdict: the successor's first frame or abort, the edge's abort, the edge crossing /// the media deadline, and the wall clock crossing the successor's arrival plus /// `budget`. Parking on every track change instead wakes each of N parked reads per append. /// /// The caller parks on a group landing above `key` first, in the way that fits /// whether this track holds it. A reader holding the group passes its `superseded`, so /// the wall clock outlives the successor's eviction; see [`Self::superseded_at`]. fn poll_drifted( &self, key: u64, servable: &Servable, budget: Duration, superseded: Option<&Superseded>, waiter: &kio::Waiter, ) -> bool { // Unbounded until a group lands above. Registered before judging, so a racing abort // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. let arrival = self.watch_abort(waiter, |state| state.arrival(key, servable)); // A group finished whole ages from its own end without one; see [`Self::ended_at`]. let Some(arrived) = self .superseded_at(arrival, servable, superseded) .or_else(|| self.ended_at(key)) else { return false; }; if let Some(deadline) = arrived.checked_add(budget) { self.cache.watch_wall(deadline, waiter); } if self.aged(arrived, budget) { return true; } let reach = loop { if let Some(end) = self.own_end(key) { break end; } // An abort can change reach even after the successor is stamped. Unbounded // until a group shows above, or the successor presents its first frame. let Some(successor) = self.watch_abort(waiter, |state| state.successor(key, servable)) else { return false; }; if let Some(reach) = self.started_at(successor) { break reach; } let successor = &successor.group; if successor.poll_started(waiter).is_pending() || successor.timestamp().is_none() { return false; } }; // Registered before the edge is resolved, so a write crossing the deadline is // either seen here or wakes us. self.cache.wakes().watch_deadline(reach, budget, waiter); loop { let edge = self.drift_edge(servable); // The edge's abort presents nothing and never touches the track, yet hands the // edge to a lower group that may already sit past the deadline. Watch it, and // re-resolve if it already landed. Terminates: an aborted group is never the // edge again, so each pass resolves a lower one. if let Some(live) = edge.presentation && let Some(slot) = self.lookup.get(&live.key) && slot.group.poll_aborted(waiter).is_ready() { continue; } return self.drifted(key, &edge, budget); } } /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] /// once it can never be served. A missing group is a failure ([`Error::NotFound`]), not an /// end-of-stream. The handler side (a refusal, or no [`Requests`] at all) lives /// in the track's fills; [`Fetching`] polls both. fn poll_fetch_cached(&self, key: u64, frames: group::Bounds) -> Poll> { // A group aborted between the lookup and the consume (an abandoned fetch cut // short) is a miss, not a hit that fails on its first read. if let Some(group) = self.covering_range(key, frames) && let Some(consumer) = group.try_consume() { // A cache hit refreshes the group: it resets both its age (expiry keys // off the last access) and its standing against the pool-wide average, // so the eviction walk keeps it over never-read groups. group.cache_refresh(); return Poll::Ready(Ok(consumer)); } if let Some(err) = self.outcome.aborted() { return Poll::Ready(Err(err.clone())); } // Past the final key: the group can never exist. if self.final_key().is_some_and(|fin| key >= fin) { return Poll::Ready(Err(Error::NotFound)); } Poll::Pending } /// Expire groups idle past the pool's wall-clock LRU window, never the latest. /// /// The window is the pool's [`expiry`](cache::Pool::expiry), not the track's /// [`max_age`](Info::max_age): max age only bounds how long a superseded group stays /// fresh and frees nothing, while this is the cache's own guard against unread content pinning RAM, shared by every /// track in the pool. /// /// One bounded, rotating scan over the eviction order, which holds every cached /// group except the protected latest. The cursor persists in the cache account, so /// entries beyond one scan window can't be starved by fresh (recently read, /// fetched, or written) entries in front of them: every position is revisited /// within a few writes. Expiry throughput is therefore EVICT_SCAN groups per write; the /// byte budget reclaims the remainder under memory pressure, and the pool's sweep /// ([`Self::expiry_scan_drain`]) covers a track that stopped writing entirely. pub(super) fn evict_expired(&mut self) { let scan = self.expiry_scan(); self.evict_expired_scan(scan); } /// Describe the next bounded expiry scan without mutating observable track state. pub(super) fn expiry_scan(&self) -> ExpiryScan { ExpiryScan { start: self.cache.next_expiry_scan(EVICT_SCAN), width: EVICT_SCAN, now: self.cache.pool().now(), max_ticks: self.cache.pool().expiry_ticks(), gc: false, } } /// Scan every cached candidate so undated accesses and old entries cannot hide /// behind fresh entries at the front of the eviction order. pub(super) fn expiry_scan_drain(&self) -> ExpiryScan { ExpiryScan { start: 0, width: self.evict.len(), now: self.cache.pool().now(), max_ticks: self.cache.pool().expiry_ticks(), gc: true, } } #[cfg(test)] pub(super) fn date_cache_accesses(&self, now: u64) { for slot in self.lookup.values() { slot.group.cache_accessed_tick(Some(now)); } } /// Whether an expiry scan would change observable track state. /// /// Mirrors [`Self::evict_expired_scan`]'s walk exactly, stop rule included: a memo /// that scanned further would report work the scan itself will not do. pub(super) fn expiry_mutation_due(&self, scan: ExpiryScan) -> bool { let len = self.evict.len(); if len > 0 { let start = scan.start % len; let mut retained = 0; for step in 0..len.min(scan.width) { let (key, stamp) = self.evict[(start + step) % len]; let Some(slot) = self.lookup.get(&key) else { continue; }; if slot.stamp != stamp { continue; } if slot.is_spent() || (!self.protects(key) && slot.is_expired(&scan)) { return true; } retained += 1; if !scan.gc && retained >= EVICT_SCAN { break; } } } self.drops_expired(scan.now) || self .arrival .front() .is_some_and(|entry| !self.is_current(entry.key, entry.stamp)) || self .evict .front() .is_some_and(|(key, stamp)| !self.is_current(*key, *stamp)) || self.evict.len() > 2 * self.lookup.len() + EVICT_SLACK || self.arrival.len() > 2 * self.lookup.len() + EVICT_SLACK } /// Expire an ended track's idle groups, whose closed channel refuses the write /// [`Self::evict_expired_scan`] takes: abort the groups in place, releasing their /// frames, and retire expired cuts, leaving entries that every read path skips. pub(super) fn expire_closed(&self, scan: ExpiryScan) { for (key, stamp) in &self.evict { let Some(slot) = self.lookup.get(key) else { continue; }; if slot.stamp != *stamp || (slot.is_aborted() && slot.cut().is_none()) || !slot.is_expired(&scan) { continue; } if slot.is_aborted() { slot.retired.store(true, Ordering::Relaxed); } else { slot.abort(Error::Old); } } } /// Apply a scan previously selected by [`Self::expiry_scan`] or /// [`Self::expiry_scan_drain`]. pub(super) fn evict_expired_scan(&mut self, scan: ExpiryScan) { let len = self.evict.len(); if len > 0 { let start = scan.start % len; let mut retained = 0; for step in 0..len.min(scan.width) { let (key, stamp) = self.evict[(start + step) % len]; let Some(slot) = self.lookup.get(&key) else { continue; }; if slot.stamp != stamp { // A historical hint; the live entry is elsewhere in the queue. continue; } // Already aborted: the frames are gone, reclaim the slot so a // later fetch can serve the key again. if slot.is_spent() { self.reclaim_aborted(key); continue; } if self.protects(key) || !slot.is_expired(&scan) { // Writes keep their scan bounded. Cleanup visits the entire // queue to date pending accesses and find idle entries behind them. retained += 1; if !scan.gc && retained >= EVICT_SCAN { break; } continue; } // An expired cut goes like the group it was, still forwarded as a drop. if slot.is_aborted() { self.reclaim_aborted(key); continue; } // Take the group out of the cache and abort it, so any consumer // still reading surfaces `Error::Old` instead of blocking forever // on a frame that will never arrive. self.evict_group(key, Error::Old); } } self.expire_drops(scan.now); // Trim dead leading arrival entries. An entry is dead once its slot is gone or // re-stamped by a newer incarnation. while let Some(entry) = self.arrival.front() { if self.is_current(entry.key, entry.stamp) { break; } self.arrival.pop_front(); } self.compact_arrival(); // Drop dead leading eviction entries so scans stay over live candidates. while let Some((key, stamp)) = self.evict.front() { if self.lookup.get(key).is_some_and(|slot| slot.stamp == *stamp) { break; } self.evict.pop_front(); } // Dead entries behind a live front can linger; rebuild once they clearly // outnumber the live slots. if self.evict.len() > 2 * self.lookup.len() + EVICT_SLACK { let lookup = &self.lookup; self.evict .retain(|(key, stamp)| lookup.get(key).is_some_and(|slot| slot.stamp == *stamp)); } } /// Drop the dead arrival entries once they clearly outnumber the cached groups, wherever /// they sit: behind a protected live edge, none ever reaches the front. A rebuild drops /// more entries than it keeps, each pushed once, so it costs a constant per push. fn compact_arrival(&mut self) { if self.arrival.len() > 2 * self.lookup.len() + EVICT_SLACK { let lookup = &self.lookup; self.arrival .retain(|entry| lookup.get(&entry.key).is_some_and(|slot| slot.stamp == entry.stamp)); } } /// The arrival entries at or after position `at`. fn arrivals_from(&self, at: usize) -> vec_deque::Iter<'_, Arrived> { let start = self.arrival.partition_point(|entry| entry.at < at); self.arrival.range(start..) } /// Whether `(key, stamp)` names the currently cached incarnation. fn is_current(&self, key: u64, stamp: u32) -> bool { self.lookup.get(&key).is_some_and(|slot| slot.stamp == stamp) } /// Whether `key` is the live edge, which eviction and expiry never take /// while a producer remains. fn protects(&self, key: u64) -> bool { !self.outcome.is_released() && Some(key) == self.latest_group } /// Publishing ended: drop the ingress scope, whose session a reader still draining the /// track must not keep open. The groups already written keep their own meters. fn release_ingress(&mut self) { self.ingress = stats::Scope::default(); } /// No producer remains: stop protecting the live edge, so the pool's idle expiry /// reclaims every group once readers stop touching it, and a stale consumer can't /// pin the cache (and its frame buffers) forever. fn close_cache(&mut self) { self.cache.end_largest(); if let Some(latest) = self.latest_group && let Some(slot) = self.lookup.get(&latest) { slot.cache_demote(); self.evict.push_back((latest, slot.stamp)); } } /// End the track abruptly with `err`: drop the open groups nobody will finish, keep the /// finished ones for readers still draining, and stop protecting the live edge. /// /// Returns where production stopped, snapshotted before the open group it may sit in /// goes (an abrupt end is exactly when a replacement route asks where to resume), and /// every unfinished copy, cached or not, to abort once the track lock is released, so a /// reader that already pulled one ends with `err` too and its writer fails. fn tear_down(&mut self, err: Error) -> (Option, OpenGroups) { self.release_ingress(); let resume = self.resume_position(); let groups = self .lookup .values() .flat_map(Slot::copies) .filter(|group| !group.is_settled()) .cloned() .chain(self.detached.drain(..).filter_map(|copy| copy.writing())) .collect(); let dropped = StreamError::from(&err); // A slot survives only on a copy holding the whole group. A finished bounded copy // serves no group reader, so it leaves the cache like an open one; a reader already // holding it keeps it, since its writer settled and it is not aborted above. let open: Vec<_> = self .lookup .iter() .filter(|(_, slot)| slot.visible && !slot.copies().any(group::Handle::is_finished)) .filter_map(|(&key, _)| Some(key..key.checked_add(1)?)) .collect(); for groups in open { self.drop_groups(groups, Hole::drop(dropped.clone())); } self.lookup .retain(|_, slot| slot.copies().any(group::Handle::is_finished)); let lookup = &self.lookup; self.visible.retain(|key| lookup.contains_key(key)); self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) .retain(|key, _| lookup.contains_key(key)); (resume, OpenGroups { groups, err }) } /// Take `key` out of the cache, along with its checked start. fn remove_slot(&mut self, key: u64) -> Option { let slot = self.lookup.remove(&key)?; self.visible.remove(&key); self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) .remove(&key); Some(slot) } /// Record that the live feed delivered `key`, then fold the gaps that aged out. fn record_arrival(&mut self, key: u64, now: crate::time::Instant) { self.arrivals.arrive(key, now); if !self.anchors.received() && self.appended.is_none_or(|appended| key >= appended) && let Some(end) = key.checked_add(1) { self.produced.insert(key..end); } // A feed still delivering keeps the gap above its last arrival open. self.refresh_tail(now); self.fold_arrivals(now); } /// Fold every gap in the arrivals a subscription's grace stopped waiting for: the grace /// the session's tail uses, from the max delay the track's subscribers ask for. fn fold_arrivals(&mut self, now: crate::time::Instant) { // Only a gap folds, so an in-order feed never reads the subscriptions. if self.arrivals.has_gaps() { self.arrivals.set_grace(self.arrival_grace()); self.arrivals.expire(now); } self.bill_arrivals(); } /// Bill the arrivals' gaps to the pool, after any change to them. The run they split is /// the track's own, so an in-order feed bills nothing. fn bill_arrivals(&mut self) { let billed = self.arrivals.len().saturating_sub(1) as u64 * ledger::RUN_OVERHEAD; let before = self.arrivals_bill.size(); if billed > before { self.arrivals_bill.add(billed - before); } else if billed < before { self.arrivals_bill.sub(before - billed); } } /// Fold the oldest gaps of the arrivals until the debt is paid or `cap` bytes are freed, /// returning the bytes freed: nothing waits on them any longer, as when their grace /// passes, so a reader parked on one judges it missing. fn shed_arrivals(&mut self, cap: u64) -> u64 { let mut paid = 0u64; while self.debt > 0 && paid < cap { let Some(gap) = self.arrivals.fold_oldest() else { break; }; let billed = self.arrivals_bill.size(); self.bill_arrivals(); let freed = billed.saturating_sub(self.arrivals_bill.size()); self.debt -= freed.min(self.debt); paid = paid.saturating_add(freed); self.cache.wakes().arrived(gap); } paid } /// How long a missing live group is waited for, from the max delay the track's /// subscribers ask for. fn arrival_grace(&self) -> Duration { let max_delay = clamp_max_delay(self.subscriptions.lock().max_delay(), self.max_age_bound()); ledger::grace(max_delay) } /// Refuse a request nobody published with `err`, waking the subscribes waiting on it. /// No producer ever wrote to it, so there is nothing to tear down, but a fill answered /// before the refusal may have cached the live edge, which nothing protects anymore. fn reject(&mut self, err: Error, retry: Option) { self.release_ingress(); let released = self.outcome.is_released(); self.outcome = Outcome::Aborted { err, end: None, resume: None, retry, }; if !released { self.close_cache(); } } /// Attach the publisher's immutable metadata without replacing it with local cache policy. fn install(&mut self, info: Info) { // A replaced max age moves every parked read's budget. if self.info.replace(info).is_some() { self.cache.wakes().wake_all(); } } /// Create the shared state for a track under `broadcast`, along with the cache /// account it and its groups charge into. /// /// The account holds a [`kio::Weak`] back to this state: a group must be able to /// settle the track's eviction debt as it writes, but the track owns its cached /// groups, so anything stronger would make the pair immortal. fn spawn(broadcast: Arc) -> kio::Producer { let state = kio::Producer::new(Self { broadcast: broadcast.clone(), ..Default::default() }); let cache = cache::Track::new(broadcast.pool.clone(), broadcast.frames.clone(), state.downgrade()); let mut new = state.write().ok().expect("a new track is open"); new.drops_bill = cache.extra(); new.arrivals_bill = cache.extra(); new.cache = cache; drop(new); state } /// Record a written key. It raises the largest too, so a later group below it never /// changes it, even one below a reached end. /// /// `answers` when it is a group subscribers receive, which answers a pending refresh. /// /// The demand wakes it owes wait in [`Self::deferred`] for the caller to fire once its guard /// is released. fn advance(&mut self, key: u64, answers: bool) { let kept = self.kept_drops(); self.max_key = Some(self.max_key.map_or(key, |max| max.max(key))); if self.kept_drops() > kept { self.rearm_drops(kept); } self.cache.advance_largest(Position::group(key)); let wakes = self.subscriptions.lock().wrote(key, answers); self.deferred .extend(wakes.into_iter().filter(|wakes| !wakes.is_empty())); } /// Take the wakes [`Self::advance`] deferred, to fire once the state guard is released. pub(super) fn deferred(&mut self) -> Deferred { Deferred(std::mem::take(&mut self.deferred)) } /// One past every key this track holds, or 0 on an empty track. No key exceeds the reserved /// `u64::MAX`, so it never overflows. fn next_key(&self) -> u64 { self.max_key.map_or(0, |max| max + 1) } /// Reject a key that is still cached; a dead (aborted or evicted) /// incarnation is removed so a fresh group can serve the key again. Subscribers may /// have seen its start, so it stays as the eviction watermark. /// /// Best effort: nothing remembers a key whose slot is already gone, so a /// publisher re-sending a long-evicted key is accepted as new. /// A group that starts above `frame_start` is also replaceable: it cannot answer a /// request from there, so the wider producer takes the slot. Readers already /// draining the old one keep their own handle. /// /// A fetched backfill never holds the key against the feed: subscribers read only /// the feed's groups, so its copy takes the slot, and the fetch's readers keep theirs. fn claim_key(&mut self, key: u64, frame_start: u64) -> Result<()> { if self.holding_group(key, frame_start).is_some() { return Err(Error::Duplicate); } self.displace(key); Ok(()) } /// The copy of the feed's `key` that still answers from `frame_start`, if any. fn holding_group(&self, key: u64, frame_start: u64) -> Option<&group::Handle> { self.lookup.get(&key).filter(|slot| slot.visible)?; self.covering_group(key, frame_start) } /// Insert a freshly-created group into the cache. /// /// Updates the live edge, demoting the previous latest into the eviction order; /// the current latest is never enqueued, which is what protects it from /// eviction. `visible` controls arrival-order delivery: publisher-produced /// groups reach subscribers, fetched backfill is served by key only. A group /// counts as content only once [`Self::reveal`] reports its first frame or finish. fn insert_group(&mut self, group: &group::Handle, visible: bool) { self.latch_settled(); let key = group.start; if visible { self.contents.remove(key); } group.register_content(&self.contents); self.next_stamp = self.next_stamp.wrapping_add(1); let stamp = self.next_stamp; // The live edge is tracked separately from `max_key`, which can run past // any cached group: demotion must still fire when the next group lands beyond it. if self.latest_group.is_none_or(|latest| key >= latest) { // Demote the previous latest: it joins the eviction order (and the // pool's access average) like any other cached group. if let Some(latest) = self.latest_group && key > latest && let Some(prev) = self.lookup.get(&latest) { prev.cache_demote(); self.evict.push_back((latest, prev.stamp)); } self.latest_group = Some(key); } if visible { if let Feed::Live { first, .. } = &mut self.feed { first.get_or_insert(key); } self.newest_produced = self.newest_produced.max(Some(key)); } if !self.protects(key) { group.cache_demote(); self.evict.push_back((key, stamp)); } self.advance(key, visible); if !visible { self.fetched += 1; } // A publisher may re-create a group it cut; the key is no longer given up. if visible && self.drops.contains_key(&key) && let Some(end) = key.checked_add(1) { self.change_drops(key..end, None); } let now = self.cache.pool().now_wall(); if visible { self.record_arrival(key, now); } self.cached += 1; let replaced = self.lookup.insert( key, Slot { group: group.clone(), stamp, cached: self.cached, visible, arrived: now, head: None, dropped: None, retired: AtomicBool::new(false), }, ); match visible { true => self.visible.insert(key), false => self.visible.remove(&key), }; if replaced.is_some() { self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) .remove(&key); } self.cache.wakes().arrived(key..=key); if visible { // Readers and group demand reach it at once. Its arrival starts the wall clock of // the reads it succeeds, but it counts as content only once it starts. self.arrival.push_back(Arrived { at: self.next_arrival, key, stamp, }); self.next_arrival += 1; self.compact_arrival(); // Publishes the landing to reads judging expiry without the track lock, then // checks for one parked. Reveal only holds a read lock and cannot use the shortcut. if self.cache.wakes().land(key) { self.wake_landed(key); } } } /// Count `group` as content now that its first frame opened or it finished, if it /// still holds its slot: wake the reads it succeeds. Returns whether an answer waits on /// it, which the caller shows with [`Self::shown`] under the write lock. /// /// Takes the read lock so a group starting wakes only the reads it can move, not every /// reader parked on the track. pub(super) fn reveal(&self, group: &group::Handle) -> bool { let key = group.start; match self.lookup.get(&key) { Some(slot) if slot.visible && slot.group.is_clone(group) && !slot.group.is_aborted() => {} _ => return false, } self.wake_landed(key); matches!(self.feed, Feed::Answered { from } if key >= from) } /// Wake the reads a group landing at `key` succeeds. fn wake_landed(&self, key: u64) { let below = self .live(..key) .rev() .find(|(_, slot)| slot.is_shown()) .map(|(below, _)| *below); let oldest = self.lookup.first_key_value().map_or(key, |(oldest, _)| *oldest); self.cache.wakes().landed(below, key, oldest); } /// Content at `key` became readable: an answer waiting on it now shows. pub(super) fn shown(&mut self, key: u64) { if let Feed::Answered { from } = self.feed && key >= from { self.go_live(Some(from)); } } /// Admit a freshly-created group: settle eviction debt first (so the newcomer /// can never be a victim of the very write that created it), insert it, then /// expire idle groups. fn commit_group(&mut self, group: &group::Handle) { self.charge_debt(); self.insert_group(group, true); self.evict_expired(); } /// Accrue and pay eviction debt for everything written since the last charge: /// this track's account, which the groups' charges feed on every frame (so /// growth on already-demoted groups and backfill is billed too). /// /// Runs BEFORE the new group is inserted, so a brand-new entry is never a /// victim of the very write that created it. A track whose oldest content is /// staler than the pool-wide average access time accrues at double rate, so /// stale-heavy tracks drain first. /// /// Also runs from the frame-write path via [`cache::Track::settle`], which is /// why it's reachable from the account, so a track that only appends frames to /// open groups still pays. pub(super) fn charge_debt(&mut self) { let written = self.cache.take_written(); let pool = self.cache.pool().clone(); // Held heads go first: they are a failed copy's frames that no track can evict. pool.reclaim_held(); match pool.accrue(written) { Some(mut accrued) => { if self.oldest_is_stale(&pool) { accrued = accrued.saturating_mul(2); } // `used` bounds what eviction could ever free, keeping a track that // can't pay (everything protected) from hoarding a stale schedule. self.debt = self.debt.saturating_add(accrued).min(pool.used()); // Cap each payment at twice what was written so one write never dumps // a deep backlog at once; the remainder carries to the next write. let cap = written.saturating_mul(2); self.pay_debt(&pool, cap); } // Under capacity there is nothing to work off, and stale debt would // cause a spurious eviction burst at the next pressure spike. None => self.debt = 0, } } /// Whether this track's oldest evictable group was accessed at or before the /// pool-wide average, doubling the debt it accrues. A dead entry at the front /// just reads as not-stale until the next payment or expiry cleans it up. fn oldest_is_stale(&self, pool: &cache::Pool) -> bool { let Some(average) = pool.average() else { return false; }; let Some((key, stamp)) = self.evict.front() else { return false; }; let Some(slot) = self.lookup.get(key) else { return false; }; slot.stamp == *stamp && !slot.is_aborted() && slot.cache_accessed() <= average } /// Abort this track's stalest groups until the outstanding debt is paid, or /// `cap` bytes have been freed by this call. /// /// Deliberately approximate, Redis-style: at most a handful of live candidates /// are examined per call, from the front of the eviction order. A group /// accessed more recently than the pool-wide average is protected and rotates /// to the back, so fresh content in this track never dies while staler content /// survives elsewhere; the unfreed bytes keep the pool over budget, shifting /// the debt onto the tracks holding that staler content. When the next victim /// is larger than the remaining debt it is left in place and the debt carries /// over, so a small write never evicts a huge group (once the debt does cover /// it, that one victim may overshoot `cap`). Returns the bytes it freed. fn pay_debt(&mut self, pool: &cache::Pool, cap: u64) -> u64 { let average = pool.average().unwrap_or(0); let mut paid = 0u64; let mut scanned = 0usize; let mut fresh = Vec::new(); // Whether every cold group was offered, so a debt left at the end is one the track // itself cannot pay. let mut swept = true; for _ in 0..self.evict.len() { if self.debt == 0 || paid >= cap || scanned >= EVICT_SCAN { swept = self.debt == 0 || paid >= cap; break; } let Some((key, stamp)) = self.evict.pop_front() else { break; }; let Some(slot) = self.lookup.get(&key) else { // Evicted or expired; discard the dead entry. continue; }; if slot.stamp != stamp { // A historical hint; the live entry is elsewhere in the queue. continue; } if slot.is_aborted() { // Aborted upstream: the frames are already gone, reclaim the slot. self.reclaim_aborted(key); continue; } if self.protects(key) { // The live edge is never enqueued, but tolerate finding it anyway. self.evict.push_back((key, stamp)); continue; } scanned += 1; // Protected: accessed more recently than the average (a fresh insert, // an active reader, or a FETCH hit, which also covers a backfill still // being filled). Rotate to the back, but shed its front below if the debt // outlives the scan. if slot.cache_accessed() > average { self.evict.push_back((key, stamp)); fresh.push(key); continue; } // The full footprint including overhead, so even empty groups repay // their share of the budget when evicted. let size = slot.cache_size(); if size > self.debt { self.evict.push_front((key, stamp)); return paid; } self.debt -= size; paid = paid.saturating_add(size); self.evict_group(key, Error::Evicted); } // Then the arrivals' oldest gaps, which costs a reader only the wait for a late group, as // the grace passing would. if self.debt > 0 && paid < cap { paid = paid.saturating_add(self.shed_arrivals(cap - paid)); } // Then the drop set, oldest keys first: losing a run costs a reader a judgment or // a fill, never content being read. if self.debt > 0 && paid < cap { paid = paid.saturating_add(self.shed_drops(cap - paid)); } // No cold group left to pay with: shed the front of the groups that could not go // whole, the ones being read and then the live edge, so neither grows without bound. fresh.extend(self.latest_group.filter(|latest| self.protects(*latest))); for key in fresh { if self.debt == 0 || paid >= cap { break; } let Some(slot) = self.lookup.get(&key) else { continue; }; for group in slot.copies() { let freed = group.shed(self.debt, cap - paid.min(cap)); self.debt -= freed.min(self.debt); paid = paid.saturating_add(freed); } } // Then the flights, which readers ride for the same reason, cached group or not. for flight in self.flights.values().flatten() { if self.debt == 0 || paid >= cap { break; } if let Some(group) = flight.held() { let freed = group.shed(self.debt, cap - paid.min(cap)); self.debt -= freed.min(self.debt); paid = paid.saturating_add(freed); } } // Last, the readers' records: nothing cheaper is left, so each reader folds what it // settled for good, and one that cannot ends rather than forgo a promise. if swept && self.debt > 0 && paid < cap { self.cache.owe_readers(&mut self.debt); } paid } /// Group `key` once cached, `None` once the feed starts past it or ends before it, /// and pending while it still may land. fn landing(&self, key: u64) -> Poll> { if let Some(slot) = self.lookup.get(&key) && slot.serves_end() { return Poll::Ready(Some(slot.serving().consume())); } match self.start.key().is_some_and(|start| start > key) || self.final_key().is_some_and(|fin| fin <= key) { true => Poll::Ready(None), false => Poll::Pending, } } /// A live group, or a verdict from its arrival record, independent of stream order: /// the error it was given up with, or `None` once it is gone or never coming. fn poll_arrival( &self, key: u64, now: crate::time::Instant, grace: Duration, waiter: &kio::Waiter, ) -> Poll>> { if let Poll::Ready(Some(group)) = self.landing(key) { return Poll::Ready(Ok(Some(group))); } if let Some(err) = self.given_up(key) { return Poll::Ready(Err(err)); } if self.landing(key).is_ready() || key.checked_add(1).is_some_and(|end| self.arrivals.covers(key..end)) || self.arrivals.folds_at(key, grace).is_some_and(|at| at <= now) { return Poll::Ready(Ok(None)); } self.cache.wakes().watch_arrival(key, waiter); Poll::Pending } /// Whether a group in `groups` may still land from the live feed, by the rule /// [`Self::poll_arrival`] applies: not accounted for, not dropped, within the feed, and /// in a gap a later arrival opened less than `grace` ago, or that nothing later opened. /// Below the first arrival only when the feed was asked to start lower, as `floor` says. /// With `delivered`, only a group it lacks counts. Parks `waiter` until the gap folds. fn may_land( &self, groups: ops::Range, floor: bool, grace: Duration, delivered: Option<&Delivered>, waiter: &kio::Waiter, ) -> bool { self.find_landable(groups, floor, grace, |open, folds| { if delivered.is_some_and(|delivered| delivered.covers(open)) { return None; } if let Some(at) = folds { self.cache.watch_wall(at, waiter); } Some(()) }) .is_some() } /// The first `f` answers of the runs of `groups` that may still land from the live /// feed, lowest first, by the rule [`Self::may_land`] applies. fn find_landable( &self, groups: ops::Range, floor: bool, grace: Duration, mut f: impl FnMut(ops::Range, Option) -> Option, ) -> Option { let start = match floor { true => groups.start, false => groups.start.max(self.arrivals.first()?), }; let start = start.max(self.start.key().unwrap_or(0)); let end = groups.end.min(self.final_key().unwrap_or(u64::MAX)); let now = self.cache.pool().now_wall(); for (gap, opened) in self.arrivals.gaps(start..end) { let folds = opened.and_then(|opened| opened.checked_add(grace)); if folds.is_some_and(|folds| now >= folds) { continue; } if let Some(found) = self.drops.gaps(&gap).find_map(|open| f(open, folds)) { return Some(found); } } None } /// The lowest group in `groups` and `window` a key-order reader still waits for: /// one only a subscription owes, so a received copy's live feed may still land it. Not a /// group accounted for by its arrivals (since evicted or aborted), dropped, /// outside the feed, or in a gap its grace stopped waiting for. A local track owes none. fn owed(&self, groups: ops::Range, window: &Window) -> Option { if !self.anchors.received() || self.outcome.is_released() || !self.readable() { return None; } let asked = self.start.key().is_some(); self.find_landable(groups, asked, self.arrival_grace(), |open, _| { window.clip(open).map(|run| run.start) }) } /// Record the declared first key of the live feed, replacing any earlier /// declaration: the signal is scoped to the current subscription's demand, /// which may legitimately move in either direction. `None` clears it (the /// demand dropped to the live edge, whose floor is unknown until declared). /// /// A declared start asks the feed for every group from it, so a gap in the arrivals at /// or above it, and a pending tail above the last arrival, is waited for a fresh grace: /// the feed may only now be sending it. fn set_start(&mut self, start: Start) { self.latch_settled(); self.start = start; if let Some(start) = start.key() { self.cache.wakes().arrived(..start); let now = self.cache.pool().now_wall(); self.arrivals.demand(start..u64::MAX, now); self.refresh_tail(now); } } /// Restart a received copy's tail grace from `now`, once it is waiting one out. fn refresh_tail(&mut self, now: crate::time::Instant) { if let Some(tail) = self.outcome.end_mut().and_then(|end| end.tail.as_mut()) { *tail = now; } } /// Record the exclusive end, rejecting a second end, one that would orphan /// already-produced groups, or one inside a group that already holds a frame past it. /// A mid-group end caps that group, so no later write can land past it either. fn set_end(&mut self, end: Position, error: Option) -> Result<()> { if !matches!(self.outcome, Outcome::Open) { return Err(Error::Closed); } self.declare(end, error, None, false) } /// Declare `end` on an open track, or one whose declared end it refines (the declared end /// already taken out of `outcome`), carrying over that end's `tail` and settling. fn declare( &mut self, end: Position, error: Option, tail: Option, settled: bool, ) -> Result<()> { let fin = end.final_key().ok_or(coding::BoundsExceeded)?; if let Some(max) = self.max_key && fin <= max { return Err(Error::ProtocolViolation); } if end.frame != 0 && let Some(slot) = self.lookup.get(&end.group) { for (capped, group) in slot.copies().enumerate() { if let Err(err) = group.limit(Some(end.frame)) { // A refused end declares nothing, so it caps nothing either. for group in slot.copies().take(capped) { let _ = group.limit(None); } return Err(err); } } } self.outcome = Outcome::Ending(End { at: end, error, tail, settled: AtomicBool::new(settled), }); self.cache.declare_end(); self.cache.wakes().arrived(fin..); Ok(()) } /// The declared end's position. fn declared(&self) -> Option { self.outcome.end().map(|end| end.at) } /// What a reader past the end gets: the end of stream, or the error the track ended with. fn ended(&self) -> Result> { match self.outcome.end().and_then(|end| end.error.as_ref()) { Some(err) => Err(err.clone()), None => Ok(None), } } /// The final key and the error the track ended with, once a reader can reach it. fn end_error(&self) -> Option<(u64, Error)> { if !self.is_complete() { return None; } Some((self.final_key()?, self.outcome.end()?.error.clone()?)) } /// The first group the end excludes whole: one past the last group that may exist. fn final_key(&self) -> Option { self.declared().and_then(Position::final_key) } /// Cap a new group the end cuts, so it never holds a frame past the end. fn cap(&self, group: &group::Handle) -> Result<()> { match self.declared() { Some(end) if end.frame != 0 && end.group == group.start => group.limit(Some(end.frame)), _ => Ok(()), } } /// Whether the track has reached its end: the final boundary is set and the live /// edge has caught up to it, so no further group can arrive. A future boundary /// (declared via [`Producer::finish_at`] ahead of the live edge) stays incomplete /// until the remaining groups are produced, the last producer drops without them, or, /// on a received copy, the grace for them passes. Drives the end-of-stream signal from /// the read methods (`recv_group` / `next_group` / `read_frame` return `None`). /// /// An abort before the end settled wins over it: a group below the boundary was /// still open, so the track was cut off rather than ended. fn is_complete(&self) -> bool { match &self.outcome { Outcome::Ending(end) => end .at .final_key() .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)), // A declared end the last producer dropped short of. Outcome::Sealed(_) => true, // An abort wins unless the end had already settled: a group below it was still open. Outcome::Aborted { end, .. } => end.as_ref().is_some_and(End::settled), Outcome::Open | Outcome::Dropped { .. } => false, } } /// Whether nothing more reaches the cache but what fills answer: the track was aborted, /// reached its end, or its session holds no subscription that feeds it. pub(super) fn nothing_arrives(&self) -> bool { self.outcome.aborted().is_some() || self.is_complete() || self.unfed } /// Whether a bounded window reads from the cache: whenever readers may, and on a copy no /// subscription feeds, whose groups in the window are the groups themselves however old /// its live edge is. An open window still waits for the feed. fn serves_bounded(&self, window: &Window) -> bool { self.readable() || self.unfed && window.end().is_some() } /// Whether the live edge reached `fin`, counting a gap up to it that was wholly given up /// (skipped or dropped), or a received copy's tail decides it instead: what it still /// misses up to `fin` is waited for only as long as the grace. fn reaches(&self, fin: u64) -> bool { let edge = self.edge(); edge >= fin || self.outcome.end().is_some_and(|end| end.tail.is_some()) || self.drops.gaps(&(edge..fin)).next().is_none() } /// Whether the largest can no longer advance while a producer is still held: the track /// reached its declared end, and the largest group's content is final. Every key below /// the end raised it already, and a group name fixes its content, so no legal write moves /// it after this: not a late group below it, nor a recreated or fetched copy of it. fn largest_settled(&self) -> bool { self.is_complete() && !self.cache.largest_open() } /// One past the highest key produced: where the live edge stands against the end. fn edge(&self) -> u64 { self.max_key.map_or(0, |max| max + 1) } /// Whether a group below `fin` the wire subscription still owes has yet to arrive. /// Only such a hole holds readers, so a tail with nothing missing ends them at once. fn awaits_tail(&self, fin: u64) -> bool { // Nothing arrives after an abort, so a hole left then is final. self.outcome.aborted().is_none() && self.owes(fin) } /// Whether the tail misses a key from where the feed starts up to `fin` that is /// still worth waiting for; see [`Self::tail_deadline`]. fn owes(&self, fin: u64) -> bool { self.tail_deadline(fin) .is_some_and(|at| at.is_none_or(|at| self.cache.pool().now_wall() < at)) } /// When a received copy's declared end stops waiting for its tail: the latest instant a /// key below `fin` it still misses is given up, or `Some(None)` when one is never given /// up, as under an unbounded max delay, whose grace runs past the clock's range. `None` /// when nothing is missing. /// /// A missing key is one no live group delivered and the publisher did /// not drop. A gap below an arrival is given up a grace after that arrival opened it, as /// the session's tail does. The gap above the last arrival is given up a grace after /// the end or a start was declared, or a group last arrived, so a feed still delivering toward an /// end declared ahead of it is not cut off. Judged by the arrivals, not the cache, so an evicted group /// counts as delivered, and the check costs the same however much is cached. fn tail_deadline(&self, fin: u64) -> Option> { let active = self.outcome.end()?.tail?; let grace = self.arrival_grace(); // Fetched backfill never reaches a reader in arrival order, so only the live feed's // arrivals count. Without a declared start, the first one stands in for it. // Where the feed starts is still unknown, so everything below the end not yet // delivered or dropped is owed. if self.start == Start::Pending(None) { let missing = self .arrivals .gaps(0..fin) .any(|(gap, _)| self.drops.gaps(&gap).next().is_some()); return missing.then(|| active.checked_add(grace)); } let floor = self.tail_floor(); // Nothing opened the gap above the last arrival, so its grace runs from `active`. self.arrivals .gaps(floor.min(fin)..fin) .filter(|(gap, _)| self.drops.gaps(gap).next().is_some()) .map(|(_, opened)| opened.unwrap_or(active).checked_add(grace)) // A gap that is never given up holds the end for good. .reduce(|a, b| a.zip(b).map(|(a, b)| a.max(b))) } /// The lowest key a received copy's tail owes: where the feed starts, or before a /// start is declared, its first arrival. fn tail_floor(&self) -> u64 { match self.start { Start::Pending(None) => 0, start => start.key().or(self.arrivals.first()).unwrap_or(0), } } /// Wake `waiter` when a received copy's declared end stops waiting for its tail. fn watch_tail(&self, waiter: &kio::Waiter) { if let Some(Some(at)) = self.final_key().and_then(|fin| self.tail_deadline(fin)) && self.cache.pool().now_wall() < at { self.cache.watch_wall(at, waiter); } } /// Whether the declared end is reached, the pending tail owes nothing below it, and /// every cached slot below it settled ([`Slot::is_settled`]), so nothing the end /// promised is still in flight. fn is_settled(&self) -> bool { let Some(fin) = self.final_key() else { return false; }; let reached = matches!(self.outcome, Outcome::Sealed(_)) || self.reaches(fin); reached && !self.owes(fin) && self.lookup.range(..fin).all(|(_, slot)| slot.is_settled()) } /// Record that the end settled before a write that could reopen it: an abort, a late /// group below the end, or a fresh grace for the tail. The clean outcome `closed()` saw /// then stands, and a later abort ends readers cleanly. A late group or a wider /// subscription is still served; it just cannot turn the outcome into an error. fn latch_settled(&mut self) { if let Outcome::Ending(end) | Outcome::Sealed(end) = &self.outcome && !end.settled() && self.is_settled() { end.settled.store(true, Ordering::Relaxed); } } /// Where a replacement route should pick this track up: one past the last frame /// produced, rolling to the start of the next group once the latest group is /// complete (nothing more can be appended to it). /// /// `None` while the track has produced nothing, which is an unbounded takeover. fn resume_position(&self) -> Option { // A snapshot taken when the open groups were released wins; the group it was // derived from may be gone. if let Outcome::Dropped { resume: Some(resume) } | Outcome::Aborted { resume: Some(resume), .. } = self.outcome { return Some(resume); } let max = self.latest_group?; match self.lookup.get(&max).and_then(|slot| slot.group.resume_frame()) { // Still open *and* carrying frames, so the replacement continues it // frame-by-frame. Some(frame) => Some(Position { group: max, frame: frame as u64, }), // A copy that wrote nothing has no frames to continue, and the reader // already holds its (empty) handle. Pointing a replacement at frame 0 would // hand the same key out twice, so roll to the next group exactly as a // finished one does. None => Some(Position::group(max.saturating_add(1))), } } /// The upstream subscription ended with the copy still held: what it cached may go /// stale, so readers get nothing from it until the route answers again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn set_idle(&mut self) { let newest = self .lookup .iter() .rev() .find(|(_, slot)| slot.is_shown()) .map(|(key, _)| *key); // An open group stops because this copy dropped its feed, not because the publisher // gave it up: its stream resets as a cut would, so it leaves the cache first, and the // next feed sends it again. let open: Vec = self .lookup .iter() .filter(|(_, slot)| slot.visible && !slot.is_aborted() && !slot.copies().any(group::Handle::is_finished)) .map(|(&key, _)| key) .collect(); for key in open { self.displace(key); } // Largest names an object, so an empty group holds nothing to judge. let held = self .lookup .iter() .rev() .filter(|(_, slot)| slot.is_shown()) .find_map(|(&group, slot)| { Some(Position { group, frame: slot.group.last_object()?, }) }); self.feed = Feed::Idle { newest, held }; } /// The route answered with its largest position (`None` for nothing yet): the cache /// is current up to there. `start` is the first group the feed serves, when the answer /// names it; otherwise the largest stands in. A feed that starts past a gap after /// everything cached leaves the cache unjudgeable, since nothing bounds how far an old /// group reached, so readers with a finite delay budget skip it; an unlimited read /// still serves it. Whatever the feed delivers stays. /// /// Readers wait until the cache shows that position: before then its newest group would /// read as the live edge, though the route already holds a newer one. /// /// [`Error::MalformedTrack`] when the answer is behind the cache; see [`Self::judge`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn set_live(&mut self, largest: Option, start: Option) -> Result<()> { self.judge(largest)?; self.answered(largest, start, true); Ok(()) } /// [`Self::set_live`] for a feed that serves no group from `largest`'s on, such as a /// bounded subscription of history: the cache can never show that group, so the answer /// alone makes it current, and relative starts count back from that group. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn set_live_ahead(&mut self, largest: Position, start: Option) -> Result<()> { self.judge(Some(largest))?; self.answered(Some(largest), start, false); Ok(()) } /// [`Self::set_live`] for an answer with no Largest, where the first group the feed /// sent, `first`, stands in for it. A group says nothing of how far the route's feed /// reaches, so the cache is not judged against it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn set_live_from(&mut self, first: Position, start: Option) { self.answered(Some(first), start, true); } /// An idle copy's route answered behind what its ended feed left in the cache: no /// Largest while the copy held an object, or a Largest before the newest one it held. /// The cache is likely another track, so it is dropped rather than served while the /// copy fails. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn judge(&mut self, largest: Option) -> Result<()> { let Feed::Idle { held: Some(held), .. } = self.feed else { return Ok(()); }; if largest.is_some_and(|largest| largest >= held) { return Ok(()); } let err = Error::MalformedTrack; let keys: Vec = self.lookup.keys().copied().collect(); for key in keys { self.evict_group(key, err.clone()); } Err(err) } /// The newest group an idle copy still shows from its ended feed. Judged by what /// survived: the cancel that idled the copy resets the group in flight. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn idle_cached(&self) -> Option { let Feed::Idle { newest, .. } = self.feed else { return None; }; self.lookup .range(..=newest?) .rev() .find(|(_, slot)| slot.is_shown()) .map(|(key, _)| *key) } /// The route answered with `largest`. With `reaches`, its feed reaches that group, so /// readers wait until the cache shows it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn answered(&mut self, largest: Option, start: Option, reaches: bool) { if !matches!(self.feed, Feed::Idle { .. }) { return; } // An answer past the newest group still cached leaves a gap after it. let cached = self.idle_cached(); // A feed asked from the cache's head resends what follows it, newest first on a // descending track, so its largest alone would read as a gap. let start = start.or(largest.map(|largest| largest.group)); if let (Some(start), Some(cached)) = (start, cached) && start > cached.saturating_add(1) { self.live_floor = Some(cached + 1); } match largest.map(|largest| largest.group) { Some(from) if reaches && !self.shows(from) => self.feed = Feed::Answered { from }, edge => self.go_live(edge), } } /// Whether readers can already see content at or past `key`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn shows(&self, key: u64) -> bool { self.lookup.range(key..).any(|(_, slot)| slot.is_shown()) } /// The newest live group the cache holds, and the number of frames it has so far. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn newest(&self) -> Option<(u64, u64)> { let (key, slot) = self.live(..).rev().find(|(_, slot)| slot.group.is_started())?; Some((*key, slot.group.frame_count() as u64)) } /// The largest position the cache holds: the newest live group's last frame (its head, /// while it has none). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn largest(&self) -> Option { self.newest().map(|(group, frames)| Position { group, frame: frames.saturating_sub(1), }) } fn poll_finished(&self) -> Poll> { if let Some(end) = self.declared() { Poll::Ready(Ok(end)) } else if let Some(err) = self.outcome.aborted() { Poll::Ready(Err(err.clone())) } else { Poll::Pending } } fn modify(producer: &kio::Producer) -> Result> { producer .write() .map_err(|r| r.outcome.aborted().cloned().unwrap_or(Error::Dropped)) } /// Observe a shared track's abort, or every producer leaving, without extending its /// publisher lifetime. A finish alone does not end it: lower groups may still arrive. // Sessions retain shared state while a FETCH owns its group writer. pub(crate) fn poll_aborted(producer: &kio::Producer, waiter: &kio::Waiter) -> Poll { producer .poll_closed(waiter) .map(|()| producer.read().outcome.aborted().cloned().unwrap_or(Error::Dropped)) } /// The track's outcome once final: `Ok` once its declared end is reached with every /// cached group below it finished, which a later abort keeps, or the abort error. An /// end declared with an error reports that error instead of `Ok`. /// /// A clean outcome latches once reported: a received copy's tail grace is judged when /// polled, and a later subscription asking for a longer one must not reopen it. fn poll_outcome(&self, waiter: &kio::Waiter) -> Poll> { if self.outcome.end().is_some_and(End::settled) { return Poll::Ready(self.ended().map(|_: Option<()>| ())); } if self.is_complete() { // An abort after the end settled keeps the clean end; until then a group below it // may still abort, which would cut the track off, so wait for each slot to settle // ([`Slot::is_settled`]). One whose last serving copy aborts leaves the outcome to // the track's own abort or drop. if self.outcome.aborted().is_none() && let Some(fin) = self.final_key() { for (_, slot) in self.lookup.range(..fin).filter(|(_, slot)| !slot.is_settled()) { // Watch every copy, since a serving head that aborts hands the slot back to // the live group, then judge again so a copy closing meanwhile is seen. for group in slot.copies() { let _ = group.poll_closed(waiter); } if !slot.is_settled() { return Poll::Pending; } } } if let Some(end) = self.outcome.end() { end.settled.store(true, Ordering::Relaxed); } Poll::Ready(self.ended().map(|_: Option<()>| ())) } else if let Some(err) = self.outcome.aborted() { Poll::Ready(Err(err.clone())) } else { self.watch_tail(waiter); Poll::Pending } } /// [`Self::poll_outcome`] read through a handle: the channel closing short of an /// outcome (every producer gone without an end or abort) reads as [`Error::Dropped`]. fn settle(poll: Poll, kio::Ref<'_, Self>>>) -> Poll> { Poll::Ready(match ready!(poll) { Ok(outcome) => outcome, Err(closed) if closed.is_complete() => closed.ended().map(|_: Option<()>| ()), Err(closed) => Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped)), }) } /// Abort shared track state without extending its publisher lifetime. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn abort(producer: &kio::Producer, err: Error) -> Result<()> { commit_abort(Self::modify(producer)?, err, None).abort(); Ok(()) } fn content_for_copy(&self, key: u64) -> Result>> { if let Some(content) = self.contents.get(key) { return Ok(Some(content)); } if let Some(slot) = self.lookup.get(&key) { for copy in slot.copies() { if let Some(content) = copy.activate_content()? { return Ok(Some(content)); } } } Ok(None) } /// Insert a group a [`Fill`] answered, setting the track's [`Info`] /// if it isn't accepted yet. The group's timescale comes from that info, so a /// fetch can serve an as-yet-unaccepted track (e.g. a relay with no live /// subscription). The group lands in the cache so a waiting /// [`Fetching`] resolves via [`Self::poll_fetch`]. `None` when a copy already covers it. pub(super) fn insert_group_request( &mut self, key: u64, frames: group::Bounds, info: Option, ) -> Result> { if let Some(err) = self.outcome.aborted() { return Err(err.clone()); } if let Some(fin) = self.final_key() && key >= fin { return Err(Error::Closed); } // Never produced, so no fill holds it either. if self.skipped(key) { return Err(Error::Skipped); } // Adopt the supplied info only if the track hasn't been accepted yet. Groups // created here charge the same account as any other, so backfill written // before the track is accepted settles its debt like the rest. Without one, the // provisional info claims no publisher max age, not an authored track's zero: the // publisher's real bound is unknown until its metadata arrives. if self.info.is_none() { self.install(info.unwrap_or_else(|| Info::default().with_max_age(None))); } let info = self.info.clone().unwrap(); if self.covering_range(key, frames).is_some() { return Ok(None); } let content = self.content_for_copy(key)?; // A copy cut short at the back never displaces a fetched one that reaches the end, // which whole-group readers still take. The fill hands it to its own fetch instead. let kept = frames.end.is_some() && self.lookup.get(&key).is_some_and(|slot| match slot.is_live() { true => slot .head .as_deref() .is_some_and(|head| !head.is_aborted() && !head.back_cut()), false => slot.serves_end(), }); if !kept && !self.lookup.get(&key).is_some_and(Slot::is_live) { self.displace(key); } let mut group = group::Producer::new(group::Info { start: key }, info, self.cache.clone()).with_meter(self.ingress.meter()); if let Some(content) = content { group.handle().share_content(content); } group.handle().register_content(&self.contents); // Start where the request did before the group is visible: a fetch looking it up // in between would otherwise see it begin at 0 and get a reader that later skips // the head it asked for. group.bound(frames)?; self.cap(&group.handle())?; // A backfill exists because someone is fetching it right now: stamp that // access so the eviction walk can't kill it before the fetch resolves. group.handle().cache_refresh(); // Settled first, like `commit_group`, which may evict the live slot itself. self.charge_debt(); let protected = self.protects(key); match self.lookup.get_mut(&key).filter(|slot| slot.is_live()) { _ if kept => self.detach(&group.handle()), Some(slot) => { // Joins the slot's standing: demoted already unless it is the live edge. if !protected { group.handle().cache_demote(); } // A replacing head inherits the drop the slot holds, announcing it if it dies. if slot.dropped.is_some() { group.handle().carry_drop(); } if let Some(head) = slot.head.replace(Box::new(group.handle())) { self.detach(&head); } // A head holding frame 0 gives a group received past it a start. self.fetched += 1; } // Invisible to arrival-order subscribers: fetched on demand, not produced // live by the publisher. None => self.insert_group(&group.handle(), false), } self.evict_expired(); Ok(Some(group)) } } impl TrackState { /// Store a flight a [`Fill`] answered: a copy of group `key` of its own, from the /// frame its riders start at, which the cache never serves in place of its cached copy. pub(super) fn insert_flight( &mut self, key: u64, frames: group::Bounds, info: Option, ) -> Result { if let Some(err) = self.outcome.aborted() { return Err(err.clone()); } if self.info.is_none() { self.install(info.unwrap_or_else(|| Info::default().with_max_age(None))); } let info = self.info.clone().unwrap(); let mut group = group::Producer::new(group::Info { start: key }, info.clone(), self.cache.clone()) .with_meter(self.ingress.meter()); // A compressed flight inflates from frame 0 with a decoder of its own, so it keeps // its own content rather than the cached copy's, whose inflated copy is further on. if !info.compressed() { if let Some(content) = self.content_for_copy(key)? { group.handle().share_content(content); } group.handle().register_content(&self.contents); } group.bound(frames)?; self.cap(&group.handle())?; self.charge_debt(); self.detach(&group.handle()); // Forget every flight gone since, so only flights still held are kept. self.flights.retain(|_, flights| { flights.retain(|flight| flight.held().is_some()); !flights.is_empty() }); self.flights.entry(key).or_default().push(group.handle().detach()); Ok(group) } /// The copy of its group `reader` rides at its frame, furthest along of those that admit /// it (see [`group::Handle::admits`]), and whether it is a flight rather than a cached /// copy. When `ahead`, only one further along than the copy it reads, which it hands /// over from. pub(super) fn ride(&self, reader: &group::Consumer, ahead: bool) -> Option<(group::Handle, bool)> { let key = reader.start; let cached = self .lookup .get(&key) .into_iter() .flat_map(Slot::copies) .map(|copy| (copy.clone(), false)); let flights = self .flights .get(&key) .into_iter() .flatten() .filter_map(|flight| Some((flight.held()?, true))); let past = ahead.then(|| reader.committed()); cached .chain(flights) .filter(|(copy, _)| !ahead || !copy.is_read_by(reader)) .map(|(copy, flight)| (copy.committed(), copy, flight)) .filter(|(committed, copy, _)| past.is_none_or(|past| *committed > past) && copy.admits(reader)) .max_by_key(|(committed, ..)| *committed) .map(|(_, copy, flight)| (copy, flight)) } } /// The copy of its group `reader` continues on from frame `index` once the copy it reads /// failed: one holding the group from there, once no re-offer of it is still held (see [`Hold`]). A /// group's name fixes its content, so any such copy continues it. pub(super) fn successor( track: &kio::Producer, reader: &group::Consumer, index: usize, waiter: &kio::Waiter, ) -> Poll> { let key = reader.start; // A copy cut short at the back still continues a reader capped no later than the cut. let frames = reader.unread(index); let found = track.poll_ref(waiter, |state| { // A hold on the copy read, or on any copy of the group that failed, promises a copy // that takes the group over, which continues this reader too. One on a copy still // arriving waits on that copy, which the reader takes instead. if state .holds .get(&key) .is_some_and(|held| held.iter().any(|copy| copy.is_read_by(reader) || copy.is_aborted())) { return Poll::Pending; } // Every copy, since one that fails or cannot serve the frame may sit before one that can. Poll::Ready( state .lookup .get(&key) .into_iter() .flat_map(Slot::copies) .filter(|copy| copy.covers(frames) && !copy.is_read_by(reader)) .filter_map(group::Handle::try_consume) .find(|copy| copy.continues(reader, index)), ) }); Poll::Ready(match found { Poll::Ready(Ok(copy)) => copy, Poll::Ready(Err(_closed)) => None, Poll::Pending => return Poll::Pending, }) } /// Where a reader at a frame its copy shed goes: see [`board`]. pub(super) enum Board { /// Ride this copy, a flight or a cached copy, which holds the frame. Ride(group::Consumer, bool), /// Wait for a flight the track's handlers were asked for. Wait(Boarding), } /// Board `reader`'s group at its frame, which the copy it read shed: ride a copy that /// admits it, else ask the track's [`Requests`] handlers for a flight from that frame, /// with the reader's priority and budget. /// `None` when no handler can serve one. /// /// A compressed group's flight starts at frame 0, since a later frame builds on the ones /// before it, and its riders skip to their frame. pub(super) fn board(track: &kio::Producer, reader: &group::Consumer) -> Option { let state = track.read(); let compress = state.info.as_ref()?.compressed(); if let Some((copy, flight)) = state.ride(reader, false) && let Some(copy) = copy.try_consume() { return Some(Board::Ride(copy, flight)); } let fills = state.fills.clone(); drop(state); // One group either way, so it never turns a wider fill it joins newest first. let asking = fill::Asking { order: Order::Ascending, ..reader.asking() }; let (key, frame) = (reader.start, if compress { 0 } else { reader.index() }); let join = fills.lock().fly(&fills, key, frame, asking)?; Some(Board::Wait(Boarding { join: Box::new(join), fills, asking, key, frame, })) } /// The copy `reader`, riding a flight, hands over to: one further along that admits it, /// and whether that is a flight. See [`TrackState::ride`]. pub(super) fn hand_over( track: &kio::Producer, reader: &group::Consumer, ) -> Option<(group::Consumer, bool)> { let (copy, flight) = track.read().ride(reader, true)?; Some((copy.try_consume()?, flight)) } /// A reader waiting on the flight it asked for. pub(super) struct Boarding { // Boxed: a reader holds one only while boarding. join: Box, // Where to ask again, with what, when a tighter budget than this reader's refused it. fills: kio::Shared, asking: fill::Asking, key: u64, frame: u64, } impl Boarding { /// The frame the flight starts at. pub(super) fn frame(&self) -> u64 { self.frame } /// Poll for a reader of the flight, or why none came: the handlers refused it, or went /// away without answering. pub(super) fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { let frames = group::Bounds { start: self.frame, end: None, }; let key = self.key; let answer = self.join.outcome().poll(waiter, |outcome| match outcome.refused(key) { Some((err, asked)) => Poll::Ready(Err((err.clone(), asked))), None if outcome.is_accepted(key) => Poll::Ready(outcome.partial(key, frames).ok_or((Error::Cancel, None))), None => Poll::Pending, }); // The closed channel's guard borrows the join, which a retry replaces. let (err, asked) = match ready!(answer.map(|answer| answer.map_err(drop))) { Ok(Ok(copy)) => return Poll::Ready(copy.try_consume().ok_or(Error::Cancel)), Ok(Err(refused)) => refused, Err(_closed) => return Poll::Ready(Err(Error::NotFound)), }; // Boarded a flight already asked under a tighter budget, which judged the group too // old for that budget, not ours: ask again with ours. Terminates, since the new // flight asks at least our budget. if !matches!(err, Error::Old) || !looser(asked, self.asking.max_delay) { return Poll::Ready(Err(err)); } let Some(join) = self.fills.lock().fly(&self.fills, key, self.frame, self.asking) else { return Poll::Ready(Err(err)); }; *self.join = join; self.poll(waiter) } } /// Whether `ours` waits longer than the budget `asked` upstream, `None` being the longest. fn looser(asked: Option, ours: Option) -> bool { match (asked, ours) { (Some(asked), Some(ours)) => ours > asked, (Some(_), None) => true, (None, _) => false, } } /// Demand wakes taken out of the track state, fired once its guard is gone. #[must_use = "fire the wakes once the guard is released"] pub(super) struct Deferred(Vec); impl Deferred { pub(super) fn wake(self) { for mut wakes in self.0 { wakes.wake(); } } } /// Record `err` and close the track, the shared tail of every abort, returning the open /// groups to abort once the guard is gone. fn commit_abort(mut state: kio::Mut<'_, TrackState>, err: Error, retry: Option) -> OpenGroups { // Decided before the open groups go: the groups below the end are the evidence. A gap // past the grace was lost before the abort rather than cut off by it. let now = state.cache.pool().now_wall(); state.fold_arrivals(now); state.latch_settled(); // Keep what finished for consumers still draining: they get it, then the abort (or // the clean end, when the end settled). Clearing here would abort finished groups a // slower reader has not pulled yet. The pool's idle expiry bounds how long they stay. let released = state.outcome.is_released(); let (resume, open) = state.tear_down(err.clone()); let end = match std::mem::take(&mut state.outcome) { Outcome::Ending(end) | Outcome::Sealed(end) => Some(end), _ => None, }; state.outcome = Outcome::Aborted { err, end, resume, retry, }; if !released { state.close_cache(); } state.close(); open } /// The unfinished groups a teardown took out of the track, aborted once the track lock /// is released. #[must_use = "abort the open groups after releasing the track lock"] struct OpenGroups { groups: Vec, err: Error, } impl OpenGroups { /// End each group with the teardown's error: its readers end and its writer fails. fn abort(self) { for group in self.groups { let _ = group.abort(self.err.clone()); } } } /// A reader's local group bounds: the floor, and the exclusive cap (`None` for none). #[derive(Clone, Copy, Debug, Default)] pub(super) struct Bounds { pub start: u64, pub end: Option, } impl Bounds { /// Whether group `key` is within these bounds. pub fn contains(&self, key: u64) -> bool { key >= self.start && super::subscription::before_end(key, self.end) } } /// Poll `cache` for a largest location within `bounds` newer than `seen`, recording it there. /// /// The largest is what the source reported and what its feed delivered since, never what /// the cache happens to hold, so `live` holds the stream until the track's feed is live: an /// idle copy's largest is only what an ended feed left. Past the cap or below the floor it /// reports nothing. `ended` polls the track for [`TrackState::largest_settled`], or its /// channel closing without ending the cache (a dropped request), either of which ends the /// stream. fn poll_largest( cache: &cache::Track, seen: &mut Option, bounds: Bounds, waiter: &kio::Waiter, live: impl FnOnce(&kio::Waiter) -> Poll<()>, ended: impl FnOnce(&kio::Waiter) -> Poll<()>, ) -> Poll> { ready!(live(waiter)); if let Poll::Ready(next) = cache.poll_largest(seen, bounds, waiter) { return Poll::Ready(next); } ready!(ended(waiter)); Poll::Ready(cache.take_largest(seen, bounds)) } /// The lowest key of `runs`, which ascend, that no hole `hole_end` reports covers. pub(super) fn first_open( runs: impl Iterator>, hole_end: impl Fn(u64) -> Option, ) -> Option { let mut floor = 0; for run in runs { let mut key = (*run.start()).max(floor); while key <= *run.end() { match hole_end(key) { Some(end) => key = end, None => return Some(key), } } floor = key; } None } /// Ready once the track's feed is live, or nothing more can move its largest. A declared /// end that is not reached yet does not count: an idle copy's largest is still only what /// its ended feed left. fn largest_live(state: &TrackState, waiter: &kio::Waiter) -> Poll<()> { match state.feed.is_live() || state.outcome.is_final() || state.largest_settled() { true => Poll::Ready(()), false => { state.watch_tail(waiter); Poll::Pending } } } /// Ready once the track's largest can no longer advance; see [`TrackState::largest_settled`]. fn largest_ended(state: &TrackState, waiter: &kio::Waiter) -> Poll<()> { match state.largest_settled() { true => Poll::Ready(()), false => { state.watch_tail(waiter); Poll::Pending } } } /// Where a received copy of a group goes. pub(crate) enum Received { /// A new group, for the copy to fill. Group(group::Producer), /// Another copy of the key that is still arriving: whether this one is needed /// depends on how that one settles. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] Held(Hold), } /// A received copy waiting on another copy of its key still arriving. While it waits, /// a reader of that copy that fails waits for this one to take the group instead of failing. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) struct Hold { arriving: group::Handle, track: kio::Weak, key: u64, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl Hold { /// The copy still arriving. pub(crate) fn arriving(&self) -> &group::Handle { &self.arriving } } impl Drop for Hold { fn drop(&mut self) { let Some(track) = self.track.upgrade() else { return; }; let Ok(mut state) = track.write() else { return; }; if let btree_map::Entry::Occupied(mut holds) = state.holds.entry(self.key) { if let Some(at) = holds.get().iter().position(|held| held.is_copy(&self.arriving)) { holds.get_mut().swap_remove(at); } if holds.get().is_empty() { holds.remove(); } } } } /// A producer for a track, used to create new groups. #[derive(Clone)] pub struct Producer { name: Arc, info: Info, // The parent broadcast's info, inherited from [`broadcast::Producer::create_track`]. // Top link of the ownership chain; carried for identity and future inheritance. broadcast: Arc, state: kio::Producer, prev_subscription: Option, // Shared with every clone and every `Requests`: its `Drop` is the teardown. alive: Arc, } impl Producer { /// The immutable publisher priority committed when this track was accepted. #[cfg(test)] #[cfg(feature = "async")] pub(crate) fn publisher_priority(&self) -> i8 { self.info.priority } /// Build a producer for the given track metadata. /// /// Crate-private: tracks are born from their broadcast via /// [`broadcast::Producer::create_track`] (or served on demand through a /// [`Request`]), which threads the broadcast's `Arc` down. The /// track opens its cache account against that broadcast's origin pool, and every /// group it creates charges into it. pub(crate) fn new( broadcast: Arc, name: impl Into>, info: impl Into>, ) -> Self { let name = name.into(); let info = info.into().unwrap_or_default(); let state = TrackState::spawn(broadcast.clone()); state.write().ok().expect("a new track is open").publish(info.clone()); let alive = Alive::new(name.clone(), state.clone()); alive.publish(None); Self { name, info, state, broadcast, prev_subscription: None, alive, } } /// Attach the parent broadcast's ingress stats scope, counting this track as one /// ingress subscription (closed when the last producer clone drops). Called by a /// tagged [`broadcast::Producer`] when it creates the track. pub(super) fn with_stats(self, scope: stats::Scope) -> Self { self.alive.publish(Some(&scope)); if let Ok(mut state) = self.state.write() { state.ingress = scope; } self } /// Keep the broadcast's listing of this track until it leaves the table. pub(super) fn listed(self, unlist: broadcast::Unlist) -> Self { self.alive.list(unlist); self } /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { &self.name } /// The parent broadcast this track belongs to. pub fn broadcast(&self) -> &broadcast::Info { &self.broadcast } /// The properties this track was created with. pub fn info(&self) -> &Info { &self.info } /// Create a new group starting at `group.start`, in the track's timescale. /// /// It says nothing about the starts around it: groups may be created out of order, and /// an aborted or evicted start created again. A gap it leaves is skipped only once the /// producer says it will not fill it, with [`Self::skip_groups`] or [`Self::append_group`]. /// A start already skipped is refused with [`Error::Skipped`], one held by another group /// with [`Error::Duplicate`], and the reserved start `u64::MAX` with /// [`Error::BoundsExceeded`]. /// /// Readers and group demand reach it at once, but the track counts it as content (its /// live edge, or the successor that ages the group before it) only once its first frame /// opens or it finishes. pub fn create_group(&self, group: group::Info) -> Result { match self.create_group_inner(group, None)? { Received::Group(group) => Ok(group), Received::Held(_) => unreachable!("only a received copy waits on another"), } } /// A received copy starts at its wire offset before any reader can see it. /// /// A copy that already answers from `start` holds the key: a finished one refuses /// this copy with [`Error::Duplicate`], and one still arriving is returned to wait on. #[cfg(feature = "async")] pub(crate) fn create_group_at(&self, group: group::Info, start: u64) -> Result { self.create_group_inner(group, Some(start)) } fn create_group_inner(&self, info: group::Info, received: Option) -> Result { // Reserved, so every group fits an exclusive range. if info.start == u64::MAX { return Err(coding::BoundsExceeded.into()); } let mut state = self.modify()?; if matches!(state.outcome, Outcome::Sealed(_)) || state.final_key().is_some_and(|fin| info.start >= fin) { return Err(Error::Closed); } // A skip says the key will never be produced, so it is final. A drop is not: a // publisher may re-create a group it cut or aborted. if state.skipped(info.start) { return Err(Error::Skipped); } let start = received.unwrap_or(0); let content = if received.is_some() { if let Some(holding) = state.holding_group(info.start, start) { if holding.is_settled() { return Err(Error::Duplicate); } let arriving = holding.clone(); state.holds.entry(info.start).or_default().push(arriving.clone()); return Ok(Received::Held(Hold { arriving, track: self.state.downgrade(), key: info.start, })); } state.content_for_copy(info.start)? } else { None }; let track = state.info.clone().unwrap(); state.claim_key(info.start, start)?; let mut group = group::Producer::new(info, track, state.cache.clone()).with_meter(state.ingress.meter()); if start != 0 { group.bound(group::Bounds { start, end: None })?; } if let Some(content) = content { group.handle().share_content(content); } state.cap(&group.handle())?; state.commit_group(&group.handle()); let deferred = state.deferred(); drop(state); deferred.wake(); Ok(Received::Group(group)) } /// Create the next group, starting at `start`: after every group this track holds. /// /// A timed track converts `start` into its timescale, as [`Self::write_frame`] does; a /// scale-less track takes [`Timestamp::unscaled`] at the group's key. A `start` that /// doesn't match the track, or that its scale can't hold, is [`Error::TimestampMismatch`]. /// /// Appending says the producer will not go back, so it skips every start below the new /// group that it never produced, from just past its previous append (before the first, /// from the lowest group it created), as [`Self::skip_groups`] would, replacing any drop /// there: consecutive appends name the span between their starts. Those starts are then /// final: creating one is refused. A `start` at or before a group this track holds is /// [`Error::TimestampMismatch`], like a rewound frame. /// /// The same broadcast, track, and group start always name the same content, and nothing /// checks it: the broadcast remembers nothing of an earlier producer of the name. A /// recreated track, or a restarted publisher keeping its instance, keeps its starts moving /// forward, as its media time does, or uses a new track name or instance. Reusing a start /// only corrupts this broadcast for its readers. Datagram sequences name no cached content /// and are exempt. pub fn append_group(&self, start: Timestamp) -> Result { let start = self.modify()?.info.as_ref().unwrap().rescale(start)?.value(); self.append_key(start) } /// [`Self::append_group`] at `start`, the group's key in the track's timescale. fn append_key(&self, start: u64) -> Result { let mut state = self.modify()?; // Reserved, so every group fits an exclusive range. if start == u64::MAX { return Err(coding::BoundsExceeded.into()); } if state.max_key.is_some_and(|max| start <= max) { return Err(Error::TimestampMismatch); } if let Some(fin) = state.final_key() && start >= fin { return Err(Error::Closed); } if state.skipped(start) { return Err(Error::Skipped); } let track = state.info.clone().unwrap(); let group = group::Producer::new(group::Info { start }, track, state.cache.clone()).with_meter(state.ingress.meter()); state.cap(&group.handle())?; state.commit_group(&group.handle()); state.appended(start); let deferred = state.deferred(); drop(state); deferred.wake(); Ok(group) } /// [`Self::append_group`] one past every group this track holds, or at 0 on an empty /// track: for tests that count groups rather than time them. #[cfg(any(test, feature = "fuzz"))] pub(crate) fn append_next(&self) -> Result { let start = self.modify()?.next_key(); self.append_key(start) } /// Append a group holding one frame at `timestamp`, which is where the group starts. /// /// The timestamp is converted into the track's timescale. On a scale-less track it is /// [`Timestamp::unscaled`] at the group's key. The group finishes after its frame, /// since nothing says where it ends (see [`group::Producer::finish`]). pub fn write_frame(&mut self, timestamp: Timestamp, frame: B) -> Result<()> { let frame = crate::IntoBytes::into_bytes(frame); if frame.len() as u64 > group::MAX_CACHE_BYTES { return Err(Error::FrameTooLarge); } // Rescaled before the group exists, so a refused frame leaves no empty group behind. let timestamp = self.modify()?.info.as_ref().unwrap().rescale(timestamp)?; let mut group = self.append_group(timestamp)?; if let Err(err) = group.write_frame(timestamp, frame) { // A rewind is only known once the group exists; end it rather than leave it open. let _ = group.abort(err.clone()); return Err(err); } group.finish()?; Ok(()) } /// End the track after its highest group, returning the [`Finished`] track: no group at /// or above that end can be created, but an older group below it still can. /// /// Other clones of this producer stay open handles, and refuse a group at or past the end /// with [`Error::Closed`]. Fails with [`Error::Closed`] if the track already declared an /// end, or with the abort error. pub fn finish(self) -> Result { self.finish_shared()?; Ok(Finished { producer: self }) } /// [`Self::finish`] through a shared handle, which a session keeps to take the lower /// groups still in flight. /// /// On a received copy whose clean end was declared at its live edge, this ends the wait /// for its tail instead: the session has every group stream it will get, so a key below /// the end that never arrived is not coming, as with sparse starts it never existed. pub(crate) fn finish_shared(&self) -> Result<()> { let mut state = self.modify()?; let end = Position::group(state.next_key()); if let Outcome::Ending(declared) = &state.outcome && declared.at == end && declared.error.is_none() && declared.tail.is_some() { let settled = declared.settled(); state.outcome = Outcome::Open; return state.declare(end, None, None, settled); } state.set_end(end, None) } /// End the track with `err` after its highest group, returning the [`Finished`] track: the /// errored counterpart of [`Self::finish`]. /// /// Unlike [`Self::abort`], the track keeps its history: readers and fills still get every /// group below the end, and a reader reaching the end gets `err` instead of the end of /// stream. Fails as [`Self::finish`] does. pub fn fail(self, err: Error) -> Result { let mut state = self.modify()?; let end = Position::group(state.next_key()); state.set_end(end, Some(err))?; drop(state); Ok(Finished { producer: self }) } /// Declare the track's exclusive end, possibly ahead of the live edge. /// /// `end` is the first position that will never be produced. [`Position::group`] /// ends before a group, so a track whose last group is 89 finishes at /// `Position::group(90)`. A mid-group end such as `{ group: 89, frame: 5 }` also /// caps group 89 at frames 0 to 4. Passing an end beyond the live edge records a /// known ending before the remaining groups arrive (e.g. learning a track ends at /// group 89 while only 87 has been received). /// /// Groups and frames below `end` may still be created afterwards; anything at or past /// it is rejected ([`Error::Closed`]). An end that would orphan a group or frame that /// already exists is [`Error::ProtocolViolation`]. Consumers only see end-of-stream /// once the live edge reaches the end's group. Use [`Self::finish`] to finish exactly /// at the live edge. /// /// Repeating the declared end is a no-op, and a mid-group end may refine the group /// boundary it falls just before (a peer may name only the boundary first). Any other /// second end is [`Error::ProtocolViolation`]. /// /// On a copy a session received, groups below the end may still be in flight: /// readers wait for a missing one until a grace passes, the subscription's max delay /// or one second without one, before ending at the boundary. /// /// The end is clean; [`Self::fail`] ends the track with an error. pub fn finish_at(&mut self, end: Position) -> Result<()> { self.end_at(end, None) } /// [`Self::finish_at`], ending with `error` when there is one, as [`Self::fail`] does: how /// a session forwards the end its publisher declared, which may lie ahead of the live edge. /// A repeated or refined end must carry the same error code. pub(crate) fn end_at(&mut self, end: Position, error: Option) -> Result<()> { let mut state = self.modify()?; let code = |err: Option<&Error>| err.map(|err| StreamError::from(err).to_code()); // A received copy's readers wait out a grace for groups still in flight below its end. let tail = state.anchors.received().then(|| state.cache.pool().now_wall()); match std::mem::take(&mut state.outcome) { Outcome::Open => state.declare(end, error, tail, false), Outcome::Ending(declared) if code(declared.error.as_ref()) != code(error.as_ref()) => { state.outcome = Outcome::Ending(declared); Err(Error::ProtocolViolation) } Outcome::Ending(declared) if declared.at == end => { state.outcome = Outcome::Ending(declared); Ok(()) } Outcome::Ending(declared) if declared.at.frame == 0 && end.frame != 0 && end.final_key() == Some(declared.at.group) => { let refined = state.declare(end, error, declared.tail.or(tail), declared.settled()); if refined.is_err() { state.outcome = Outcome::Ending(declared); } refined } outcome => { state.outcome = outcome; Err(Error::ProtocolViolation) } } } /// Declare the first group the live feed serves (the wire's SUBSCRIBE_START, /// or the start the subscription itself requested): groups below `key` /// are not waited for, so a reader waiting for one fails over instead of /// stalling. One may still arrive late, and a fetch can still retrieve them. /// /// Scoped to the current subscription's demand, so a later declaration /// replaces this one in either direction: a re-subscription may start /// earlier, and a narrowed subscription skips groups an earlier declaration /// still promised. Pass `None` to clear it, for demand at the live edge: /// its floor is unknown until the feed declares one. /// /// Resolves a start a subscriber session left pending. pub fn start_at(&mut self, key: impl Into>) -> Result<()> { let start = key.into().map_or(Start::Unknown, Start::Declared); self.modify()?.set_start(start); Ok(()) } /// [`Self::start_at`] a start the publisher resolved from its subscriber's budget. An /// update can cross the response, so that budget is unknown: every older group stays /// fillable, and a fill that finds none ends empty. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn start_resolved(&mut self, key: u64) -> Result<()> { self.modify()?.set_start(Start::Resolved(key)); Ok(()) } /// Mark this track as a copy received from a remote publisher, before writing to it. /// /// Its declared end waits for groups still in flight below it; see [`Self::finish_at`]. /// Its readers' relative and time anchors, other than [`Range::live`](crate::track::Range::live) /// and [`Range::oldest`](crate::track::Range::oldest), fail with [`Error::Unsupported`] /// unless its session's wire lets it resolve them, since it holds only part of the track. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn received(&self) { if let Ok(mut state) = self.modify() && state.anchors == Anchors::Local { state.anchors = Anchors::Refused; state.placing += 1; } } /// Let this received copy's readers resolve relative and time anchors, for a wire that /// reports the live edge and media time and asks for every group the budget allows. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn anchored(&self) { if let Ok(mut state) = self.modify() { state.anchors = Anchors::Wire; state.placing += 1; } } /// Have this received copy's readers ask no fills below the start its publisher resolved /// from the budget, for a wire whose fills and timestamps cannot bound them by it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn fills_from_start(&self) { if let Ok(mut state) = self.modify() { state.fills_from_start = true; } } /// Record where the upstream publisher started each of this lite-07 copy's live wire /// ranges, as `(time, start)` pairs by the time each asked from, replacing the last set. /// A reader's time start this copy cannot place among its own groups resolves to it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn time_starts(&self, starts: impl IntoIterator) { let starts: BTreeMap = starts.into_iter().collect(); if let Ok(mut state) = self.modify() && state.time_starts != starts { state.starts_past = starts .iter() .filter(|&(asked, start)| asked < start) .map(|(_, &start)| start) .max(); state.time_starts = starts; state.placing += 1; } } /// Record where the upstream publisher started each of this lite-07 copy's live wire /// ranges counting back, as `(back, start, latest)` triples, replacing the last set. A /// reader's relative anchor counting back `back` groups from `latest` resolves to `start`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn relative_starts(&self, starts: impl IntoIterator) { let starts: BTreeMap = starts .into_iter() .map(|(back, start, latest)| (back, RelativeStart { start, latest })) .collect(); if let Ok(mut state) = self.modify() && state.relative_starts != starts { state.relative_starts = starts; } } /// Where this copy's readers count a relative start back from now (see /// [`TrackState::latest_live`]). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn latest_live(&self) -> Option { self.state.read().latest_live() } /// Have this received copy's readers start a time anchor at the group starting there, /// held or not, for a wire whose publisher starts one at the first group at or after it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn starts_at_keys(&self) { if let Ok(mut state) = self.modify() { state.starts_at_keys = true; state.placing += 1; } } /// Record that the publisher gave up `groups` with `err`: none will arrive, so a reader /// waiting on one moves past it, a group still arriving in them ends with `err`, the /// declared end does not wait for them, and a relay serving this track forwards the drop. #[cfg(any(test, feature = "fuzz"))] pub(crate) fn drop_groups(&self, groups: ops::Range, err: StreamError) { self.give_up(groups, Hole::drop(err)); } /// Declare that this producer will never produce `groups`: none will arrive, so a reader /// waiting on one moves past it, the declared end does not wait for them, and a relay /// serving this track forwards the skip (lite-07 SUBSCRIBE_SKIP). Unlike a drop, it ends /// no reader. /// /// This and [`Self::append_group`] are the only sources of a skip: a gap between created /// groups says nothing, since groups may be created out of order, so only the producer /// knows a key will never exist. A skip is final: creating a group at a skipped /// key is refused with [`Error::Skipped`], and a later drop of it is ignored. /// /// A skip replaces any drop of the same keys, whatever its error: a reader the drop /// already ended stays ended, and every later reader and relay sees the skip. A group /// already cached keeps what it is, since it was produced after all. pub fn skip_groups(&self, groups: ops::Range) { self.skip_received(groups); } /// [`Self::skip_groups`], returning the cached keys it left alone: a skip naming a /// group that arrived contradicts it, which a subscriber reports. pub(crate) fn skip_received(&self, groups: ops::Range) -> RangeSet { let Ok(mut state) = self.modify() else { return RangeSet::new(); }; let cached = state.skip_groups(groups); state.charge_debt(); let deferred = state.deferred(); drop(state); deferred.wake(); cached } /// The parts of `groups` recorded skipped, never to be produced: a cost in the skipped /// runs it overlaps, not its length. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn skipped(&self, groups: ops::Range) -> RangeSet { if groups.is_empty() { return RangeSet::new(); } let state = self.state.read(); state .drops .overlapping(&groups) .filter(|(_, hole)| matches!(hole, Hole::Skip)) .map(|(run, _)| run.start.max(groups.start)..run.end.min(groups.end)) .collect() } /// Forget every run held as stale: a new upstream subscription judges them afresh, and /// may deliver them, so they are unknown again, and a reader that wants one fills it. /// Returns the runs forgotten; a group that arrived since its stale drop replaced it, so /// it is not among them. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn supersede_stale(&self) -> RangeSet { let Ok(mut state) = self.modify() else { return RangeSet::new(); }; state.supersede_stale() } /// Forget what was given up in `groups`, as the cache pool's expiry or debt does. #[cfg(all(test, feature = "async"))] pub(crate) fn forget_drops(&self, groups: ops::Range) { self.modify().unwrap().forget_drops(groups); } /// Record `groups` given up as `hole` describes, returning the runs recorded: a drop /// leaves a skipped key skipped, and a group a copy still serves whole, since a group /// stream wins over its drop: a stale drop leaves it still arriving or not, and a final one /// ends the live copy still arriving but leaves a group finished here or served by a /// fetched head. A head still filling holds that final drop until it finishes, which /// wins, or dies, which leaves the drop as the slot's cut. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn give_up(&self, groups: ops::Range, hole: Hole) -> RangeSet { let error = match &hole { Hole::Drop { error, .. } => error.clone(), Hole::Skip => { let mut runs = RangeSet::new(); runs.insert(groups.clone()); for cached in self.skip_received(groups).iter() { runs.remove(cached.clone()); } return runs; } }; if groups.is_empty() { return RangeSet::new(); } let (runs, open, carried, cache): (_, Vec, Vec, _) = match self.modify() { Ok(mut state) => { let stale = hole.stale_budget().is_some(); let mut left = RangeSet::new(); left.insert(groups.clone()); let mut open = Vec::new(); let mut carried = Vec::new(); for (&key, slot) in state.lookup.range_mut(groups) { if !slot.visible { continue; } // A final drop ends only the live feed's copy still arriving: a drop judges // the subscription, so a fetched head or fetch-only backfill may still complete. let arriving = !slot.group.is_aborted() && !slot.group.is_settled(); if !stale && arriving { open.push(slot.group.clone()); } // A lite-07 publisher may serve a group after its stale drop, and the group's // stream may land first. A group a copy still serves whole stays, whichever // copy serves it, as a re-sent copy the subscriber refused leaves it: naming // it given up would hide it from every reader. let ended = !stale && arriving && std::ptr::eq(slot.serving(), &slot.group); if !ended && slot.serves_end() && let Some(end) = key.checked_add(1) { left.remove(key..end); // A fetched head still filling may yet die, leaving the live copy this // drop ended: the slot keeps the drop to surface as its cut then, and // the head announces its death to cut watchers. A copy that finished // already won. if !stale && !slot.copies().any(group::Handle::is_settled) && let Some(head) = slot.head.as_deref() { slot.dropped = Some(Error::Stream(error.clone())); head.carry_drop(); carried.push(key); } } } let mut runs = RangeSet::new(); for run in left.iter() { runs.extend(state.drop_groups(run.clone(), hole.clone()).iter().cloned()); } state.charge_debt(); let cache = state.cache.clone(); (runs, open, carried, cache) } Err(_) => return RangeSet::new(), }; // Settle the readers of a group still arriving now, rather than once its stream's // reset lands, with the track guard released. for group in open { group.abort_open(Error::Stream(error.clone())); } // A head, or one replacing it, that died before the live copy above ended announced its // drop while that copy still served, so a cursor may have passed it by: announce every // drop the slot now shows as its cut. Pairs with the fence in a head's abort: a head // this misses announces after it, with the live copy ended. if !carried.is_empty() { std::sync::atomic::fence(std::sync::atomic::Ordering::SeqCst); let cut: Vec = { let state = self.state.read(); carried .into_iter() .filter(|key| state.lookup.get(key).and_then(Slot::cut).is_some()) .collect() }; for key in cut { cache.aborted(key, true); } } runs } /// Readers get nothing from the cache until [`Self::set_live`]: the upstream /// subscription ended with this copy still held, so what it cached may go stale. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn set_idle(&mut self) { if let Ok(mut state) = self.modify() { state.set_idle(); } } /// The route answered with its largest position, `None` for nothing yet, and the first /// group its feed serves when the answer names it; see [`Self::set_idle`]. Readers /// resume once the cache shows the largest position. A no-op unless idle. /// /// [`Error::MalformedTrack`] when the answer is behind the cache (no Largest while the /// copy shows groups, or a Largest before its newest position): the cache is likely /// another track, so the caller fails the copy and the next request builds a fresh one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn set_live(&mut self, largest: Option, start: Option) -> Result<()> { match self.modify() { Ok(mut state) => state.set_live(largest, start), Err(_) => Ok(()), } } /// [`Self::set_live`] for a feed that serves no group from `largest`'s on, such as a /// bounded subscription of history: readers resume at once, counting relative starts /// back from `largest`'s group. A no-op unless idle. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn set_live_ahead(&mut self, largest: Position, start: Option) -> Result<()> { match self.modify() { Ok(mut state) => state.set_live_ahead(largest, start), Err(_) => Ok(()), } } /// [`Self::set_live`] for an answer with no Largest, where `first`, the first group the /// feed sent, stands in for it. The cache is not judged against it. A no-op unless idle. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn set_live_from(&mut self, first: Position, start: Option) { if let Ok(mut state) = self.modify() { state.set_live_from(first, start); } } /// Whether a subscription feeds this copy: its session holds one upstream for it, or /// none, so nothing reaches it but what fills answer. Only the session knows, from its /// own state; the cache never tells. While unfed, a bounded read's holes are misses up /// to its end at once, and it reads the cache whether or not the copy is idle. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn set_unfed(&self, unfed: bool) { // Unchanged wakes nobody: every write wakes the readers. if self.state.read().unfed == unfed { return; } if let Ok(mut state) = self.modify() { state.unfed = unfed; } } /// The group of the newest Largest Object the upstream publisher's answers reported, /// kept however far the cache has moved past it: every group below it is history there. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn reported_group(&self) -> Option { self.state.read().object_edge.map(|((group, _), _)| group) } /// The live feed's route answered that it reaches `group`: a relative start counts back /// from it, rather than from whichever group lands first. A no-op unless live. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn report_edge(&self, group: u64) { if let Ok(mut state) = self.modify() && let Feed::Live { first, edge } = &mut state.feed { *edge = (*edge).max(Some(group)); first.get_or_insert(group); } } /// Record the upstream Object ID separately from the frame ordinal readers count. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn report_object(&self, group: u64, object: u64) { if let Ok(mut state) = self.modify() { // A frame received past the report outdates it on arrival. A frame's ordinal never // exceeds its Object ID, so one past the reported object proves the report stale. if state .cache .largest() .is_some_and(|largest| largest > Position { group, frame: object }) { return; } let edge = state.object_edge().max(Some((group, object))); state.cache.advance_largest(Position::group(group)); let mark = state.cache.largest().unwrap_or(Position::group(group)); state.object_edge = edge.map(|edge| (edge, mark)); } } /// Resolve relative anchors other than [`Range::live`](crate::track::Range::live)'s /// against this copy before sending a wire that can name only groups: a start becomes /// the group it resolved to, and an end the last group it includes. Fails with /// [`Error::Unsupported`] while one waits on a latest group. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn resolve_subscription(&self, subscription: &Subscription) -> Result { use super::subscription::{Anchor, Range}; let relative: Vec = (0..subscription.ranges.len()) .filter(|index| !subscription.ranges[*index].is_wire()) .collect(); if relative.is_empty() { return Ok(subscription.clone()); } let ranges: Vec = relative.iter().map(|index| subscription.ranges[*index]).collect(); let state = self.state.read(); let window = Window::new(&ranges, None, state.latest_live(), &*state)?; if !window.is_settled() { return Err(Error::Unsupported); } let mut subscription = subscription.clone(); for (index, resolved) in relative.into_iter().zip(window.resolved()) { let resolved = resolved.ok_or(Error::Unsupported)?; let range = &mut subscription.ranges[index]; if matches!(range.start, Anchor::Relative(back) if back != 0) { range.start = Anchor::Time(resolved.start); } if matches!(range.end, Some(Anchor::Relative(_))) { *range = match resolved.end.map(|end| end.checked_sub(1)) { None => Range { end: None, ..*range }, Some(Some(last)) => Range { end: Some(Anchor::Time(last)), ..*range }, Some(None) => Range::EMPTY, }; } } Ok(subscription) } /// The source reported its largest frame, which [`Self::largest`] follows before the /// frame itself arrives, and whether its group ended there, which a relay forwards. A /// hint about the edge: it never makes the frame readable. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn report_largest(&self, largest: Position, finished: bool) { self.state.read().cache.report_largest(largest, finished); } /// Whether readers may take from the cache; see [`Self::set_idle`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn is_live(&self) -> bool { self.state.read().feed.is_live() } /// While idle, the newest group cached when the track went idle: a route asked from /// its head sends it again, and its first frame says whether the cache is current. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn idle_newest(&self) -> Option { let state = self.state.read(); match state.feed { Feed::Idle { newest, .. } => newest, _ => None, } } /// [`Request::request_start`] for a copy already accepted, which a held copy's next /// subscription asks for again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn request_start(&mut self, key: Option) -> Result<()> { self.modify()?.set_start(Start::Pending(key)); Ok(()) } /// The declared first key of the live feed, once [`Self::start_at`] /// declared one. `None` while nothing was declared. #[cfg(test)] #[cfg(feature = "async")] pub(crate) fn start_key(&self) -> Option { self.state.read().start.key() } /// Evict group `key` from the cache, as expiry does. #[cfg(all(test, feature = "async"))] pub(crate) fn evict(&self, key: u64) { self.modify().unwrap().evict_group(key, Error::Evicted); } /// The exclusive end, once [`Self::finish`] or [`Self::finish_at`] declared one. /// /// `None` while the track is still open ended. Both methods reject a second end, so /// callers that may have already declared one check here first. pub fn finished(&self) -> Option { self.state.read().declared() } /// Abort the track with the given error. /// /// Consumes the handle, since nothing can be written to an aborted track. Consumers /// still draining get the finished groups, then the abort error. Open groups, fetched /// copies included, are aborted with the same error and leave the cache, so a consumer /// that already pulled one ends with it too, and their writers fail. The pool's idle expiry (see /// [`cache::Config::with_expiry`]) reclaims the rest, the latest group included, so a /// stale [`Consumer`] can't pin them in memory forever. /// /// If the declared end had settled (the final key from /// [`finish_at`](Self::finish_at) was reached and every group below it finished), /// consumers get that end instead of the error, as after [`finish`](Self::finish). /// /// [`finish`](Self::finish) is deliberately not terminal: it declares the final /// key, and lower-numbered groups may still be written afterwards. pub fn abort(self, err: Error) -> Result<()> { // Unlisted first, so the track leaves the listing before its name is free again. self.alive.unlist(false); commit_abort(self.modify()?, err, None).abort(); Ok(()) } /// Abort the track as an upstream refusal that asked to be tried again after `retry`, so /// a relay refuses the readers still waiting with that interval. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn refuse(self, err: Error, retry: Duration) -> Result<()> { self.alive.unlist(false); commit_abort(self.modify()?, err, Some(retry)).abort(); Ok(()) } /// Abort an unwanted track, returning the producer unchanged if consumers remain. /// /// This is how an on-demand publisher ends a track once [`Demand::poll_wanted_changed`] /// reports nobody wants it. Dropping the producer instead races a reader joining in /// between, which then reads [`Error::Dropped`]. /// /// Consumer creation and the unwanted check share a lock, so demand returning /// after [`Demand::wanted`] went to `None` prevents the abort. `Ok(())` means /// the track is closed, including when it was already closed; existing handles /// may still observe its final state, and a later lookup requests the track afresh. /// `Err(producer)` leaves the track unchanged so the caller can continue serving and /// wait for `None` on a fresh clone of its [`Demand`]: one that already returned `None` /// misses a reader that came and went, and waits on. /// /// The abort is the same as [`Self::abort`]. A reader still in a group it was handed /// counts as a consumer until it reads the group's end. #[cfg_attr( target_pointer_width = "64", expect( clippy::result_large_err, reason = "return the owned producer without allocating on an idle check" ) )] pub fn abort_unwanted(self, err: Error) -> std::result::Result<(), Self> { let wanted = match self.state.write_unwanted() { kio::Unwanted::Idle(guard) => { commit_abort(guard, err, None).abort(); false } kio::Unwanted::Closed => false, kio::Unwanted::Wanted => true, }; if wanted { return Err(self); } // After the track's own lock is released; a later track of the name, listed in // between, keeps its listing. self.alive.unlist(false); Ok(()) } /// Resolves once the track's outcome is final: `Ok` once its declared end is reached /// (its error, if declared with one), or the abort error, or [`Error::Dropped`] once every /// producer left without either. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { TrackState::settle(self.state.poll_ref(waiter, |state| state.poll_outcome(waiter))) } /// Poll until the track is aborted, or every producer left; ready with the cause. // Sessions stop serving on an abort, but keep taking lower groups past a finish. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_aborted(&self, waiter: &kio::Waiter) -> Poll { TrackState::poll_aborted(&self.state, waiter) } /// Return true if the track has been closed. pub fn is_closed(&self) -> bool { self.state.read().is_closed() } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn latest_key(&self) -> Option { self.state.read().max_key } /// The track's largest location: its newest group at the newest frame (0 while empty), `None` before any. pub fn largest(&self) -> Option { self.state.read().cache.largest() } /// Ready once the track holds content past `key`, or is closed: a later group whose /// first frame opened or that finished. A frameless group header /// does not count, since nothing says the copy reached it yet. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_past(&self, key: u64, waiter: &kio::Waiter) -> Poll<()> { let Some(next) = key.checked_add(1) else { return Poll::Pending; }; let past = self.state.poll_ref(waiter, |state| { if state.shows(next) { return Poll::Ready(()); } // A group starting wakes no reader parked on the track, so wait on each one past // it that has yet to start. let started = state .lookup .range(next..) .filter(|(_, slot)| slot.is_live()) .any(|(_, slot)| slot.group.poll_started(waiter).is_ready() && !slot.group.is_aborted()); match started { true => Poll::Ready(()), false => Poll::Pending, } }); past.map(|_| ()) } /// Return true if this is the same track. pub fn is_clone(&self, other: &Self) -> bool { self.state.same_channel(&other.state) } /// Create a weak reference that doesn't prevent auto-close. pub(super) fn weak(&self) -> TrackWeak { TrackWeak { name: self.name.clone(), state: self.state.weak(), } } /// A handle that aborts the track without keeping it open, for an owner that hands this /// producer out and must end the track on its own shutdown. pub fn abort_handle(&self) -> AbortHandle { AbortHandle(Arc::downgrade(&self.alive)) } /// Create a [`Demand`]: a cloneable, watch-only handle to this track's /// subscriber demand. /// /// Lets a publisher gate work (e.g. on-demand capture) on whether anyone is /// subscribed, without the ability to publish frames or close the track. The /// handle is weak, so holding one neither keeps the track alive nor pins its /// cached groups. pub fn demand(&self) -> Demand { Demand::new(self.name.clone(), self.state.weak()) } /// Get a consumer handle for this in-process track. /// /// Unlike a wire subscription, the info is already known, so a subscription /// opened from this handle resolves immediately. pub fn consume(&self) -> Consumer { Consumer::new(self.name.clone(), self.state.weak().consume_weak()) } /// Subscribing to this in-process track, resolving synchronously. /// /// The info is fixed at creation, so there's nothing to wait for (no /// SUBSCRIBE_OK round trip). Pass `None` for [`Subscription::default`]. /// /// Reads only the groups the subscription's ranges cover. /// [`Delay::max`] is what asks for data: delivery skips everything in range /// that the budget convicts, so the default budget of zero delivers only the latest /// group and a larger one reaches back over what it can still use. pub fn subscribe(&self, subscription: impl Into>) -> Subscriber { let preferences = subscription.into().unwrap_or_default(); // Info is fixed at creation and survives a close/abort, so read it without // requiring a live producer state. If the track already ended, the returned // subscriber surfaces the close/abort on its first read; the preferences are // simply never registered (nothing aggregates them anymore). let info = self.info.clone(); let subscription = Preferences::register(self.state.read(), preferences); // Hoisted: an inline `read()` guard would live to the end of the struct literal, // deadlocking against the `consume()` below. let broadcast = self.state.read().broadcast.clone(); Subscriber { name: self.name.clone(), broadcast, info, inner: Inner::Plain(Cursor::new(self.state.consume(), subscription)), // A producer-side (in-process) subscribe is not egress: stay untagged. stats: stats::Scope::default(), _stats_sub: stats::Subscription::default(), largest: Reported::default(), parks: false, } } /// A non-blocking snapshot of the current aggregate subscription, or `None` /// when there are no live subscribers. Unlike [`Self::subscription`], this /// doesn't wait for a change or advance the change cursor. /// /// The aggregate's [`Delay::max`] is the subscribers' own, capped only by /// the local cache ceiling ([`broadcast::Info::cache_duration`]). The publisher /// enforces its own [`Info::max_age`], which each reader's lateness budget also /// applies, so the demand asked upstream is the same before and after the info lands. pub fn subscription(&self) -> Option { let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); snapshot_subscription(&subs, bound) } /// Arm `waiter` for the next change to the subscribers' demand, without reading it: a /// caller that follows each subscriber's own edits needs the wake, not the aggregate, /// whose ranges cost every subscriber's to build. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn watch_subscription(&self, waiter: &kio::Waiter) { watch_combined(&self.state, waiter); } /// The aggregate subscription once it changes: the most demanding request across all /// live subscribers, or `None` once the last one drops. Errors once the track is aborted. /// /// Used by relays to forward downstream demand upstream (e.g. SUBSCRIBE_UPDATE). #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_subscription_changed(&mut self, waiter: &kio::Waiter) -> Poll>> { // Surface an abort as the stream ending. `poll_closed` parks on the closed // waiters, so per-group churn on the track state never wakes this poll. if self.state.poll_closed(waiter).is_ready() { let abort = self.state.read().outcome.aborted().cloned(); return Poll::Ready(Err(abort.unwrap_or(Error::Dropped))); } // Read the bound before locking `subs`, so the aggregation never nests the two locks. let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); poll_combined_changed(&subs, bound, &mut self.prev_subscription, waiter).map(Ok) } /// Count `seen` as the aggregate already acted on, so /// [`Self::poll_subscription_changed`] reports only a change from it: the demand a /// subscriber session asked upstream for before accepting the track, or `None` /// after abandoning that setup. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn seen_subscription(&mut self, seen: Option) { self.prev_subscription = seen; } /// How long a subscriber session's tail waits for a late group: `budget`, bounded by how /// long the publisher keeps one. The aggregate asked upstream is not bounded by it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn tail_grace(&self, budget: Duration) -> Duration { ledger::grace(clamp_max_delay(budget, self.state.read().max_age_bound())) } /// An empty bill against this track's cache account, for a subscription's tail ledger. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn tail_bill(&self) -> cache::Reader { self.state.read().cache.tail() } /// Create a [`Requests`] handle that serves on-demand fetches of uncached /// (old) groups. Most producers never need this; a relay creates one to fetch /// past groups from upstream. pub fn requests(&self) -> Requests { Requests::new(self.name.clone(), self.state.clone(), self.alive.clone()) } fn modify(&self) -> Result> { TrackState::modify(&self.state) } } /// A track whose producer [finished](Producer::finish) it: its end is declared, and it can /// still take or serve an older group below that end, be read, or be aborted, but not append. /// /// Dropping it releases this handle like dropping a [`Producer`]: once no clone is left, the /// track leaves its broadcast's listing as finished, so a new reader can no longer find it. /// /// ```compile_fail,E0599 /// # fn cannot_append(track: moq_net::track::Finished) { /// let _ = track.append_next(); /// # } /// ``` #[must_use = "dropping the last handle releases the track, which new readers can then no longer find"] pub struct Finished { producer: Producer, } impl Finished { /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { self.producer.name() } /// The parent broadcast this track belongs to. pub fn broadcast(&self) -> &broadcast::Info { self.producer.broadcast() } /// The properties this track was created with. pub fn info(&self) -> &Info { &self.producer.info } /// Serve on-demand fetches of uncached groups below the end; see [`Producer::requests`]. pub fn requests(&self) -> Requests { self.producer.requests() } /// Create an older group below the end, which arrived late; see [`Producer::create_group`]. /// /// A key at or past the end is refused with [`Error::Closed`]. pub fn create_group(&self, group: group::Info) -> Result { self.producer.create_group(group) } /// Abort the track with the given error; see [`Producer::abort`]. /// /// Once every group below the end finished, readers still get the end the track declared: the /// clean end after [`Producer::finish`], or the error [`Producer::fail`] gave. pub fn abort(self, err: Error) -> Result<()> { self.producer.abort(err) } /// Resolves once the track's outcome is final; see [`Producer::poll_closed`]. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { self.producer.poll_closed(waiter) } /// Get a consumer handle for this in-process track. pub fn consume(&self) -> Consumer { self.producer.consume() } /// Subscribe to this in-process track; see [`Producer::subscribe`]. pub fn subscribe(&self, subscription: impl Into>) -> Subscriber { self.producer.subscribe(subscription) } } /// Serves the groups a track's readers miss, the group-level analogue of /// [`broadcast::Requests`]. /// /// Most tracks never serve old content, so this capability lives on a dedicated /// handle rather than [`Producer`]: a relay creates one (via /// [`Producer::requests`] or [`Request::requests`]) to pull past groups /// from upstream. While at least one is alive a reader's miss queues a [`Fill`] for it /// to serve; with none, the miss is named dropped, and a cache-miss /// [`Consumer::fetch_group`] on an accepted track fails fast with [`Error::NotFound`]. /// Keep taking fills until it drops: a reader waits on its [`FillLookup`] until it is /// answered or dropped, so a handle left alive but unread stalls that reader. /// /// A live handle also lets a reader at a frame a group copy shed replay it: a [`FillGroups`] of /// that one group from that frame asks for a flight, a copy of its own paced by the readers /// riding it, which they leave for the copy ahead once it holds their frame within reach. /// Serve it like any group, and stop once nobody reads it: see [`FillGroups::is_wanted`]. pub struct Requests { name: Arc, // Kept to insert served groups into the cache and observe track abort. state: kio::Producer, // The fills this handle drains; its handler count gates queueing. fills: kio::Shared, // Shared with the track's producers: a handler still serving fills keeps the // track alive, like a producer clone does. alive: Arc, // Takes only fetches; see [`Self::fetches_only`]. fetches_only: bool, } impl Requests { fn new(name: Arc, state: kio::Producer, alive: Arc) -> Self { let fills = state.read().fills.clone(); fills.lock().add_handler(); Self { name, state, fills, alive, fetches_only: false, } } /// Take only [`Consumer::fetch_group`]'s fills from now on, as a session does for a /// copy whose subscription ended: a reader's miss settles at once, as with no handler, /// while a fetch still goes upstream. Queued misses are dropped; taken fills stay. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn fetches_only(mut self) -> Self { if !self.fetches_only { self.fills.lock().fetches_only(); self.fetches_only = true; } self } /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { &self.name } /// The next [`Fill`] readers ask: a run of groups they miss, to serve via /// [`FillGroups::accept`] and [`FillGroups::drop_groups`], or a time to look up. /// /// A relay fetches the run upstream; an origin serves the groups its cache evicted /// from its own store, as `moq-archive` does. Errors once the track is aborted. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_requested_fill(&self, waiter: &kio::Waiter) -> Poll> { fill::poll_requested(&self.state, &self.fills, waiter) } /// [`Self::poll_requested_fill`] for a handler that can't resolve times: each lookup is /// dropped unanswered, so its reader falls back to the groups held. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] #[cfg_attr(feature = "async", kio::wrapper)] pub(crate) fn poll_requested_groups(&self, waiter: &kio::Waiter) -> Poll> { loop { if let Fill::Groups(fill) = ready!(self.poll_requested_fill(waiter))? { return Poll::Ready(Ok(fill)); } } } /// The next fill, which a test expects to name a single group. #[cfg(test)] #[cfg_attr(feature = "async", kio::wrapper)] pub(crate) fn poll_requested_fill_group(&self, waiter: &kio::Waiter) -> Poll> { self.poll_requested_groups(waiter).map_ok(|fill| { assert_eq!(fill.end() - fill.start().group, 1, "a one-group fill"); fill.group(fill.start().group) }) } /// Watch subscriber demand without keeping the track alive. pub fn demand(&self) -> Demand { Demand::new(self.name.clone(), self.state.weak()) } } impl Clone for Requests { fn clone(&self) -> Self { // Count each live handle (mirrors `broadcast::Requests`). match self.fetches_only { true => self.fills.lock().add_fetcher(), false => self.fills.lock().add_handler(), } Self { name: self.name.clone(), state: self.state.clone(), fills: self.fills.clone(), alive: self.alive.clone(), fetches_only: self.fetches_only, } } } impl Drop for Requests { fn drop(&mut self) { // Unlike `broadcast::Requests`, dropping the last handle doesn't abort the track: // a live `Producer` may still be serving the subscription. It just stops serving // fills: a queued one no handler will ever take closes, so its readers stop // waiting on it, while one already taken stays with its `Fill`. match self.fetches_only { true => self.fills.lock().remove_fetcher(), false => self.fills.lock().remove_handler(), } } } /// Ends the track when the last [`Producer`] or [`Requests`] drops. /// /// A refcount rather than a "am I the last one?" check inside `Drop`: that answer is a /// snapshot, and acting on it is exactly what invalidates it. The track state's own /// producer count can't answer it either, since a group settling its eviction debt /// upgrades the account's weak handle and counts there for the duration (see /// [`cache::Track::settle`]). Holding a producer of its own also keeps the state /// writable until the teardown has run, whatever order the last owner's fields drop in. struct Alive { name: Arc, state: kio::Producer, // Set when a `Producer` is first minted, so a `Request` nobody accepted (its // `Requests` holds this guard too) isn't reported as an abandoned publisher. published: AtomicBool, // Ingress subscription for this track, opened by the tagged producer that claimed // it and closed when this guard drops. stats: OnceLock, // The broadcast's listing of a track its producer created, withdrawn once the track // leaves the table: on abort, or when this guard drops. listed: kio::Lock>, } impl Alive { fn new(name: Arc, state: kio::Producer) -> Arc { Arc::new(Self { name, state, published: Default::default(), stats: Default::default(), listed: kio::Lock::new(None), }) } fn list(&self, unlist: broadcast::Unlist) { *self.listed.lock() = Some(unlist); } /// Withdraw the track from its broadcast's listing, as one that `finished` or left /// unfinished. Never under the broadcast's lock. fn unlist(&self, finished: bool) { let unlist = self.listed.lock().take(); if let Some(unlist) = unlist { unlist.unlist(finished); } } /// Note that a [`Producer`] was minted from this track, optionally under a tagged /// broadcast's ingress scope (counted as one subscription for as long as the track /// has a publisher). fn publish(&self, stats: Option<&stats::Scope>) { self.published.store(true, Ordering::Relaxed); if let Some(scope) = stats { // At most one scope ever arrives: a track is minted either through // `Producer::new` (+ `with_stats`) or through `Request::accept`, never both. let _ = self.stats.set(scope.subscribe()); } } } impl Drop for Alive { fn drop(&mut self) { // Withdrawn before the state closes, so the track leaves the listing before a // later one can take its name. It finished if its producer declared its end. let finished = self.state.read().outcome.end().is_some(); self.unlist(finished); if let Ok(mut state) = self.state.write() { state.release_ingress(); } // A request nobody accepted was never publishing; there's nothing to tear down. if !self.published.load(Ordering::Relaxed) { return; } // The last producer going away ends the track: nothing protects its live edge // anymore, so the pool's idle expiry reclaims what a stale consumer would pin. // Without a finish it is an abrupt teardown, the same as an explicit abort: // the open groups go and the finished ones stay for readers still draining. // `abort()` closes the channel, so `write()` returns `Err(Ref)`. `finish()` // leaves it open with `end` set, so inspect both outcomes. match self.state.write() { Ok(mut state) => match std::mem::take(&mut state.outcome) { Outcome::Open => { tracing::warn!( track = %self.name, "track::Producer dropped without finish() or abort()" ); let (resume, open) = state.tear_down(Error::Dropped); state.outcome = Outcome::Dropped { resume }; state.close_cache(); drop(state); open.abort(); } // Groups still missing below the boundary can no longer arrive, so a reader // waiting on one ends cleanly instead of with `Dropped`. Outcome::Ending(end) => { state.outcome = Outcome::Sealed(end); state.close_cache(); } outcome => state.outcome = outcome, }, Err(state) => { if !matches!(state.outcome, Outcome::Open) { return; } tracing::warn!( track = %self.name, "track::Producer dropped without finish() or abort()" ); } } } } /// Poll the counted aggregate without registering on individual readers. fn poll_combined_changed( subs: &kio::Lock, bound: Option, prev: &mut Option, waiter: &kio::Waiter, ) -> Poll> { subs.lock().poll_changed(bound, prev, waiter) } /// Arm `waiter` for the next change to the counted aggregate, reading none of it. fn watch_combined(state: &kio::Producer, waiter: &kio::Waiter) { let subs = state.read().subscriptions.clone(); subs.lock().watch(waiter); } /// Read the counted aggregate without walking the audience. fn snapshot_subscription(subs: &kio::Lock, bound: Option) -> Option { clamp_combined(subs.lock().combined(), bound) } /// Clamp a drift budget to a max age: nobody can wait for a late group longer than it /// stays fresh. /// /// The single clamp point. Subscribers hold their preferences verbatim, so what they asked /// for stays readable, and it is applied here on both sides of the aggregate: to the /// combined request the publisher sees ([`clamp_combined`]), with the local cache ceiling /// ([`TrackState::demand_bound`]), and to one subscriber's own budget when it decides a /// group is stale ([`TrackState::is_stale`]), with the publisher's window too /// ([`TrackState::max_age_bound`]). The publisher applies its own window to the request, /// and `min` distributes over the `max` that combines the subscribers, so a group is stale /// under the request exactly when it is stale for one of them. `bound` is `None` when /// nothing imposes a window. fn clamp_max_delay(mut max_delay: Duration, bound: Option) -> Duration { if let Some(bound) = bound { max_delay = max_delay.min(bound); } max_delay } /// Clamp the aggregate's max delay budget to `bound`; see [`clamp_max_delay`]. A bound that /// doesn't lower the budget leaves it as is, so an unbounded demand stays `None` on the wire /// rather than becoming a huge value no version can carry. fn clamp_combined(combined: Option, bound: Option) -> Option { let mut combined = combined?; if let Some(bound) = bound && bound < combined.budget() { combined.delay.max = Some(bound); } Some(combined) } /// A front's logical track served from one route's copy, for benches. #[cfg(any(feature = "fuzz", all(test, feature = "async")))] pub(crate) struct Routed { /// The logical track readers subscribe to. pub logical: Producer, /// The route's copy, which its source writes. pub copy: Producer, /// Concludes the logical track once dropped. _routes: super::resume::Producer, } #[cfg(any(feature = "fuzz", all(test, feature = "async")))] impl Routed { /// A logical track named `name`, and a copy of it serving it, both with `info`. pub(crate) fn new(name: &str, info: Info) -> Self { let broadcast = Arc::new(broadcast::Info::default()); let routes = super::resume::Producer::new(); let logical = Request::new(broadcast.clone(), name) .routes(routes.consume()) .accept(info.clone()); let copy = Producer::new(broadcast, name, info); routes.serve(copy.consume()); Self { logical, copy, _routes: routes, } } } /// Aborts a track without keeping it open, from [`Producer::abort_handle`]. /// /// Holding one neither keeps the track alive nor counts as its producer, so a track whose /// producers all dropped closes as usual and the handle then does nothing. #[derive(Clone)] pub struct AbortHandle(std::sync::Weak); impl AbortHandle { /// Whether the track closed: aborted, or every producer gone. pub fn is_closed(&self) -> bool { self.0.upgrade().is_none_or(|alive| alive.state.is_closed()) } /// Abort the track with `err`, as [`Producer::abort`] does, unless it already closed. pub fn abort(&self, err: Error) { let Some(alive) = self.0.upgrade() else { return; }; // Unlisted first, as `Producer::abort` does, so the name is free only once it left. alive.unlist(false); let Ok(state) = alive.state.write() else { return; }; commit_abort(state, err, None).abort(); } } /// A weak reference to a track that doesn't prevent auto-close. #[derive(Clone)] pub(super) struct TrackWeak { name: Arc, state: kio::ProducerWeak, } impl TrackWeak { /// Join the cached track's demand, or `None` once it has closed. /// /// The count moves under the same lock the close takes, which is the other half /// of [`Producer::abort_unwanted`]: a request either joins in time to decline the /// teardown, or gets nothing and re-requests the track. It is never handed a track /// that is already on its way out. pub(super) fn try_consume( &self, subscription: Option<&Subscription>, ) -> Option<(kio::Consumer, Option)> { // Register before the first consumer can wake the serving session, while // still refusing a copy whose idle teardown already committed. let preferences = subscription.map(|value| Preferences::register(self.state.read(), value.clone())); Some((self.state.try_consume()?, preferences)) } /// Whether a producer serves the track, rather than a request still waiting for one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn minted(&self) -> bool { self.state.read().minted() } /// Read the cached track without joining its demand. pub(super) fn peek(&self) -> kio::ConsumerWeak { self.state.consume_weak() } /// The shared name handle, for use as a broadcast lookup key (clone is a /// refcount bump, and the same `Arc` is shared with the track's handles). pub(super) fn name(&self) -> &Arc { &self.name } /// Reject a track nothing ever served, resolving its pending subscribes with `err`. /// /// A track whose [`Producer`] was minted is left alone and this returns false; /// so is one a handler claimed, and one that already carries an abort reason. Fetched backfill can install /// [`Info`] before acceptance, so metadata alone does not prove a publisher exists. /// /// Closes the state like [`Producer::abort`], so a [`Request`] still held by the /// publisher can't `accept` its way back to life afterwards. pub(super) fn reject(&self, err: Error) -> bool { let Some(producer) = self.state.produce() else { return false; }; let Ok(mut state) = producer.write() else { return false; }; if state.publication != Publication::Pending || state.outcome.aborted().is_some() { return false; } state.reject(err, None); state.close(); true } /// Whether its producer declared the track's end, which is how its listing ends too. pub(super) fn is_finished(&self) -> bool { self.state.read().outcome.end().is_some() } /// Whether anyone is consuming the track right now. A closed track doesn't /// count even if consumers linger to drain its cache: no new work is owed. pub(super) fn is_wanted(&self) -> bool { !self.state.is_closed() && self.state.wanted() } /// Whether any handle still reads the track, closed or not, such as a fetch of a finished /// track still reading its group. pub(super) fn is_held(&self) -> bool { self.state.wanted() } /// A [`Demand`] on the cached track, for a broadcast to watch. pub(super) fn demand(&self) -> Demand { Demand::new(self.name.clone(), self.state.clone()) } /// [`Self::subscription`] without its ranges, costing none of them. pub(super) fn terms(&self) -> Option<(Subscription, usize)> { let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); let subs = subs.lock(); Some((clamp_combined(subs.terms(), bound)?, subs.len())) } /// The readers' aggregate demand and contribution count, or `None` without one. pub(super) fn subscription(&self) -> Option<(Subscription, usize)> { let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); let subs = subs.lock(); Some((clamp_combined(subs.combined(), bound)?, subs.len())) } /// End the track with `err` unless a reader holds it, atomically with lookups: a /// lookup either gets a consumer in time to keep it, or finds it closed and asks /// afresh. True once the track is gone. pub(super) fn abort_unwanted(&self, err: Error) -> bool { let Some(producer) = self.state.produce() else { return true; }; match producer.write_unwanted() { kio::Unwanted::Idle(guard) => { commit_abort(guard, err, None).abort(); true } kio::Unwanted::Closed => true, kio::Unwanted::Wanted => false, } } /// `Ready(Ok)` while anyone reads the open track, `Ready(Err)` once it closed, and /// otherwise park `waiter` for the next reader. Only `Pending` registers: act on the /// answer, not a later [`Self::is_wanted`], or a reader leaving in between is never seen. pub(super) fn poll_wanted(&self, waiter: &kio::Waiter) -> Poll> { self.state.poll_wanted_changed(waiter, false).map_ok(drop) } /// `Ready` once [`Self::is_held`] differs from `last`, closed or not, and otherwise park /// `waiter` for the change. pub(super) fn poll_held_changed(&self, waiter: &kio::Waiter, last: bool) -> Poll { self.state.poll_held_changed(waiter, last) } /// `Ready(Ok)` while nobody reads the track, `Ready(Err)` once it closed, and /// otherwise park `waiter` for the last reader leaving. Only `Pending` registers, /// as with [`Self::poll_wanted`]. pub(super) fn poll_unwanted(&self, waiter: &kio::Waiter) -> Poll> { self.state.poll_wanted_changed(waiter, true).map_ok(drop) } } impl super::WeakEntry for TrackWeak { fn is_closed(&self) -> bool { self.state.is_closed() } fn same_channel(&self, other: &Self) -> bool { self.state.same_channel(&other.state) } } /// A cloneable, watch-only handle to a track's subscriber demand. /// /// Obtained from [`Producer::demand`], [`Request::demand`], or [`Requests::demand`]. /// A publisher uses it to learn which group anyone subscribed, querying, or fetching /// wants (on-demand capture / encoding) without being able to publish frames or close the /// track. A bare [`Consumer`] is not demand. A group a subscription or fetch handed out /// keeps its track wanted until a reader reads its end, and a wanted track makes its /// broadcast wanted ([`broadcast::Demand`]). It's a weak handle, so it neither keeps /// the track alive nor pins its cached groups; once the owning [`Producer`] /// goes away, [`poll_wanted_changed`](Self::poll_wanted_changed) reports the track's /// closure. Each clone watches on its own, so its first /// [`poll_wanted_changed`](Self::poll_wanted_changed) reports the current value. pub struct Demand { name: Arc, state: kio::ProducerWeak, // The value `poll_wanted_changed` last returned, or `None` before it returned any. prev: Option, } impl Clone for Demand { fn clone(&self) -> Self { Self::new(self.name.clone(), self.state.clone()) } } impl Demand { fn new(name: Arc, state: kio::ProducerWeak) -> Self { Self { name, state, prev: None, } } /// The track name this handle is bound to. pub fn name(&self) -> &str { &self.name } /// Whether anyone consumes the open track: a subscription, query, fetch, or a group one /// of them handed out and a reader has not finished. pub fn wanted(&self) -> bool { self.is_wanted() } /// [`wanted`](Self::wanted) once it differs from what this handle last returned: /// immediately on the first call, then on each change. Errors once the track is closed /// or aborted. /// /// So `while demand.wanted_changed().await? {}` runs while anyone wants the track, and /// `while !demand.wanted_changed().await? {}` waits for someone to. End a track nobody /// wants with [`Producer::abort_unwanted`], which keeps it for a reader that joined since. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_wanted_changed(&mut self, waiter: &kio::Waiter) -> Poll> { let active = match self.poll_state(waiter) { DemandState::Closed => return Poll::Ready(Err(self.abort_reason())), DemandState::Active => true, DemandState::Idle => false, }; if self.prev == Some(active) { return Poll::Pending; } self.prev = Some(active); Poll::Ready(Ok(active)) } /// Ready while a consumer asks for a new group: the first group of a track with none /// written, or a live join or [`Subscriber::refresh`] judged by [`Info::refresh`], once /// its interval has passed. The next group appended answers it, so a publisher waits /// here, opens a group (a keyframe or a snapshot), and waits again. Errors once the track /// is closed or aborted. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_refresh(&self, waiter: &kio::Waiter) -> Poll> { match self.poll_state(waiter) { DemandState::Closed => return Poll::Ready(Err(self.abort_reason())), DemandState::Idle => return Poll::Pending, DemandState::Active => {} } match self.poll_asked(waiter) { Some(_) => Poll::Ready(Ok(())), None => Poll::Pending, } } /// The group start consumers ask for when it is past every key written, whether or not /// anyone consumes now. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn asked(&self) -> Option { self.check_asked(None) } /// [`Self::asked`], arming `waiter` for a change otherwise, including a deferred refresh /// coming due. pub(crate) fn poll_asked(&self, waiter: &kio::Waiter) -> Option { self.check_asked(Some(waiter)) } fn check_asked(&self, waiter: Option<&kio::Waiter>) -> Option { let state = self.state.read(); let now = state.cache.pool().now_wall(); let cache = state.cache.clone(); let subscriptions = state.subscriptions.clone(); drop(state); let mut subs = subscriptions.lock(); if subs.asking(now) { let wanted = subs.wanted(now); let mut wakes = subs.sent(now); drop(subs); wakes.wake(); return Some(wanted); } let waiter = waiter?; subs.watch(waiter); let deferred = subs.deferred(now); drop(subs); if let Some(at) = deferred { cache.watch_wall(at, waiter); } None } /// The lowest group start consumers want, `None` while nobody consumes: past every key /// written while they ask for a new group. #[cfg(test)] pub(crate) fn wanted_start(&self) -> Option { if !self.is_wanted() { return None; } let now = self.state.read().cache.pool().now_wall(); Some(self.state.read().subscriptions.clone().lock().wanted(now)) } /// Resolves once the track's outcome is final; see [`Producer::poll_closed`]. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { TrackState::settle( self.state .consume_weak() .poll(waiter, |state| state.poll_outcome(waiter)), ) } /// The publisher's tie-break priority, as set in [`Info::priority`]. pub(super) fn priority(&self) -> i8 { // Always Some once the track exists; a closed one reads its last value. self.state .read() .info .as_ref() .map_or(DEFAULT_PRIORITY, |info| info.priority) } /// Whether the track is open and anyone consumes it right now, without waiting. pub(super) fn is_wanted(&self) -> bool { !self.state.is_closed() && self.state.wanted() } /// Whether the track is gone, without waiting. pub(super) fn is_closed(&self) -> bool { self.state.is_closed() } /// Read the current demand and arm `waiter` for the next transition. /// /// Level-triggered: it answers what the demand is *now* and wakes on whichever way it /// can move next, so a poll-driven caller watching several tracks at once doesn't have /// to track which edge it's waiting for. pub(super) fn poll_state(&self, waiter: &kio::Waiter) -> DemandState { loop { match self.state.poll_wanted_changed(waiter, false) { Poll::Ready(Err(_)) => return DemandState::Closed, // Not wanted, and armed for it becoming wanted. Poll::Pending => return DemandState::Idle, // Wanted, so arm for the reverse. Poll::Ready(Ok(_)) => match self.state.poll_wanted_changed(waiter, true) { Poll::Ready(Err(_)) => return DemandState::Closed, Poll::Pending => return DemandState::Active, // Went idle between the two reads, so neither poll armed anything. // Start over rather than returning a state with no waker behind it. Poll::Ready(Ok(_)) => continue, }, } } } /// The recorded abort reason, or [`Error::Dropped`] if the track closed without one. fn abort_reason(&self) -> Error { self.state.read().outcome.aborted().cloned().unwrap_or(Error::Dropped) } } #[cfg(all(test, feature = "async"))] impl Demand { /// Wait until [`Self::wanted`] satisfies `f`, returning it. Errors once the track closes. pub(crate) async fn until(&mut self, f: impl Fn(bool) -> bool) -> Result { kio::wait(|waiter| { loop { match self.poll_wanted_changed(waiter) { Poll::Ready(Ok(wanted)) if f(wanted) => return Poll::Ready(Ok(wanted)), Poll::Ready(Ok(_)) => continue, Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => return Poll::Pending, } } }) .await } } /// What [`Demand::poll_state`] found. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub(super) enum DemandState { /// At least one subscriber. Active, /// No subscribers, but the track is still open. Idle, /// The track is gone; it will never be demanded again. Closed, } /// A watch-only handle to a track's live arrivals. // Legacy fetch batches share one ledger read per poll. pub(crate) struct Arrivals { state: kio::ConsumerWeak, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl Arrivals { /// Whether both handles watch the same track. // A fetch batch may share only one track's arrival ledger. pub(crate) fn same_channel(&self, other: &Self) -> bool { self.state.same_channel(&other.state) } /// Read the ledger and wake its owner when arrivals or terminal bounds change. // One owner watch opens batch deadlines without waking each held fetch. pub(crate) fn poll(&self, waiter: &kio::Waiter) -> ArrivalRead<'_> { let _ = self.state.poll(waiter, |_| Poll::<()>::Pending); let state = self.state.read(); let max_delay = clamp_max_delay(state.subscriptions.lock().max_delay(), state.max_age_bound()); ArrivalRead { state, grace: ledger::grace(max_delay), } } } /// A locked arrival ledger, so a batch never copies or re-locks its runs per fetch. pub(crate) struct ArrivalRead<'a> { state: kio::Ref<'a, TrackState>, grace: Duration, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl ArrivalRead<'_> { /// Whether the live feed can no longer deliver any missing group. // The batch checks closure without re-locking the track. pub(crate) fn is_closed(&self) -> bool { self.state.is_closed() } /// Gaps that can expire held fetches, including the gap before the first arrival. // Only the session's held-fetch deadline index needs all live gaps. pub(crate) fn folds(&self) -> impl Iterator, crate::time::Instant)> + '_ { self.state.arrivals.folds(self.grace) } /// Judge one live group and register only for its own arrival or terminal bound: the /// group, the error it was given up with, or `None` once it is never coming. // Legacy wires lack FETCH, so held requests need the live-feed verdict. pub(crate) fn poll_group( &self, key: u64, now: crate::time::Instant, waiter: &kio::Waiter, ) -> Poll>> { match self.state.poll_arrival(key, now, self.grace, waiter) { Poll::Pending if self.state.is_closed() => Poll::Ready(Ok(None)), res => res, } } /// The groups of `groups` [`Self::poll_group`] gives up on, judged by run, so a span a /// peer made wide costs what the cache and ledger hold rather than its width. // Legacy wires lack FETCH, so a held fill settles what the feed gave up in bulk. pub(crate) fn given_up(&self, groups: ops::Range, now: crate::time::Instant) -> RangeSet { let state = &*self.state; let mut gone = RangeSet::new(); let mut give_up = |groups: ops::Range| { if !groups.is_empty() { gone.insert(groups); } }; let closed = self.state.is_closed(); let start = state.start.key().unwrap_or(0); let fin = state.final_key().unwrap_or(u64::MAX); for hole in miss::holes(state, groups) { if closed { give_up(hole); continue; } give_up(hole.start..hole.end.min(start)); give_up(hole.start.max(fin)..hole.end); for (dropped, _) in state.drops.overlapping(&hole) { give_up(dropped.start.max(hole.start)..dropped.end.min(hole.end)); } // Every group of a gap folds at once: its grace runs from the run above it. let folded = |gap: u64| state.arrivals.folds_at(gap, self.grace).is_some_and(|at| at <= now); let mut from = hole.start; for arrived in state.arrivals.accounted(hole.clone()) { if from < arrived.start && folded(from) { give_up(from..arrived.start); } from = arrived.end; give_up(arrived); } if from < hole.end && folded(from) { give_up(from..hole.end); } } gone } } /// A lazy handle to a single track within a broadcast. /// /// Obtained from [`broadcast::Consumer::track`]. Holding it sends nothing and is not /// [`Demand`]: the track is requested, and counts as demanded, only while an /// [`info`](Self::info), [`subscribe`](Self::subscribe) (a live, ongoing stream of /// groups), or [`fetch_group`](Self::fetch_group) (one group) made from it is held. The /// same handle can be used any number of times, and clones are cheap. #[derive(Clone)] pub struct Consumer { name: Arc, // The broadcast this track belongs to, so a catalog track can name the path its // relative references resolve against. Rebound by `broadcast::Consumer::track` to // that handle's view, which may name the broadcast differently than its producer did. broadcast: Arc, source: Source, // Egress stats scope, set by a tagged [`broadcast::Consumer`] via // [`Self::with_stats`]. Empty (no-op) for an untagged track. stats: stats::Scope, // Last position this handle yielded through `largest`. largest: Option, } /// Where a [`Consumer`] finds its track. Neither joins the track's consumer count, which /// is its demand: only the requests made from the handle do. #[derive(Clone)] enum Source { /// One track instance, from its producer or request. Track(kio::ConsumerWeak), /// Whatever the broadcast serves under the name when a request is made, asking for /// the track only then. Once the broadcast has ended, the track it served when the /// handle was made, if any: a handle already handed out keeps reading it. Boxed so the /// handle stays small. Broadcast(Box, Option>), } impl Consumer { /// Whether these handles resolve to the same cached track, without subscribing. pub(super) fn same_channel(&self, other: &Self) -> bool { self.with_state(|left| { other.with_state(|right| left.zip(right).is_some_and(|(left, right)| left.same_channel(right))) }) } /// An already declared end, without resolving a pending subscription or registering a wake. pub(super) fn declared_end(&self) -> Option { self.with_state(|state| state?.read().declared()) } /// The error the declared end carries, reached or not. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn declared_error(&self) -> Option { self.with_state(|state| state?.read().outcome.end()?.error.clone()) } fn new(name: Arc, state: kio::ConsumerWeak) -> Self { let broadcast = state.read().broadcast.clone(); Self { name, broadcast, source: Source::Track(state), stats: stats::Scope::default(), largest: None, } } /// A handle resolving `name` through `broadcast` on each request. Called by /// [`broadcast::Consumer::track`] with that handle's view of the broadcast, and the /// track it serves under the name now, if any. pub(super) fn lookup( name: Arc, info: Arc, broadcast: broadcast::Consumer, current: Option>, ) -> Self { Self { name, broadcast: info, source: Source::Broadcast(Box::new(broadcast), current), stats: stats::Scope::default(), largest: None, } } /// The track a request made now reads, joined to its consumer count. A broadcast /// that cannot serve the name answers with a track closed with its error. fn resolve(&self, subscription: Option<&Subscription>) -> (kio::Consumer, Option) { let state = match &self.source { Source::Track(state) => state.clone(), Source::Broadcast(broadcast, current) => match broadcast.resolve_track(&self.name, subscription) { Ok(resolved) => return resolved, Err(Error::Unroutable) if let Some(current) = current => current.clone(), Err(err) => { let request = Request::new(self.broadcast.clone(), self.name.clone()); let state = request.peek(); request.reject(err); state } }, }; let preferences = subscription.map(|value| Preferences::register(state.read(), value.clone())); (state.consume(), preferences) } /// Run `f` on the track instance this handle reads now, without joining its consumer /// count: `None` when an open broadcast serves nothing under the name now. fn with_state(&self, f: impl FnOnce(Option<&kio::ConsumerWeak>) -> R) -> R { match &self.source { Source::Track(state) => f(Some(state)), Source::Broadcast(broadcast, current) => f(broadcast.peek_track(&self.name, current.as_ref()).as_ref()), } } /// Attach an egress stats scope, inherited by the subscriptions, fetches, and /// groups derived from this handle. Called by a tagged [`broadcast::Consumer`]. pub(super) fn with_stats(mut self, scope: stats::Scope) -> Self { self.stats = scope; self } /// This handle with its stats scope cleared, so the subscriptions, fetches, and groups /// derived from it count toward no stats session: for reads a consumer makes for itself /// rather than on behalf of whoever its origin is metered for. pub fn unmetered(mut self) -> Self { self.stats = stats::Scope::default(); self } /// The track name this handle is bound to. pub fn name(&self) -> &str { &self.name } /// The broadcast this track belongs to, as reached through the handle it came from. /// Its [`path`](broadcast::Info::path) is what a catalog's relative `broadcast` /// references resolve against. pub fn broadcast(&self) -> &broadcast::Info { &self.broadcast } /// Open a live subscription. /// /// Registers the subscription on the track and returns a [`kio::Pending`] that resolves to the /// [`Subscriber`] once the track info is available, or the track's abort error (or /// [`Error::Dropped`]) if it is already closed. /// /// Reads only the groups the subscription's ranges cover. /// [`Delay::max`] is what asks for data: delivery skips everything in range /// that the budget convicts, so the default budget of zero delivers only the latest /// group and a larger one reaches back over what it can still use. pub fn subscribe(&self, subscription: impl Into>) -> kio::Pending { let (state, subscription) = self.resolve(Some(&subscription.into().unwrap_or_default())); let subscription = subscription.expect("a subscription was requested"); kio::Pending::new(Subscribing { name: self.name.clone(), broadcast: self.broadcast.clone(), state, subscription, stats: self.stats.clone(), }) } /// Poll for the first group the live feed serves, once the serving session has /// resolved it: `Some` for a declared start, `None` for none (the live edge, or a /// source that never declares one). Parks while a lite-06+ session still owes its /// SUBSCRIBE_START (see [`Producer::request_start`]); a closed track is ready with /// whatever it last declared, since nothing will resolve it anymore. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn poll_state_start(state: &kio::Consumer, waiter: &kio::Waiter) -> Poll> { let res = state.poll(waiter, |state| match state.start { Start::Pending(_) if state.outcome.aborted().is_none() => Poll::Pending, start => Poll::Ready(start.key()), }); match ready!(res) { Ok(start) => Poll::Ready(start), Err(state) => Poll::Ready(state.start.key()), } } /// The upstream publisher's Largest Object, independently of which groups are cached. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn reported_object(&self) -> Option<(u64, u64)> { if let Some(serving) = self.serving() { return serving.reported_object(); } self.with_state(|state| state?.read().object_edge()) } /// The newest group, when it is already cached: resolved synchronously, without /// counting as a fetch or a delivery. The IETF publisher snapshots its frame count to /// resolve Largest Object; a group that is not immediately available reads as no edge. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn peek_latest(&self) -> Option { if let Some(serving) = self.serving() { return serving.peek_latest(); } let key = self.with_state(|state| state?.read().max_key)?; self.peek_group(key) } /// The nearest cached group below `key`, under the same terms as /// [`Self::peek_group`]. Walks the cache's own order, so gaps in the group numbering /// are crossed and aborted (evicted) entries are skipped. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn peek_before(&self, key: u64) -> Option { if let Some(serving) = self.serving() { return serving.peek_before(key); } self.with_state(|state| { state? .read() .lookup .range(..key) .rev() .find(|(_, slot)| slot.serves_end()) .map(|(_, slot)| slot.serving().consume()) }) } /// A cached group by key, under the same terms as [`Self::peek_latest`]. Unlike a /// fetch, a peek does not refresh the group's cache standing, so it never keeps a /// group alive over one a subscriber actually read; an aborted (evicted) group, or a /// copy cut short at the back, is a miss. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn peek_group(&self, key: u64) -> Option { if let Some(serving) = self.serving() { return serving.peek_group(key); } self.with_state(|state| { let state = state?.read(); let slot = state.lookup.get(&key).filter(|slot| slot.serves_end())?; Some(slot.serving().consume()) }) } /// The live arrivals of the track this handle reads now, without joining its demand. // This batch view avoids per-fetch ledger copies and locks in the session. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn arrivals(&self) -> Option { self.with_state(|state| state.map(|state| Arrivals { state: state.clone() })) } /// Poll for a cached group, a terminal disposition, or a gap outliving its grace: the /// group, the error a dropped or cut group was given up with, or `None` once it is gone. pub(crate) fn poll_group(&self, key: u64, waiter: &kio::Waiter) -> Poll>> { self.with_state(|state| { let Some(state) = state else { return Poll::Ready(Ok(None)); }; let res = state.poll(waiter, |state| { let max_delay = clamp_max_delay(state.subscriptions.lock().max_delay(), state.max_age_bound()); let grace = ledger::grace(max_delay); let now = state.cache.pool().now_wall(); let res = state.poll_arrival(key, now, grace, waiter); if res.is_pending() { let Some(at) = state.arrivals.folds_at(key, grace) else { // No later arrival yet: only this wait uses the track's change channel. return Poll::Pending; }; state.cache.watch_wall(at, waiter); } // A known gap waits on its own arrival and deadline, not unrelated writes. Poll::Ready(res) }); let res = match res { Poll::Ready(Ok(res)) => res, Poll::Ready(Err(_)) => Poll::Ready(Ok(None)), Poll::Pending => Poll::Pending, }; if res.is_pending() && state.poll_closed(waiter).is_ready() { // A cached arrival may have raced closure after the first lookup. let state = state.read(); return Poll::Ready(match state.landing(key) { Poll::Ready(Some(group)) => Ok(Some(group)), _ => state.given_up(key).map_or(Ok(None), Err), }); } res }) } /// Lend `head` to the cached copies of group `key`; see /// [`group::Handle::adopt_head`]. pub(super) fn adopt_head(&self, key: u64, head: &group::Head) { let copies: Vec = self.with_state(|state| { let Some(state) = state else { return Vec::new(); }; let state = state.read(); state .lookup .get(&key) .map_or_else(Vec::new, |slot| slot.copies().cloned().collect()) }); // With the track lock released (lock order is track then group): adopting settles // the group's cache debt. for group in copies { group.adopt_head(head); } } /// Poll for group `key` falling a full `budget`, clamped to this track's max age, /// behind its live edge, as a reader would judge it, whether the track holds it or not; /// see `TrackState::drifted`. Parks in this track's expiry index, so only a change that /// can make the group stale, or land it here, wakes it. Pending for good once the track closes. pub(super) fn poll_stale(&self, key: u64, budget: Duration, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| { let Some(state) = state else { return Poll::Pending; }; let state = state.read(); let budget = clamp_max_delay(budget, state.max_age_bound()); state.cache.wakes().watch_held(key, waiter); match state.poll_drifted(key, &Servable::ALL, budget, None, waiter) { true => Poll::Ready(()), false => Poll::Pending, } }) } /// How many groups arrived here so far; see [`Self::poll_arrived`]. pub(super) fn arrived(&self) -> usize { self.with_state(|state| state.map_or(0, |state| state.read().next_arrival)) } /// Ready once more than `seen` groups arrived here. A received copy still takes the groups /// in flight below an end its readers stopped waiting at. Pending for good once the track closes. pub(super) fn poll_arrived(&self, seen: usize, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| { let Some(state) = state else { return Poll::Pending; }; match state.poll(waiter, |state| match state.next_arrival > seen { true => Poll::Ready(()), false => Poll::Pending, }) { Poll::Ready(Ok(())) => Poll::Ready(()), _ => Poll::Pending, } }) } /// Whether a group in `groups` may still land here; see `TrackState::may_land`. Never /// once no producer is left to write one. Parks `waiter` until its gap folds. pub(super) fn may_land( &self, groups: ops::Range, floor: bool, grace: Duration, delivered: Option<&Delivered>, waiter: &kio::Waiter, ) -> bool { if let Some(serving) = self.serving() { return serving.may_land(groups, floor, grace, delivered, waiter); } self.with_state(|state| { state.is_some_and(|state| { !state.is_closed() && state.read().may_land(groups, floor, grace, delivered, waiter) }) }) } /// The wall clock this track's cache judges ages by. pub(super) fn now(&self) -> crate::time::Instant { if let Some(serving) = self.serving() { return serving.now(); } self.with_state(|state| match state { Some(state) => state.read().cache.pool().now_wall(), None => super::clock::wall(), }) } /// Poll for this track's wall clock reaching `at`; see [`Self::now`]. pub(super) fn poll_wall(&self, at: crate::time::Instant, waiter: &kio::Waiter) -> Poll<()> { if let Some(serving) = self.serving() { return serving.poll_wall(at, waiter); } self.with_state(|state| { let Some(state) = state else { return Poll::Ready(()); }; let state = state.read(); if state.cache.pool().now_wall() >= at { return Poll::Ready(()); } state.cache.watch_wall(at, waiter); Poll::Pending }) } /// The group a front's parked copy would serve for `key` from `frame_start`, when it /// is whole: finished from frame 0 on a copy still open. Anything less (open, headless, /// aborted, or on a failed copy) needs the route. The fetch joins the copy's demand, so /// a copy kept busy only by fetches stays cached at its session. fn parked_whole(&self, key: u64, frame_start: u64) -> Option<(kio::Consumer, group::Consumer)> { let copy = self.with_state(|state| state?.read().routes.as_ref()?.parked())?; copy.with_state(|state| { let state = state?; let group = { let track = state.read(); let group = track.covering_group(key, frame_start)?; let whole = !track.is_closed() && track.outcome.aborted().is_none() && group.live_first_frame() == Some(0) && group.is_finished(); if !whole { return None; } let consumer = group.try_consume()?; group.cache_refresh(); consumer }; Some((state.consume(), group)) }) } /// The serving route's copy, for a front's logical track; see [`super::resume`]. fn serving(&self) -> Option { let routes = self.with_state(|state| state?.read().routes.clone())?; routes.serving() } /// Ask the instance this handle reads for a new group; see [`Subscriber::ask`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn ask(&self, ask: Ask) { if let Some(serving) = self.serving() { return serving.ask(ask); } self.with_state(|state| { if let Some(state) = state { TrackState::ask(state.read(), ask); } }); } /// Request a new group from the instance this handle reads; see [`Subscriber::refresh`]. fn request(&self) -> Refreshing { self.with_state(|state| match state { Some(state) if state.read().routes.is_some() => Refreshing::front(state), Some(state) => TrackState::request(state), None => Refreshing::done(), }) } /// Whether the cache is fed live, so the newest object it holds is the track's largest. /// Fetch-only, closed, and parked copies are not, even once they learn the end. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn is_live(&self) -> bool { if let Some(serving) = self.serving() { return serving.is_live(); } // A front's track with no copy spliced is not fed at all. self.with_state(|state| { state.is_some_and(|state| { let state = state.read(); state.routes.is_none() && state.minted() && !state.outcome.is_released() && state.feed.is_live() }) }) } /// Fetching a single past group, without holding a live subscription. /// /// Returns a [`kio::Pending`] that resolves to the [`group::Consumer`]: /// immediately if the group is cached, otherwise once a [`Requests`] serves /// the request (a wire FETCH for a relay). `options` accepts `None`, a [`group::Fetch`], /// or `group::Fetch::default()`. /// /// The returned future resolves to [`Error::NotFound`] when the group can never be served /// (past the final key, or no [`Requests`] on the track), the handler's /// rejection (a relay's upstream miss is [`StreamError::NotFound`](StreamError::NotFound)), /// or the track's abort error if it's already closed. A miss queues a one-group [`FillGroups`], /// which concurrent fetches of the same key join. pub fn fetch_group(&self, key: u64, options: impl Into>) -> kio::Pending { self.fetching( key, FetchOptions { options: options.into().unwrap_or_default(), end: None, }, ) } /// Fill a wire request, widening a cold or compressed group to a reusable prefix. It asks /// on a reader's behalf, so a key the serving copy gave up for good is answered from /// that drop rather than asked upstream again, as the copy's own readers are. pub(crate) fn fetch_range(&self, key: u64, mut request: FetchOptions) -> kio::Pending { let state = self.resolve(None).0; { let state = state.read(); if state.lookup.get(&key).is_none_or(Slot::is_aborted) || state.info.as_ref().is_none_or(|info| info.compressed()) { request.options.frame_start = 0; } // A compressed stream needs the entire dictionary; an unknown track is conservative // until TRACK_INFO has told the relay which representation it will receive. if state.info.as_ref().is_none_or(|info| info.compressed()) { request.end = None; } } self.fetching(key, request) } fn fetching(&self, key: u64, request: FetchOptions) -> kio::Pending { let FetchOptions { options, end: frame_end, } = request; // One fetch per calling context, counted here (coalesced upstream work is // still one request served). Independent of `subscriptions` and the viewer // refcount. self.stats.fetch(); // Nothing more feeds a whole group, so no route needs to stay up for it: the copy a // front parked serves it without reviving the logical track for one read. if let Some((state, group)) = self.parked_whole(key, options.frame_start) { let fills = state.read().fills.clone(); return kio::Pending::new(Fetching { state, fills, key, frame_start: options.frame_start, frame_end, priority: options.priority, max_delay: options.max_delay, judges_drift: true, result: None, hit: Some(Box::new(group)), stats: self.stats.clone(), resume: None, }); } let state = self.resolve(None).0; let resume = state.read().routes.clone(); // A front's logical track caches nothing: the serving route answers. if let Some(resume) = resume { return kio::Pending::new(Fetching { state: state.clone(), fills: state.read().fills.clone(), key, frame_start: options.frame_start, frame_end, priority: options.priority, max_delay: options.max_delay, judges_drift: true, result: None, hit: None, stats: self.stats.clone(), resume: Some(Box::new(resume.fetch_group( key, FetchOptions { options, end: frame_end, }, state.clone(), ))), }); } kio::Pending::new(Fetching::cached_or_queued( state, key, FetchOptions { options, end: frame_end, }, self.stats.clone(), )) } /// The group `back` groups before `latest` among those the cache holds, `latest` itself /// counting as the first: `None` once fewer are held. What a wire's relative start counts /// back to, since keys are starts and no arithmetic on them counts groups. #[cfg(feature = "async")] pub(crate) fn group_back(&self, latest: u64, back: u64) -> Option { if let Some(serving) = self.serving() { return serving.group_back(latest, back); } self.with_state(|state| Holdings::back(&*state?.read(), latest, back)) } /// The exclusive end of the run at `key` that no group will fill: one declared skipped, /// or with `dropped`, also one given up for good. See [`Subscriber::next_in_range`]. pub(super) fn hole_end(&self, key: u64, dropped: bool) -> Option { if let Some(serving) = self.serving() { return serving.hole_end(key, dropped); } self.with_state(|state| state?.read().hole_end(key, dropped)) } /// The start of the group before `start`; see [`Subscriber::previous_start`]. pub(super) fn previous_start(&self, start: u64) -> Option { if let Some(serving) = self.serving() { return serving.previous_start(start); } self.with_state(|state| state?.read().previous_start(start)) } /// Whether the cache holds the group at `key`; see [`Subscriber::holds`]. pub(super) fn holds(&self, key: u64) -> bool { if let Some(serving) = self.serving() { return serving.holds(key); } self.with_state(|state| state.map(|state| state.read().holds_group(key))) .unwrap_or(false) } /// The keys of `span` the cache holds, lowest first. #[cfg(feature = "async")] pub(crate) fn cached_groups(&self, span: ops::Range) -> Vec { if let Some(serving) = self.serving() { return serving.cached_groups(span); } self.with_state(|state| { let Some(state) = state else { return Vec::new(); }; let state = state.read(); state .lookup .range(span) .filter(|(_, slot)| slot.serves_end()) .map(|(key, _)| *key) .collect() }) } /// Resolve the track's [`Info`] without subscribing. /// /// A [`Consumer`] is a lazy handle, so the info may not be known yet: this waits /// for the producer to [`Request::accept`] the track (a wire TRACK_INFO round-trip /// for a relay), and errors with the track's abort error if it closes first. /// [`Subscriber::info`] is the already-resolved counterpart. pub fn info(&self) -> kio::Pending { kio::Pending::new(Resolving { name: self.name.clone(), broadcast: self.broadcast.clone(), state: self.resolve(None).0, stats: self.stats.clone(), }) } /// The track's largest location, its newest group at the newest frame (0 while empty): /// the current one, then each newer one, and `None` once the track ends. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_largest(&mut self, waiter: &kio::Waiter) -> Poll> { let mut seen = self.largest; let next = self.poll_largest_after(&mut seen, Bounds::default(), waiter); self.largest = seen; next } /// Whether `position` is the largest and its group ended there, so no frame follows it. A /// front's logical track asks the route serving it. pub(super) fn largest_finished(&self, position: Position) -> bool { self.with_state(|state| { let Some(state) = state else { return false; }; let (routes, cache) = { let state = state.read(); (state.routes.clone(), state.cache.clone()) }; match routes { Some(routes) => routes.largest_finished(position), None => cache.largest_finished(position), } }) } /// Poll for a largest frame within `bounds` newer than `seen`, recording it there. A /// front's logical track follows the route serving it. Reads the instance served now /// without asking for it, so a broadcast serving nothing under the name ends the stream. pub(super) fn poll_largest_after( &self, seen: &mut Option, bounds: Bounds, waiter: &kio::Waiter, ) -> Poll> { self.with_state(|state| { let Some(state) = state else { return Poll::Ready(None); }; let (routes, cache) = { let state = state.read(); (state.routes.clone(), state.cache.clone()) }; match routes { Some(routes) => routes.poll_largest(seen, bounds, waiter), None => poll_largest( &cache, seen, bounds, waiter, |waiter| state.poll(waiter, |state| largest_live(state, waiter)).map(|_| ()), |waiter| state.poll(waiter, |state| largest_ended(state, waiter)).map(|_| ()), ), } }) } pub(crate) fn latest_key(&self) -> Option { if let Some(serving) = self.serving() { return serving.latest_key(); } self.with_state(|state| state?.read().max_key) } /// See [`TrackState::skipped_below`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn skipped_below(&self, key: u64) -> u64 { if let Some(serving) = self.serving() { return serving.skipped_below(key); } self.with_state(|state| state.map_or(0, |state| state.read().skipped_below(key))) } /// Ready once [`Self::latest_key`] is `Some`: the track holds a group. Stays pending /// on a track that closed without one. pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| { let Some(state) = state else { return Poll::Pending; }; let routes = state.read().routes.clone(); if let Some(routes) = routes { return routes.poll_grouped(waiter); } match state.poll(waiter, |state| match state.max_key { Some(_) => Poll::Ready(()), None => Poll::Pending, }) { Poll::Ready(Ok(())) => Poll::Ready(()), Poll::Ready(Err(closed)) if closed.max_key.is_some() => Poll::Ready(()), _ => Poll::Pending, } }) } /// The first group the declared end excludes whole, or `None` while the track is open ended. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn final_key(&self) -> Option { if let Some(serving) = self.serving() { return serving.final_key(); } self.with_state(|state| state?.read().final_key()) } /// The frame-precise point a replacement route should resume from: one past the /// last frame this copy produced. `None` if it produced nothing. /// /// Survives the track aborting, which is when a route change asks. #[cfg(test)] pub(crate) fn resume_position(&self) -> Option { self.with_state(|state| state?.read().resume_position()) } /// Poll for the track closing: no producer is left to write it, so nothing more /// will ever arrive. A name the broadcast serves nothing under reads as closed. pub(super) fn poll_closed(&self, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| state.map_or(Poll::Ready(()), |state| state.poll_closed(waiter))) } /// Poll for the track reaching a terminal state: `Ok(())` once it is complete /// (the final group was produced), `Err` once it closed or aborted before /// completing. This tells a track that truly ended from one whose serving route /// died mid-stream. pub(super) fn poll_complete(&self, waiter: &kio::Waiter) -> Poll> { self.with_state(|state| { let Some(state) = state else { return Poll::Ready(Err(Error::Dropped)); }; match ready!(state.poll(waiter, |state| { if state.is_complete() { Poll::Ready(()) } else { state.watch_tail(waiter); Poll::Pending } })) { Ok(_) => Poll::Ready(Ok(())), // Closed before completing. Read through the returned guard: it holds // the lock, so re-locking the channel here would deadlock. Err(closed) => Poll::Ready(Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped))), } }) } } /// The pollable state of a [`Consumer::subscribe`]; awaited via the /// [`kio::Pending`] wrapper, whose `DerefMut` exposes [`Self::update`]. pub struct Subscribing { name: Arc, broadcast: Arc, state: kio::Consumer, subscription: Preferences, stats: stats::Scope, } impl Subscribing { /// Resolve the same track's info with this subscription already registered. pub(super) fn info(&self) -> Resolving { Resolving { name: self.name.clone(), broadcast: self.broadcast.clone(), state: self.state.clone(), stats: self.stats.clone(), } } /// How long the upstream that refused the track asked to wait before asking again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn retry(&self) -> Option { self.state.read().outcome.retry() } /// Poll until the peer confirms the subscription, yielding the [`Subscriber`]. /// Errors if the track is aborted or not found. pub fn poll_ok(&self, waiter: &kio::Waiter) -> Poll> { // Wait until the track info is available let info = ready!(self.state.poll(waiter, |state| state.poll_info())) .map_err(|e| e.outcome.aborted().cloned().unwrap_or(Error::Dropped))??; let resume = self.state.read().routes.clone(); let inner = match resume { Some(resume) => Inner::Resume( Box::new(resume.subscribe(self.subscription.clone())), Logical { state: self.state.clone(), record: Record::default(), }, ), None => Inner::Plain(Cursor::new(self.state.clone(), self.subscription.clone())), }; Poll::Ready(Ok(Subscriber { name: self.name.clone(), broadcast: self.broadcast.clone(), info, inner, stats: self.stats.clone(), _stats_sub: self.stats.subscribe(), largest: Reported::default(), parks: false, })) } /// Change the subscription preferences before (or after) it resolves. /// /// Returns [`Error::Closed`] if the track already ended; the update is /// meaningless at that point and can usually be ignored. pub fn update(&mut self, subscription: Subscription) -> Result<()> { self.subscription.update(subscription) } /// The subscription's preferences, which [`Preferences::edit`] changes by key. pub(super) fn preferences(&self) -> &Preferences { &self.subscription } } impl kio::Task for Subscribing { type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { self.poll_ok(waiter) } } /// The pollable state of a [`Consumer::info`]; awaited via the /// [`kio::Pending`] wrapper. /// /// Counts as the track's [`Demand`] while held, answered or not. pub struct Resolving { name: Arc, broadcast: Arc, state: kio::Consumer, stats: stats::Scope, } impl Resolving { /// How long the upstream that refused the track asked to wait before asking again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn retry(&self) -> Option { self.state.read().outcome.retry() } /// A handle on the track instance this resolves, whatever its broadcast serves later. pub fn track(&self) -> Consumer { Consumer { name: self.name.clone(), broadcast: self.broadcast.clone(), source: Source::Track(self.state.weak()), stats: self.stats.clone(), largest: None, } } /// Poll until the track's [`Info`] is known, without subscribing to its groups. pub fn poll_ok(&self, waiter: &kio::Waiter) -> Poll> { // Wait until the track info is available let info = ready!(self.state.poll(waiter, |state| state.poll_info())) .map_err(|e| e.outcome.aborted().cloned().unwrap_or(Error::Dropped))??; Poll::Ready(Ok(info)) } } impl kio::Task for Resolving { type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { self.poll_ok(waiter) } } /// Join the fill covering `key`, or queue a one-group fill; `None` when no handler /// exists to take one, or for the reserved key `u64::MAX`, which no group holds. fn queue(fills: &kio::Shared, key: u64, request: &FetchOptions) -> Option> { let options = &request.options; let end = key.checked_add(1)?; let ask = fill::Ask { start: Position { group: key, frame: options.frame_start, }, end, frame_end: request.end, asking: fill::Asking { priority: options.priority, max_delay: options.max_delay, // One group either way, so it never turns a wider fill it joins newest first. order: Order::Ascending, }, time: None, fetch: true, }; // The handler gate is atomic with a handler dropping (no fill stranded on a queue // nobody drains); with no handler, `Fetching::poll` fails fast. let (joins, _) = fills.lock().join(fills, &ask); joins.into_iter().next().map(Box::new) } /// One transport request, whose end may cut the stored copy short. #[derive(Clone)] pub(crate) struct FetchOptions { pub options: group::Fetch, pub end: Option, } /// The pollable state of a [`Consumer::fetch_group`]. /// /// Awaited via the [`kio::Pending`] wrapper; resolves to the /// [`group::Consumer`] once the group lands in the track's cache (already present, /// or produced after a wire FETCH), or [`Error::NotFound`] if it can never exist. pub struct Fetching { state: kio::Consumer, fills: kio::Shared, key: u64, // This caller's own start, so a cached group that begins above it is a miss // rather than a short answer. frame_start: u64, frame_end: Option, // This caller's priority, for a fetch that moves to a front's routes. priority: i8, // This caller's own age cap. None means only the publisher's max age. max_delay: Option, // Whether this fetch refuses a group a newer cached one superseded, judged across the // whole cache. A cursor's head fill leaves it to the cursor, which judges its window. judges_drift: bool, // The fill this fetch joined; `None` on a cache hit, or when no handler existed to // queue on. Boxed: a miss already allocates, and the handle stays small. result: Option>, // The group already cached when the fetch was made, held so it stays wanted. // Boxed: a group consumer dwarfs the rest of the handle. hit: Option>, // Egress stats scope, so the resolved group carries a payload meter (and counts // as one delivered group). Empty (no-op) for an untagged track. stats: stats::Scope, // A front's logical track: fetched from whichever route serves it. Boxed: rare. resume: Option>, } impl Fetching { /// A fetch of `key` from the track's own cache, queueing a fill for it only when /// the cache can't resolve it (cached, aborted, or past-final all resolve through /// [`Self::poll`] without one). fn cached_or_queued( state: kio::Consumer, key: u64, request: FetchOptions, stats: stats::Scope, ) -> Self { let frames = group::Bounds { start: request.options.frame_start, end: request.end, }; let (fills, cached) = { let state = state.read(); (state.fills.clone(), state.poll_fetch_cached(key, frames)) }; let result = match cached.is_pending() { true => queue(&fills, key, &request), false => None, }; // Hold a hit until the caller polls: the held consumer keeps the group wanted, // so an abandoned fetch filling it can't abort it in between. let hit = match cached { Poll::Ready(Ok(group)) => Some(Box::new(group)), _ => None, }; Self { state, fills, key, frame_start: request.options.frame_start, frame_end: request.end, priority: request.options.priority, max_delay: request.options.max_delay, judges_drift: true, result, hit, stats, resume: None, } } /// Leave staleness to the caller, a cursor filling a head: it judges the group against /// its own window when it hands it out, where a newer group outside that window /// supersedes nothing. fn unjudged(mut self) -> Self { self.judges_drift = false; self } } impl kio::Task for Fetching { type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { let res = ready!(self.poll_answer(waiter)); // Picked up, so a batch fill this fetch joined may serve its next group, though the // caller keeps the fetch and its join. if let Some(join) = &self.result { let mut picked = RangeSet::new(); picked.insert(self.key..self.key.saturating_add(1)); join.saw(&picked); } Poll::Ready(res) } } impl Fetching { fn poll_answer(&mut self, waiter: &kio::Waiter) -> Poll> { // Made before a front took its logical track, which caches nothing: the serving // route answers instead. if self.resume.is_none() && let Some(routes) = self.state.read().routes.clone() { let options = group::Fetch { priority: self.priority, max_delay: self.max_delay, frame_start: self.frame_start, }; self.resume = Some(Box::new(routes.fetch_group( self.key, FetchOptions { options, end: self.frame_end, }, self.state.clone(), ))); } if let Some(resume) = &mut self.resume { let group = ready!(kio::Task::poll(&mut **resume, waiter))?; return Poll::Ready(Ok(group.with_meter(self.stats.meter()))); } let (state, fills, key, frame_start, result) = ( &self.state, &self.fills, self.key, self.frame_start, self.result.as_ref(), ); let cache = { let state = state.read(); let budget = clamp_max_delay(self.max_delay.unwrap_or(Duration::MAX), state.max_age_bound()); let _ = state.cache.wakes().watch_landing(key, waiter); if self.judges_drift && state.poll_drifted(key, &Servable::ALL, budget, None, waiter) { return Poll::Ready(Err(Error::Old)); } state.cache.clone() }; // Hand back a consumer sitting where the caller asked rather than at the group's // own start. Coverage was checked first, so this can only skip frames they excluded. let expiry: Arc = Arc::new(FetchExpiry { state: state.weak(), cache, reading: Reading::new(state.clone()), key, priority: self.priority, max_delay: self.max_delay, superseded: Superseded::default(), }); let resolve = |group: group::Consumer| { group .with_range(frame_start..) .with_end(self.frame_end) .with_meter(self.stats.meter()) .with_expiry(expiry.clone()) }; if let Some(group) = &self.hit { return Poll::Ready(Ok(resolve((**group).clone()))); } // Track side: the cached group, the abort error, or past-final. The outer // error is the channel closing without any of those. let frames = group::Bounds { start: frame_start, end: self.frame_end, }; let cached = |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(key, frames)) { Poll::Ready(Ok(res)) => Poll::Ready(res.map(resolve)), Poll::Ready(Err(closed)) => Poll::Ready(Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped))), Poll::Pending => Poll::Pending, }; if let Poll::Ready(res) = cached(waiter) { return Poll::Ready(res); } // Handler side. let Some(result) = result else { // No group is ever created at the reserved key, so no handler is asked for it. if key == u64::MAX { return Poll::Ready(Err(Error::NotFound)); } // Never queued: no handler existed when the fetch was made. Fail fast while // that's still true; a handler that appeared since may yet fill the cache. return match fills.poll(waiter, |fills| match fills.takes_fetches() { false => Poll::Ready(()), true => Poll::Pending, }) { Poll::Ready(_guard) => Poll::Ready(Err(Error::NotFound)), Poll::Pending => Poll::Pending, }; }; // An answered fill fails a joined fetch the cache cannot cover. The channel // closing without an answer means its handlers went away. let (err, asked, partial) = match result.outcome().poll(waiter, |outcome| match outcome.refused(key) { Some((err, asked)) => Poll::Ready((err.clone(), asked, None)), None if outcome.is_accepted(key) => Poll::Ready((Error::NotFound, None, outcome.partial(key, frames))), None => Poll::Pending, }) { Poll::Ready(Ok(res)) => res, Poll::Ready(Err(_closed)) => (Error::NotFound, None, None), Poll::Pending => return Poll::Pending, }; // An accept caches the group before it answers the channel, which may land // between the track check above and this one: look again before failing. if let Poll::Ready(res) = cached(waiter) { return Poll::Ready(res); } // A later copy displaced the one accepted for this fetch from the cache. if let Some(group) = partial.as_ref().and_then(group::Handle::try_consume) { return Poll::Ready(Ok(resolve(group))); } // Joined after a tighter budget went on the wire, which judged the group too old // for that budget, not ours: ask again with ours. Terminates, since the new // fill asks at least our budget. if matches!(err, Error::Old) && looser(asked, self.max_delay) { let options = group::Fetch { priority: self.priority, max_delay: self.max_delay, frame_start: self.frame_start, }; self.result = queue( &self.fills, self.key, &FetchOptions { options, end: self.frame_end, }, ); return self.poll_answer(waiter); } Poll::Ready(Err(err)) } } /// A live subscription to a track, used to read its groups. /// /// Created via [`Consumer::subscribe`](Consumer::subscribe), or /// directly from a [`Producer`] for an in-process track. Carries this /// subscriber's [`Subscription`] preferences, which feed the producer's aggregate. /// /// # Local cursor vs wire preference /// /// Group bounds exist at two levels: /// /// - [`Subscription::ranges`], set via [`Self::update`], are both a **request to the /// publisher** and **this subscriber's read filter**. The publisher sees the union across /// every live subscriber, but each subscriber reads only the groups its own ranges cover, /// resolved where it is served: a relative anchor counted back from the latest group (or /// the first to arrive, before any exists), again whenever the ranges are replaced. /// - [`Self::with_range`] narrows **this subscriber's reads** further, without changing the /// publisher's demand. /// /// A group outside either is held rather than skipped past, so widening either one later /// makes it available again. Within both, [`Delay::max`] decides what is /// still worth delivering, measured against the track's live edge. pub struct Subscriber { name: Arc, // The broadcast this track belongs to; see [`Self::broadcast`]. broadcast: Arc, info: Info, inner: Inner, // Egress stats scope, used to meter the groups this subscriber reads. Empty // (no-op) for an untagged track. stats: stats::Scope, // The subscription guard: bumps `subscriptions` (and the egress viewer refcount) // while held, closing them on drop. Empty (no-op) for an untagged track. _stats_sub: stats::Subscription, largest: Reported, /// Whether a read past a bounded window's end parks rather than ending: a wire /// publisher's, whose peer would read the end of its stream as the track's. parks: bool, } /// What a [`Subscriber`]'s `largest` stream last reported. #[derive(Default)] struct Reported { position: Option, // The stream ended: moving the bounds afterwards does not bring it back. ended: bool, // The parked poll, woken when the bounds move. Its registration dies with its waiter, // so a dropped `largest()` future leaves nothing behind. parked: kio::WaiterList, } impl Reported { /// The bounds moved: wake a parked poll, since the largest may now be within them. fn bounds_moved(&mut self) { self.parked.wake(); } } /// Unread groups a logical reader can still take from a replaced copy. pub(super) struct Unread<'a> { pub start: u64, pub end: Option, pub delivered: &'a Delivered, } /// How a [`Subscriber`] reads: its track's own cache, or a front's routes. #[allow( clippy::large_enum_variant, reason = "the plain cursor is the hot path, so it stays inline and the front's reader is boxed; an allow, since the size gap varies by build" )] enum Inner { Plain(Cursor), /// A front's logical track, read straight from its routes' copies; see /// [`super::resume`]. Holds the logical track so it counts as a reader. /// Boxed: the plain cursor is the hot path. Resume(Box, Logical), } /// A front's logical track, held so it counts as a reader, and what its reader settled. struct Logical { state: kio::Consumer, record: Record, } /// A group a [`Subscriber`] reserved without handing it out; see /// [`Subscriber::poll_peek_group`]. It holds the group and the step that moves past it, /// so taking it never asks the cursor again: a route or bound that moved meanwhile cannot /// lose it. #[allow( clippy::large_enum_variant, reason = "a plain read, the hot path, passes its group on the stack instead of allocating" )] pub(super) enum Reserved { Plain(Peeked), Resume(Box), } impl Reserved { pub(super) fn key(&self) -> u64 { match self { Self::Plain(peeked) => peeked.group.start, Self::Resume(candidate) => candidate.key(), } } /// Whether it was found in key order rather than arrival order. pub(super) fn ordered(&self) -> bool { match self { Self::Plain(peeked) => peeked.arrival.is_none(), Self::Resume(candidate) => candidate.ordered(), } } } /// One poll's view of how far this subscription may drift: the clamped budget and the /// live edge to measure a candidate group against. Resolved once, then applied to every /// group that poll considers. #[derive(Clone)] struct Drift { budget: Duration, /// The budget is the track's max age bound, which no widening passes: its verdicts are final. bounded: bool, edge: Edge, /// The groups this reader reads; see [`Coverage`]. coverage: Coverage, } /// The groups a reader reads, and the exclusive end it stops at, or `None` before its ranges /// resolve. type Coverage = Option<(Cover, Option)>; /// Whether a reader judging under `budget` with `coverage` is settled by `hole` over a run /// from `start`. Any hole but a received stale one settles every reader. A stale verdict /// holds for a reader no wider than the budget it was judged under that reads every group /// above the run's start: its own successor and edge are then no older than those the /// verdict was judged against, whatever ranges the upstream judged it under, since a group's /// verdict looks only above it. Any other reader treats the run as unknown, and fills it on /// a request of its own: the upstream subscription never delivers a group it dropped. fn settles(budget: Duration, coverage: &Coverage, start: u64, hole: &Hole) -> bool { let Some(stale) = hole.stale_budget() else { return true; }; let Some((groups, cap)) = coverage else { return false; }; let Some(above) = start.checked_add(1) else { return budget <= stale; }; // Whether the reader reads every group of `run`, an exclusive end of `u64::MAX` past the // live edge. let reads = |run: ops::Range| { let open = run.end == u64::MAX; if cap.is_some_and(|cap| open || run.end > cap) { return false; } groups.first(run.start, run.start).is_some_and(|held| match open { true => *held.end() == u64::MAX, false => *held.end() >= run.end - 1, }) }; budget <= stale && reads(above..u64::MAX) } /// The starts at which a received stale run settles a reader whose coverage grew from `old` /// to `new` under the same budget and did not before, or `None` inside for none: `None` when /// the coverage changed other than by growing. See [`settles`]: only an uncapped reader is /// settled, by a run whose next key the open run of its groups holds, and that run's start /// only falls as the groups grow. fn settling(old: &Coverage, new: &Coverage) -> Option>> { let ((old, old_cap), (new, cap)) = match (old, new) { (None, None) => return Some(None), (Some(old), Some(new)) => (old, new), _ => return None, }; if old_cap != cap || !old.gained(new, 0, u64::MAX).is_empty() { return None; } if cap.is_some() { return Some(None); } // The lowest start settled: the one just below the open run, or only the last key. let settled = |groups: &Cover| groups.open_from().map_or(u64::MAX, |start| start.saturating_sub(1)); let run = settled(new)..settled(old); Some((!run.is_empty()).then_some(run)) } impl Drift { /// Whether `hole`, over a run from `start`, settles this reader; see [`settles`]. fn settles(&self, start: u64, hole: &Hole) -> bool { settles(self.budget, &self.coverage, start, hole) } } /// Keeps one handed-out group tied to the subscription whose cursor selected it. struct GroupExpiry { state: kio::ConsumerWeak, /// The track's cache account, whose wakes judge a group with nothing above it without /// locking `state`. cache: Arc, reading: Reading, subscription: kio::Consumer, /// What the cursor can be served; see [`Cursor::published`]. servable: kio::Consumer, key: u64, superseded: Superseded, } /// Keeps one fetched group tied to this caller's age budget. struct FetchExpiry { state: kio::ConsumerWeak, /// See [`GroupExpiry::cache`]. cache: Arc, reading: Reading, key: u64, priority: i8, max_delay: Option, superseded: Superseded, } /// The track demand a handed-out group holds for its readers, and the frames they take, /// until one of them reads the group's end: from then on the track owes it nothing. struct Reading(Mutex>>); impl Reading { fn new(track: kio::Consumer) -> Self { Self(Mutex::new(Some(track))) } /// The same hold on the copy a reader moved to, unless the group already ended. fn moved(&self, track: &kio::ConsumerWeak) -> Self { let held = self.0.lock().unwrap_or_else(PoisonError::into_inner).is_some(); Self(Mutex::new(held.then(|| track.consume()))) } fn release(&self) { let track = self.0.lock().unwrap_or_else(PoisonError::into_inner).take(); // Dropped outside the lock: the last reader leaving wakes the track's watchers. drop(track); } } /// The successor a handed-out group was last judged by, so the group stays judgeable once /// the cache evicts both; see [`TrackState::superseded_at`]. Each reader keeps its own, so /// the memory is bounded by the groups readers hold. #[derive(Default)] struct Superseded(Mutex>); /// A successor, when it arrived, and whether it has since failed. struct Successor { key: u64, arrived: crate::time::Instant, fate: group::Fate, } impl group::Expiry for FetchExpiry { fn for_track(&self, track: &Consumer) -> Arc { // A replacement copy is always one track instance; the fallback is never read. let state = track.with_state(|state| state.cloned()); let (state, cache) = match state { Some(state) => { let cache = state.read().cache.clone(); (state, cache) } None => (self.state.clone(), self.cache.clone()), }; Arc::new(Self { reading: self.reading.moved(&state), state, cache, key: self.key, priority: self.priority, max_delay: self.max_delay, superseded: Superseded::default(), }) } fn asking(&self) -> Asking { Asking { priority: self.priority, max_delay: self.max_delay, order: Order::Ascending, } } fn ended(&self) { self.reading.release(); } fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. if !self.cache.wakes().watch_landing(self.key, waiter) { return false; } let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); let state = self.state.read(); let budget = match (self.max_delay, max_delay) { (Some(fetch), Some(route)) => fetch.min(route), (Some(fetch), None) => fetch, (None, Some(route)) => route, (None, None) => Duration::MAX, }; let budget = clamp_max_delay(budget, state.max_age_bound()); state.poll_drifted(self.key, &Servable::ALL, budget, Some(&self.superseded), waiter) } } impl group::Expiry for GroupExpiry { fn for_track(&self, track: &Consumer) -> Arc { // A replacement copy is always one track instance; the fallback is never read. let state = track.with_state(|state| state.cloned()); let (state, cache) = match state { Some(state) => { let cache = state.read().cache.clone(); (state, cache) } None => (self.state.clone(), self.cache.clone()), }; Arc::new(Self { reading: self.reading.moved(&state), state, cache, subscription: self.subscription.clone(), servable: self.servable.clone(), key: self.key, superseded: Superseded::default(), }) } fn asking(&self) -> Asking { let subscription = self.subscription.read(); Asking { priority: subscription.priority, max_delay: subscription.delay.max, order: subscription.order, } } fn ended(&self) { self.reading.release(); } /// A group nothing ever landed above has no successor, so neither clock can convict it: /// that verdict, the common one at the live edge, takes neither the track's lock, the /// subscription's, nor a logical reader's. Nothing it skips parking on can move it until /// a group lands above, which wakes the read; a reader's preferences reach its copies /// without this poll, since every update mirrors them at once. fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. if !self.cache.wakes().watch_landing(self.key, waiter) { return false; } let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); let max_delay = max_delay.unwrap_or_else(|| { let mut max_delay = Duration::default(); let _ = self.subscription.poll(waiter, |subscription| { max_delay = subscription.budget(); Poll::<()>::Pending }); max_delay }); let mut servable = Servable::ALL; let _ = self.servable.poll(waiter, |current| { servable = current.servable.clone(); Poll::<()>::Pending }); let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); state.poll_drifted(self.key, &servable, budget, Some(&self.superseded), waiter) } } /// The group a poll's drift is measured against, identified well enough to tell it apart /// from whatever may occupy its key by the time a candidate is judged. #[derive(Clone)] struct Edge { /// This track's own edge, revalidated before it convicts anything. presentation: Option, /// What the edge was resolved within, so per-candidate reach lookups measure /// against the same servable window. servable: Servable, } /// What a cursor's arrival walk reads: its window from the floor, below the cap. Holds a /// snapshot of the window's cover, so taking it again after the window moves diffs the two /// snapshots and costs what moved, not every run of the window. #[derive(Clone, Default)] struct Admitted { /// The window's groups, `None` while a relative anchor waits and every group is in it. groups: Option, floor: u64, cap: Option, } impl Admitted { fn new(window: &Window, floor: u64, cap: Option) -> Self { Self { groups: window.groups().cloned(), floor, cap, } } /// The last group admitted, `None` for none. fn last(&self) -> Option { match self.cap { Some(cap) => cap.checked_sub(1).filter(|last| *last >= self.floor), None => Some(u64::MAX), } } /// The admitted runs reaching into `run`, clipped to it, lowest first. fn overlapping(&self, run: ops::RangeInclusive) -> Vec> { let Some(last) = self.last() else { return Vec::new(); }; let (lo, hi) = ((*run.start()).max(self.floor), (*run.end()).min(last)); if lo > hi { return Vec::new(); } match &self.groups { None => std::iter::once(lo..=hi).collect(), Some(groups) => groups .runs(lo) .take_while(|run| *run.start() <= hi) .map(|run| *run.start()..=(*run.end()).min(hi)) .collect(), } } /// Every admitted run, lowest first. fn runs(&self) -> Vec> { self.overlapping(0..=u64::MAX) } /// What this admits and `old` doesn't within `lo..=hi`, when neither waits on a relative /// anchor: `None` otherwise. Costs where the window, the floor, or the cap moved between /// them, never the runs that stayed. fn gained(&self, old: &Self, lo: u64, hi: u64) -> Option>> { let (groups, old_groups) = (self.groups.as_ref()?, old.groups.as_ref()?); let Some(last) = self.last() else { return Some(Vec::new()); }; let (lo, hi) = (lo.max(self.floor), hi.min(last)); if lo > hi { return Some(Vec::new()); } let mut moved: RangeInclusiveSet = groups.moved(old_groups).into_iter().collect(); if self.floor != old.floor { moved.insert(self.floor.min(old.floor)..=self.floor.max(old.floor) - 1); } let old_last = old.last(); match old_last { Some(old_last) if old_last != last => { moved.insert(old_last.min(last) + 1..=old_last.max(last)); } Some(_) => {} // The old admitted nothing, so all of this is new. None => moved.insert(lo..=hi), } let mut gained = Vec::new(); for span in moved.overlapping(lo..=hi) { for run in groups.within((*span.start()).max(lo), (*span.end()).min(hi)) { let (start, end) = (*run.start(), *run.end()); // What lies outside the old bounds is new whatever its runs held. let inside = old_last.map(|old_last| (start.max(old.floor), end.min(old_last))); match inside.filter(|(from, to)| from <= to) { None => gained.push(run), Some((from, to)) => { if start < from { gained.push(start..=from - 1); } gained.extend(super::subscription::minus(from..=to, old_groups)); if to < end { gained.push(to + 1..=end); } } } } } Some(gained) } } /// `runs`, lowest first, below the exclusive `cap`. /// What a reader asks its fills with: read only once it asks, since every probe that asks /// nothing would otherwise take the subscription's lock for it. fn asking(subscription: &Preferences) -> miss::Asking { let subscription = subscription.read(); miss::Asking { priority: subscription.priority, max_delay: subscription.delay.max, order: subscription.order, } } fn below(runs: impl Iterator>, cap: Option) -> RangeInclusiveSet { let mut below = RangeInclusiveSet::new(); let last = match cap { Some(end) => match end.checked_sub(1) { Some(last) => last, None => return below, }, None => u64::MAX, }; for run in runs { if *run.start() > last { break; } below.insert(*run.start()..=(*run.end()).min(last)); } below } /// A cursor's [`Servable`] as the groups it handed out see it, derived from its two /// inputs: the window, which a [`Control`] replacing the ranges republishes, and the cap, /// which only the cursor moves. Each writer replaces its own input and derives the servable /// under the same write, so neither can publish one cut from the other's stale input. #[derive(Clone, PartialEq)] struct Published { /// The window's groups, resolved for `revision` of the ranges. runs: Servable, /// The [`Preferences::ranges_revision`] `runs` were resolved for. revision: u64, cap: Option, /// `runs` below `cap`. servable: Servable, } impl Published { fn new(runs: Servable, revision: u64, cap: Option) -> Self { let servable = runs.below(cap); Self { runs, revision, cap, servable, } } } /// [`TrackState::live`]'s walk, one branch per call rather than a chain checking both on /// every step: every search for live content goes through it. enum LiveWalk { /// The cache, which holds only the live feed. Cached(C), /// The live index, each key looked up in the cache. Indexed(I), } impl, I: Iterator> Iterator for LiveWalk { type Item = T; fn next(&mut self) -> Option { match self { Self::Cached(cached) => cached.next(), Self::Indexed(indexed) => indexed.next(), } } } impl, I: DoubleEndedIterator> DoubleEndedIterator for LiveWalk { fn next_back(&mut self) -> Option { match self { Self::Cached(cached) => cached.next_back(), Self::Indexed(indexed) => indexed.next_back(), } } } /// The groups a reader can be served, the only ones its drift is measured against: one it /// never reads, past its cap or in a gap between its ranges, never stands in for one it does. #[derive(Clone, PartialEq)] enum Servable { /// From `start` up to the exclusive `end`, or onward without one: one run, the common /// case, kept as cheap to copy and look up as a cap. Its start still bounds a group /// handed out before the run moved above it, which is judged against the run. Run { start: u64, end: Option }, /// Several runs, as the window holds them. Runs(Arc), } /// A servable window of several runs: a snapshot of the window's cover, below the exclusive /// `cap`, with the reader's index of the live groups inside that cover. /// /// Built again whenever the window or cap moves, which a time end does on each group. The /// snapshot is a pointer clone, and the index carries over: a move of only the cap changes /// nothing in it, and a move of the cover costs the runs that moved. struct Several { groups: Cover, cap: Option, live: Arc, } impl PartialEq for Several { fn eq(&self, other: &Self) -> bool { self.groups == other.groups && self.cap == other.cap } } /// A reader's index of the live feed's cached groups inside its window's cover, so a /// search costs the live groups it passes, never the runs between them: thousands of runs /// in the gaps of a sparsely numbered feed would otherwise each cost a lookup. /// /// Taken in from the arrival order, which every live group enters once, on the reader's own /// search: each arrival costs it one lookup in the cover. A slot that leaves or is /// replaced leaves its entry stale, dropped when a search meets it or once stale entries /// could outnumber the live groups. /// /// One index per track searched: a group's expiry that moved to a replacement copy (see /// [`GroupExpiry::for_track`]) searches its reader's servable on that copy's track, whose /// arrival order and stamps are its own. /// /// Each index is for the cover it last searched: a search under an edited cover moves only /// the runs the edit moved, so a SUBSCRIBE_UPDATE costs its own ranges, never the cache. #[derive(Default)] struct Live(Mutex>); struct Indexed { /// The track indexed, by its cache account. Held weakly, so a track that went away /// leaves an entry another track's can't be mistaken for. track: Weak, /// The cover the entries were taken in for. cover: Cover, /// How far into the arrival order this has taken in, as an absolute index. taken: usize, /// The stamp of each live group inside the cover, by key, as it arrived. groups: BTreeMap, } /// How many index entries a search takes per lock: the search answers with the lock /// released, since the answer reads each group's own state. const LIVE_BATCH: usize = 16; impl Indexed { /// Whether this indexes `state`'s track. fn indexes(&self, state: &TrackState) -> bool { std::ptr::eq(self.track.as_ptr(), Arc::as_ptr(&state.cache)) } /// Take in what arrived since the last call, under `groups`. fn catch_up(&mut self, state: &TrackState, groups: &Cover) { if self.taken == 0 { // Nothing taken in yet: every live group has an arrival entry, so the walk below // takes in all of them under `groups`. self.cover = groups.clone(); } else if self.cover != *groups { self.recover(state, groups); } for entry in state.arrivals_from(self.taken) { if groups.contains(entry.key) { self.groups.insert(entry.key, entry.stamp); } } self.taken = state.next_arrival; // A live entry is a visible slot, so past twice their count most entries are stale, // and a prune costs each entry it drops one lookup. if self.groups.len() > 2 * state.visible.len().max(LIVE_BATCH) { self.groups.retain(|key, stamp| current(state, *key, *stamp).is_some()); } } /// Move the entries to `groups` from the cover they were taken in for: only where the /// two differ, which for an edited copy costs the runs the edit moved and the live groups /// in them. A group not yet taken in from the arrival order still is, as an entry again. fn recover(&mut self, state: &TrackState, groups: &Cover) { let old = std::mem::replace(&mut self.cover, groups.clone()); for span in groups.moved(&old) { let (lo, hi) = (*span.start(), *span.end()); for run in old.within(lo, hi) { for lost in minus(run, groups) { let keys: Vec = self.groups.range(lost).map(|(key, _)| *key).collect(); for key in keys { self.groups.remove(&key); } } } for run in groups.within(lo, hi) { for gained in minus(run, &old) { for (key, slot) in state.live(gained) { self.groups.insert(*key, slot.stamp); } } } } } } /// The live slot at `key`, if it is still the incarnation stamped `stamp`. fn current(state: &TrackState, key: u64, stamp: u32) -> Option<&Slot> { state .lookup .get(&key) .filter(|slot| slot.stamp == stamp && slot.visible) } impl Live { /// The index of `state`'s track among `indexes`, made on its first search. Indexes of /// tracks that went away are dropped then. fn index<'a>(indexes: &'a mut Vec, state: &TrackState) -> &'a mut Indexed { let position = match indexes.iter().position(|indexed| indexed.indexes(state)) { Some(position) => position, None => { indexes.retain(|indexed| indexed.track.strong_count() > 0); indexes.push(Indexed { track: Arc::downgrade(&state.cache), cover: Cover::default(), taken: 0, groups: BTreeMap::new(), }); indexes.len() - 1 } }; &mut indexes[position] } /// The first live group inside `groups`, within `lo..=hi`, for which `f` answers, lowest /// first or, with `rev`, highest first, with its key. fn find<'a, R>( &self, state: &'a TrackState, groups: &Cover, mut lo: u64, mut hi: u64, rev: bool, f: &mut impl FnMut(&'a Slot) -> Option, ) -> Option<(u64, R)> { let mut caught_up = false; loop { if lo > hi { return None; } let mut batch = [(0, 0); LIVE_BATCH]; let taken = { let mut indexes = self.0.lock().unwrap_or_else(PoisonError::into_inner); let indexed = Self::index(&mut indexes, state); if !caught_up { indexed.catch_up(state, groups); caught_up = true; } let entries = indexed.groups.range(lo..=hi).map(|(&key, &stamp)| (key, stamp)); let mut taken = 0; let mut put = |entry| { batch[taken] = entry; taken += 1; }; match rev { true => entries.rev().take(LIVE_BATCH).for_each(&mut put), false => entries.take(LIVE_BATCH).for_each(&mut put), } taken }; let batch = &batch[..taken]; let mut stale = Vec::new(); let mut found = None; for &(key, stamp) in batch { match current(state, key, stamp) { Some(slot) => { if let Some(answer) = f(slot) { found = Some((key, answer)); break; } } None => stale.push(key), } } if !stale.is_empty() { let mut indexes = self.0.lock().unwrap_or_else(PoisonError::into_inner); let indexed = Self::index(&mut indexes, state); for key in stale { indexed.groups.remove(&key); } } if found.is_some() || taken < LIVE_BATCH { return found; } let &(last, _) = batch.last().expect("a full batch"); match rev { true => hi = last.checked_sub(1)?, false => lo = last.checked_add(1)?, } } } } impl Servable { /// Every group, for a reader no window narrows. const ALL: Self = Self::Run { start: 0, end: None }; /// No group. const NONE: Self = Self::Run { start: 0, end: Some(0) }; /// Whether group `key` is among these. fn contains(&self, key: u64) -> bool { match self { Self::Run { start, end } => key >= *start && end.is_none_or(|end| key < end), Self::Runs(several) => several.cap.is_none_or(|cap| key < cap) && several.groups.contains(key), } } /// The groups `window` reads, searched through `live`, the reader's index of the live /// groups, which follows the cover it is searched under. fn of(window: &Window, live: &Arc) -> Self { match window.groups() { None => Self::ALL, Some(groups) => Self::covering(groups.clone(), None, live), } } /// The groups in `groups` below the exclusive `cap`. Only several runs allocate: every /// subscriber resolves one, so the common single run stays off the heap. fn covering(groups: Cover, cap: Option, live: &Arc) -> Self { let last = match cap { Some(cap) => match cap.checked_sub(1) { Some(last) => last, None => return Self::NONE, }, None => u64::MAX, }; let (first, more) = { let mut runs = groups.runs(0).take_while(|run| *run.start() <= last); (runs.next(), runs.next().is_some()) }; let Some(first) = first else { return Self::NONE; }; if !more { return Self::Run { start: *first.start(), end: (*first.end()).min(last).checked_add(1), }; } Self::Runs(Arc::new(Several { groups, cap, live: live.clone(), })) } /// These groups below the exclusive `cap`. fn below(&self, cap: Option) -> Self { let Some(cap) = cap else { return self.clone(); }; match self { Self::Run { start, end } => { let end = end.map_or(cap, |end| end.min(cap)); match end > *start { true => Self::Run { start: *start, end: Some(end), }, false => Self::NONE, } } Self::Runs(several) => { let cap = several.cap.map_or(cap, |own| own.min(cap)); Self::covering(several.groups.clone(), Some(cap), &several.live) } } } } /// The newest servable group that has presented at least one frame. #[derive(Clone, Copy)] struct PresentationEdge { key: u64, /// The slot incarnation this timestamp was read from, so an eviction or a re-served /// key between resolving the edge and using it is detectable. stamp: u32, /// The newest frame this group has presented, not its first. The candidate side of the /// comparison is an upper bound on what a group could still reach, so the edge side has /// to be the newest content that actually exists, or the two meet and nothing is ever /// convicted. Only ever grows as the group fills. timestamp: Timestamp, } impl PresentationEdge { /// Whether a group that can present until `reach` is a full `budget` behind this edge. fn behind(&self, reach: Option, budget: Duration) -> bool { reach .and_then(|reach| self.timestamp.checked_sub(reach).ok()) .and_then(age) .is_some_and(|age| age >= budget) } } /// A span of presentation time as a duration: a scale-less track's bare units count as /// milliseconds, over the whole key range. fn age(span: Timestamp) -> Option { match span.scale() { Some(_) => Duration::try_from(span).ok(), None => Some(Duration::from_millis(span.value())), } } /// The read cursor behind a [`Subscriber`]. struct Cursor { state: kio::Consumer, /// The track's cache account, handed to each group's [`GroupExpiry`] without locking /// `state` per delivery. cache: Arc, subscription: Preferences, /// Arrival-order cursor used by `recv_group`. index: usize, /// Minimum key to return from any `recv` method. Set by `set_range`. min_key: u64, /// This subscriber's own ranges, resolved. Boxed: every reader embeds a cursor. filter: Box, /// One past the highest key returned by `next_group`. /// Used only by that method to skip late arrivals; does not affect `recv_group`. next_key: u64, /// The absolute arrival index [`Self::late`] has read up to. late: usize, /// Exclusive upper key bound for `next_group` and `recv_group`. `None` means no /// cap. Set by `set_range`; /// can be raised, lowered, or unset at any time. Groups at or past the cap stay in /// the producer's cache and become eligible again when the cap rises (or is removed). /// `Some(0)` is the empty range. end_key: Option, /// What this cursor can be served, the only groups its drift is measured within: its /// window below its cap, so a range behind the live edge is judged against its own /// newest group, and a group in a gap between ranges against nothing. servable: Servable, /// [`Self::servable`], shared with the groups this cursor hands out so their expiry /// measures drift against the same groups after the cap or ranges move, and with its /// [`Control`]s, which republish it when they replace the ranges. published: kio::Producer, /// This cursor's index of the live groups its servable window covers, kept across every /// move of the window, so an edit to the ranges costs the runs it moved. live: Arc, /// Groups the drift budget skipped since the count was last drained. Accumulated /// here rather than metered in place because the stats scope lives on the owning /// [`Subscriber`], which drains this after every read. stale: stats::Content, /// Groups the seek path has convicted but whose keys no caller has committed /// past yet, keyed by key, with the budget that convicted each; see /// [`Self::commit_seek_stale`]. seek_pending: BTreeMap, /// What this cursor handed out, skipped, gave up, and reported. record: Record, /// The drops a wire publisher forwards, once it first polls for one. drops: Option>, /// What this cursor misses and asks fills for, once it first reads. misses: Option>, /// The heads a bounded window asked for, once it first finds a group missing one. heads: Option>, /// The group read that last parked, answered again without the walk until something /// it parked on fires, once it first parks. idle: Option>>, /// [`Self::state`] for [`Self::poll_finished`], which a publisher re-polls on every /// wake: one the track has not changed since skips the lock. Made on the first poll. finished: Option>>, } /// A time start a [`Cursor`]'s window looks up through a fill. struct Lookup { time: u64, /// The fill looking it up, `None` once its answer was taken in. join: Option, } /// A group a cursor found but has not moved past; see [`Cursor::take`]. pub(super) struct Peeked { group: group::Consumer, /// How taking it moves the cursor: an arrival read past where it came from, or a /// key read past its key. arrival: Option, } /// Where a recv-order candidate came from, which taking it advances past. #[derive(Clone, Copy)] enum Taken { /// The arrival walk, which moves to this index. Walk(usize), /// What a widening owes. Owed, /// What a fill answered. Filled, /// A head a bounded window asked for. Head, } /// The groups a bounded window read missing their head, which it hands out only once a /// fill answered from their first frame: a whole-group read never starts short. #[derive(Default)] struct Heads { /// Asked, by key. fetching: BTreeMap, /// Answered and not handed out yet. filled: BTreeMap, } /// The keys a [`DropCursor`] reports from: a cursor's span, narrowed to its window, and /// the drift that says which received stale runs settle it. struct Within<'a> { span: ops::Range, window: &'a Window, revision: u64, drift: Option<&'a Drift>, } /// The span, window revision, and the budget and coverage that settle received stale runs /// (see [`Drift::settles`]) a [`DropCursor`] last scanned the whole span under; between them /// it folds in only the runs that changed, and what the window gained. type Scan = (ops::Range, u64, Option<(Duration, Coverage)>); /// What a wire publisher's cursor gave up, found as it changes rather than by rescanning /// the cache on every poll. struct DropCursor { /// Every group the publisher cuts. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] cuts: super::aborts::Watch, /// What [`Record::pending`] last folded in; see [`Scan`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] scanned: Option, /// The [`TrackState::drops_changes`] revision [`Record::pending`] folded in. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] seen: u64, /// The window's groups as of [`Self::scanned`]'s revision, so a window that only grew /// since folds in what it gained rather than rescanning the span. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] groups: Option, /// The budget and coverage of the last drift judged, which settle received stale runs /// until the next: a queue-only poll judges none, and must not unsettle them. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] drift: Option<(Duration, Coverage)>, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl DropCursor { fn new(state: &TrackState) -> Self { // Watch before the first scan, so a cut landing between the two is still seen. Self { cuts: state.cache.watch_cuts(), scanned: None, seen: 0, groups: None, drift: None, } } /// Record a run this cursor's fills gave up on as a miss, as `hole` says; a reason /// already pending, such as the track's own drop, is kept. fn give_up(record: &mut Record, run: ops::Range, hole: Hole) { let gaps: Vec<_> = record.pending.gaps(&run).collect(); for gap in gaps { record.pending.insert(gap, Pending::own(hole.clone())); } } /// Record `slot`, cached at `key`, if it holds a cut no fetched copy covers, or a /// final drop a fetched head died before winning over: a copy still serving the group's /// end delivers it, so it is never also dropped. fn cut(record: &mut Record, key: u64, slot: &Slot) { if slot.visible && !slot.serves_end() && let Some(err) = slot.verdict() && let Some(end) = key.checked_add(1) { record .pending .insert(key..end, Pending::own(Hole::drop(StreamError::from(err)))); } } /// What a scan of `span` would newly find since the last, when the span is the same and /// the window and the reader's coverage only grew: the runs the window gained, and each /// received stale run a wider coverage now settles. `None` when anything else moved. fn grown( &self, state: &TrackState, window: &Window, old_drift: Option<&(Duration, Coverage)>, span: &ops::Range, ) -> Option>> { let last = span.end.checked_sub(1).filter(|last| *last >= span.start); let (groups, old) = (window.groups()?, self.groups.as_ref()?); let mut grown: Vec<_> = match last { // Losing a group of the span may leave a reported run no longer owed. Some(last) if old.gained(groups, span.start, last).is_empty() => groups.gained(old, span.start, last), Some(_) => return None, None => Vec::new(), } .into_iter() .map(|run| *run.start()..run.end().saturating_add(1)) .collect(); let settling = match (old_drift, &self.drift) { (None, None) => None, (Some((old_budget, old)), Some((budget, new))) if old_budget == budget => settling(old, new)?, _ => return None, }; // A scan judges a drop at its start within the span, so each one starting where a // stale run newly settles is folded in whole. if let Some(settling) = settling { for (run, _) in state.drops.overlapping(&settling) { let start = run.start.max(span.start); if settling.contains(&start) { grown.push(start..run.end); } } } Some(grown) } /// The lowest unreported run of `within` given up, now marked reported, with why. fn next( &mut self, record: &mut Record, state: &TrackState, cut: BTreeSet, within: &Within<'_>, ) -> Option<(ops::Range, Hole)> { let (span, window) = (&within.span, within.window); // A cut already reclaimed from the cache is in the track's drops instead. for key in cut { if let Some(slot) = state.lookup.get(&key) { Self::cut(record, key, slot); } } // Rescan the span only after the window moves other than by growing, or what settles a // stale run does, since out-of-window and unsettling runs were skipped and may be owed // now; otherwise fold in only the runs that changed and what the window gained, so a // drop costs each cursor the runs it touches, even one that starts a range. if let Some(drift) = within.drift { self.drift = Some((drift.budget, drift.coverage.clone())); } let scan = (span.clone(), within.revision, self.drift.clone()); let rescan = std::mem::take(&mut record.rescan); let grown = match &self.scanned { _ if rescan => None, Some(scanned) if *scanned == scan => Some(Vec::new()), Some((old_span, _, old_drift)) if old_span == span => self.grown(state, window, old_drift.as_ref(), span), _ => None, }; let full = grown.is_none(); self.scanned = Some(scan); self.groups = window.groups().cloned(); let changed: Vec> = match grown { Some(grown) => { // What the window newly reads may hold cuts the scan skipped. for run in &grown { for (&key, slot) in state.lookup.range(run.clone()) { Self::cut(record, key, slot); } } state.drops_changes.since(self.seen).chain(grown).collect() } None => { let received: Vec<_> = record .pending .iter() .filter(|(_, pending)| pending.received) .map(|(run, _)| run.clone()) .collect(); for run in received { record.pending.remove(run); } vec![span.clone()] } }; self.seen = state.drops_changes.revision(); for changed in changed { let changed = changed.start.max(span.start)..changed.end.min(span.end); if changed.is_empty() { continue; } let received: Vec<_> = record .pending .overlapping(&changed) .filter(|(_, pending)| pending.received) .map(|(run, _)| run.start.max(changed.start)..run.end.min(changed.end)) .collect(); for run in received { record.pending.remove(run); } for (dropped, hole) in state.drops.overlapping(&changed) { let run = dropped.start.max(changed.start)..dropped.end.min(changed.end); // A received stale run that does not settle this reader is unknown to it. let settled = match &self.drift { Some((budget, coverage)) => settles(*budget, coverage, run.start, hole), None => hole.stale_budget().is_none(), }; if settled { record.pending.insert( run, Pending { hole: hole.clone(), received: true, }, ); } } } if full { for (&key, slot) in state.lookup.range(span.clone()) { Self::cut(record, key, slot); } } while let Some((range, pending)) = record .pending .iter() .next() .map(|(range, pending)| (range.clone(), pending.clone())) { record.pending.remove(range.clone()); // A received run the track has since forgotten, shed or superseded, is unknown // again: only the part the track still holds as it was folded in is owed. let range = match pending.received { true => { let Some((held, _)) = state.drops.overlapping(&range).find(|(_, hole)| **hole == pending.hole) else { continue; }; let held = range.start.max(held.start)..range.end.min(held.end); if held.end < range.end { record.pending.insert(held.end..range.end, pending.clone()); } held } false => range, }; // Only what this subscriber asked for is owed: the first run of its ranges in it, // the rest of the range waiting its turn. let Some(clipped) = window.clip(range.start.max(span.start)..range.end.min(span.end)) else { continue; }; if clipped.end < range.end { record.pending.insert(clipped.end..range.end, pending.clone()); } if let Some(gap) = record.report(clipped, &pending.hole) { if gap.end < range.end { record.pending.insert(gap.end..range.end, pending.clone()); } return Some((gap, pending.hole)); } } None } } /// What changed in a cursor's ranges since its window resolved them. enum Change { /// The ranges keyed so, each as it stands now: `None` for one retired. Edit(Vec<(u64, Option)>), /// Every range, in key order, when the changes are no longer known one by one. All(Vec<(u64, Range)>), } /// A cursor's own ranges, resolved, and what keeps them current. struct Filter { /// Only the groups in it are read. window: Window, /// The [`Preferences::ranges_revision`] the window was resolved from. resolved: u64, /// Bumped whenever the window moves, so the drop scan knows to look again. revision: u64, /// Why the ranges can't be served here: a time anchor on an untimed track. refused: Option, /// The window was left as it was by a refused change, so the next one resolves every range. stale: bool, /// What the arrival walk reads: the window from the floor, below the cap. The walk /// passes a group outside it holding nothing, so widening it re-scans the cache. admitted: Admitted, /// The floor, [`Self::revision`], and cap [`Self::admitted`] was taken at. admitted_at: Option<(u64, u64, Option)>, /// The spans a widening newly covers, from the ranges, the floor, the cap, or the /// budget, scanned in the cache in the subscription's order and handed out ahead of the /// walk: each cached group in them is visited once. owed: RangeInclusiveSet, /// The time starts [`Self::window`] looks up from the track's history, each asked once. lookups: Vec, } impl Cursor { fn new(state: kio::Consumer, subscription: Preferences) -> Self { // The ranges floor the cursor, raised for a live join by its Min Age, and the budget // picks where above it to start, timed or not: delivery skips the cached groups it // already judges stale, on whichever clock convicts them. if subscription.read().joins_live() { TrackState::ask(state.read(), Ask::Join); } let (filter, cache) = { let preferences = subscription.read(); let ranges_revision = preferences.revision; let state = state.read(); let (window, refused) = match Window::keyed( preferences.ranges(), preferences.delay.min, state.latest_live(), &*state, ) { Ok(window) => (window, None), Err(err) => ( Window::new(&[], None, None, &*state).expect("nothing to refuse"), Some(err), ), }; let filter = Filter { window, resolved: ranges_revision, revision: 0, stale: refused.is_some(), refused, admitted: Admitted::default(), admitted_at: None, owed: RangeInclusiveSet::new(), lookups: Vec::new(), }; (filter, state.cache.clone()) }; let live = Arc::::default(); let published = Published::new(Servable::of(&filter.window, &live), filter.resolved, None); Self { state, cache, subscription, min_key: 0, filter: Box::new(filter), index: 0, next_key: 0, late: 0, end_key: None, // Uncapped, the servable groups are the window's. servable: published.servable.clone(), published: kio::Producer::new(published), live, stale: stats::Content::default(), seek_pending: BTreeMap::new(), record: Record::default(), drops: None, misses: None, heads: None, idle: None, finished: None, } } /// The next run given up, judging the live edge first only when `probe`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn poll_drop(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { // What the group read walks, which this poll may move outside that read. let walks = (self.filter.revision, self.fill_view()); self.sync(); if self.drops.is_none() { self.drops = Some(Box::new(DropCursor::new(&self.state.read()))); if let Some(misses) = &self.misses { for run in misses.given_up().iter() { DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(None)); } } } let mut judged = None; if probe && self.filter.refused.is_none() && let Poll::Ready(Ok(drift)) = self.poll_drift(&self.servable, waiter) { // From where the reader reads next: an ordered one asks nothing below what it took. let from = self.next_key.max(self.floor()); self.probe(from, &drift, waiter); judged = Some(drift); } // The window settled, or a fill was asked or answered, on this poll's waiter rather // than the one the group read parked on: that read, and every gate it nests in, reads // again. if (self.filter.revision, self.fill_view()) != walks { idle::Idle::reopen(&mut self.idle); } // A cap at or below the floor leaves nothing to report, not a reversed span. let floor = self.floor(); let within = Within { span: floor..self.end_key.unwrap_or(u64::MAX).max(floor), window: &self.filter.window, revision: self.filter.revision, drift: judged.as_ref(), }; let drops = match &mut self.drops { Some(drops) => drops, None => self.drops.insert(Box::new(DropCursor::new(&self.state.read()))), }; // A cut closes its group without modifying the track state, so its own watch wakes // this. Taken before reading the track, so a cut after the read still wakes. let cut = drops.cuts.poll(waiter); let record = &mut self.record; match self.state.poll(waiter, |state| { drops .next(record, state, cut, &within) .map_or(Poll::Pending, Poll::Ready) }) { Poll::Ready(Ok(range)) => Poll::Ready(range), Poll::Ready(Err(state)) => drops .next(record, &state, BTreeSet::new(), &within) .map_or(Poll::Pending, Poll::Ready), Poll::Pending => Poll::Pending, } } /// The keys that landed in this cursor's window since it last looked, below `below`: a key /// cursor already read past them, so it never hands them out. fn late(&mut self, below: u64) -> Vec { let state = self.state.read(); let from = self.late; self.late = state.next_arrival; state .arrivals_from(from) .map(|entry| entry.key) .filter(|key| *key < below && self.filter.window.contains(*key)) .collect() } /// Resolve [`Self::servable`] again once the window or cap moved, and publish it to the /// groups already handed out. Costs the ranges, never the cache. fn update_servable(&mut self) { // A moved window or cap notifies nothing this cursor parked on. idle::Idle::disarm(&mut self.idle); let runs = Servable::of(&self.filter.window, &self.live); let (revision, cap) = (self.filter.resolved, self.end_key); self.servable = runs.below(cap); let next = |current: &Published| { // Ranges a `Control` republished past this window stay until this cursor syncs // to them; the cap is this cursor's own. let runs = match current.revision > revision { true => current.runs.clone(), false => runs.clone(), }; let next = Published::new(runs, revision.max(current.revision), cap); (next != *current).then_some(next) }; // A no-op write is skipped: every handed-out group's expiry watches the channel. if next(&self.published.read()).is_none() { return; } if let Ok(mut current) = self.published.write() && let Some(next) = next(¤t) { *current = next; } } /// Bring [`Self::window`] up to date: the ranges the subscriber changed resolved, counting /// relative anchors back from the latest group then, and settled as the groups it waits on /// arrive. Returns whether the window moved. fn sync(&mut self) -> bool { let (changed, min) = { let preferences = self.subscription.read(); let changed = (preferences.revision != self.filter.resolved).then(|| { let keys = match self.filter.stale { true => None, false => preferences.changes(self.filter.resolved), }; let change = match keys { Some(keys) => Change::Edit(keys.into_iter().map(|key| (key, preferences.range(key))).collect()), None => Change::All(preferences.ranges().collect()), }; (preferences.revision, change) }); (changed, preferences.delay.min) }; // A settled window has nothing to read from the track, whose lock every subscriber // of it takes. if changed.is_none() && self.filter.window.is_settled() { return false; } let state = self.state.read(); let holdings = &*state; let moved = match changed { Some((revision, change)) => { self.filter.resolved = revision; let applied = match change { Change::Edit(changes) => { let added: Vec<(u64, Range)> = changes .iter() .filter_map(|(key, range)| Some((*key, (*range)?))) .collect(); self.filter.window.set_min(min); let applied = self.filter.window.edit( changes.iter().map(|(key, _)| *key), &added, state.latest_live(), holdings, ); // A time looked up and answered is asked again for a range that names it anew. self.filter.lookups.retain(|lookup| lookup.join.is_some()); applied } Change::All(ranges) => Window::keyed(ranges, min, state.latest_live(), holdings).map(|window| { self.filter.window = window; self.filter.lookups.clear(); }), }; // A refused change leaves the window as it was, which no longer matches the ranges, // so the next change resolves them all. (self.filter.refused, self.filter.stale) = match applied { Ok(()) => (None, false), Err(err) => (Some(err), true), }; true } None if self.filter.window.is_settled() => false, // The live feed had no group when the ranges resolved, so the newest one then was // the first to arrive, or the newest once it went live, however many have arrived // or left since. A live join not yet started reaches back by the Min Age stated now. None if self.filter.window.is_pending() => { self.filter.window.set_min(min); self.filter.window.settle(state.first_live(), holdings) } None => { // A live join waiting on a group to measure the live edge by has not started either. self.filter.window.set_min(min); self.filter.window.settle(state.latest_live(), holdings) } }; drop(state); if moved { self.filter.revision += 1; self.update_servable(); } moved } /// Settle a parked read's window on the groups that arrived since: a window waiting on a /// group parks on the track, which every arrival wakes. Returns whether the window /// moved, so the read goes again rather than parking. fn watch_time(&mut self) -> bool { if self.filter.window.is_settled() || self.filter.refused.is_some() { return false; } self.sync() } /// The lowest group this cursor reads: its own floor, raised to its window's. fn floor(&self) -> u64 { self.min_key.max(self.filter.window.floor().unwrap_or(0)) } /// Whether a bounded window has nothing left to deliver: every group in its ranges was /// handed out (in any of `taken`), dropped, skipped as stale, or given up as unfillable. /// An open window, or one with a fill still out, always has more, and so does one its /// cap keeps a group out of, since a reader may raise a cap again. Groups under `from` are /// not judged: a time start resolves to the group containing it, which may lie under a /// copy's resume point. fn is_exhausted(&self, logical: &[&Delivered], from: u64) -> bool { let taken: Vec<&Delivered> = std::iter::once(&self.record.delivered) .chain(logical.iter().copied()) .collect(); let Some(end) = self.filter.window.end() else { return false; }; if self.filter.window.is_pending() || self.awaits_fill() { return false; } let given_up = self.misses.as_ref().map(|misses| misses.given_up()); let coverage = self.coverage(); let state = self.state.read(); let budget = clamp_max_delay(self.subscription.read().budget(), state.max_age_bound()); let accounted = |gap: ops::Range| { let mut left = RangeSet::new(); left.insert(gap.clone()); for run in self.record.written_off.overlapping(&gap) { left.remove(run.clone()); } if let Some(given_up) = given_up { for run in given_up.overlapping(&gap) { left.remove(run.clone()); } } // A received stale run that does not settle this reader is unknown to it, so it // still wants those groups. for (run, _) in state .drops .overlapping(&gap) .filter(|(run, hole)| settles(budget, &coverage, run.start.max(gap.start), hole)) { left.remove(run.clone()); } for key in self.seek_pending.range(gap).map(|(key, _)| *key) { left.remove(key..key + 1); } left.is_empty() }; self.filter .window .runs(self.filter.window.floor().unwrap_or(0).max(from)) .take_while(|run| *run.start() < end) .all(|run| { let run = *run.start()..run.end().checked_add(1).map_or(end, |next| next.min(end)); Delivered::gaps_in_all(&taken, run).into_iter().all(&accounted) }) } /// Whether group `key`, at or above [`Self::floor`], is one this cursor reads now: /// in its window and below its cap. One that isn't may be later. fn admits(&self, key: u64) -> bool { super::subscription::before_end(key, self.end_key) && self.filter.window.contains(key) } /// [`Filter::owed`] as of `admitted`: what was owed and what the walk newly reads since /// [`Filter::admitted`] was taken, both cut to `admitted`. Costs every run of both. fn owed(&self, admitted: &Admitted) -> RangeInclusiveSet { // Nothing is owed before the walk first reads: it starts at the oldest arrival. if self.filter.admitted_at.is_none() { return RangeInclusiveSet::new(); } let mut owed = RangeInclusiveSet::new(); for run in admitted.runs() { let mut from = Some(*run.start()); for held in self.filter.admitted.overlapping(run.clone()) { if let Some(gap) = from.filter(|from| from < held.start()) { owed.insert(gap..=*held.start() - 1); } from = held.end().checked_add(1); } if let Some(gap) = from.filter(|from| from <= run.end()) { owed.insert(gap..=*run.end()); } for span in self.filter.owed.overlapping(&run) { owed.insert((*span.start()).max(*run.start())..=(*span.end()).min(*run.end())); } } owed } /// Take in a move of the floor, the ranges, or the cap: whatever it newly admits is /// owed. Costs the ranges, never the cache, and only when one moved: what they gain is /// owed only where the cache holds a group now, the only place a scan finds one, and /// what they lose is cut only from what is owed. fn admit(&mut self) { let at = (self.floor(), self.filter.revision, self.end_key); if self.filter.admitted_at == Some(at) { return; } // The arrival walk reads the window from the floor. let admitted = Admitted::new(&self.filter.window, self.floor(), self.end_key); let moved = self.filter.admitted_at.and_then(|_| { let cached = { let state = self.state.read(); state .lookup .first_key_value() .map(|(first, _)| *first) .zip(state.lookup.last_key_value().map(|(last, _)| *last)) }; let gained = match cached { Some((first, last)) => admitted.gained(&self.filter.admitted, first, last)?, None => admitted.gained(&self.filter.admitted, 1, 0)?, }; let owed = self .filter .owed .first() .map(|run| *run.start()) .zip(self.filter.owed.last().map(|run| *run.end())); let lost = match owed { Some((first, last)) => self.filter.admitted.gained(&admitted, first, last)?, None => Vec::new(), }; Some((gained, lost)) }); match moved { Some((gained, lost)) => { for run in lost { self.filter.owed.remove(run); } for run in gained { self.filter.owed.insert(run); } } None => self.filter.owed = self.owed(&admitted), } self.filter.admitted = admitted; self.filter.admitted_at = Some(at); } /// Move each group written off as stale that nothing can offer again, since its group /// left the cache or was judged at the bound, into `taken`, which compacts what it /// accounts for below its floor. Only a group a wider budget may still offer stays /// written off, so the write-offs never outnumber the stale skips the cache holds. fn retire_written_off(&mut self) { // Only a released key can settle: every other write-off is still skipped. Taken even // with nothing written off, so the released keys never pile up. let released = self.record.skipped.take_released(); // Most cursors never write a group off, and this runs on every hand-out. if self.record.written_off.is_empty() || released.is_empty() { return; } // By run, so a long stale run costs one lookup rather than one per group. let mut settled: Vec> = Vec::new(); for released in released.iter() { let span = *released.start()..released.end().saturating_add(1); for run in self.record.written_off.overlapping(&span) { let (start, end) = (run.start.max(span.start), run.end.min(span.end)); if start < end { let gaps = self.record.skipped.gaps(start..=end - 1); settled.extend(gaps.into_iter().map(|gap| *gap.start()..gap.end() + 1)); } } } for run in settled { self.record.written_off.remove(run.clone()); self.record.delivered.insert_run(run); } } /// The next cached group still owed in the subscription's order, newest first unless /// ascending, consuming the spans scanned on the way: only one the walk could have read, /// never fetched backfill or a withheld group. The group itself stays owed until /// [`Self::advance_arrival`] takes it, so a peek keeps it. fn next_owed(&mut self) -> Option { if self.filter.owed.is_empty() { return None; } let ascending = self.subscription.read().order == Order::Ascending; let state = self.state.read(); // Nothing is read from the cache now, so leave what is owed until it is. if !state.serves_bounded(&self.filter.window) { return None; } loop { let run = match ascending { true => self.filter.owed.first(), false => self.filter.owed.last(), } .cloned()?; let mut slots = state.lookup.range(run.clone()).map(|(_, slot)| slot); let found = match ascending { true => slots.find(|slot| slot.is_read()), false => slots.rev().find(|slot| slot.is_read()), } .map(Slot::serving); let Some(group) = found else { self.filter.owed.remove(run); continue; }; let scanned = match ascending { true => group.start.checked_sub(1).map(|below| *run.start()..=below), false => group.start.checked_add(1).map(|above| above..=*run.end()), }; if let Some(scanned) = scanned.filter(|scanned| !scanned.is_empty()) { self.filter.owed.remove(scanned); } return Some(group.clone()); } } /// Record `key` as skipped stale under `budget`, so a wider one owes it again. fn skip(&mut self, key: u64, budget: Duration) { self.record.skipped.insert(key, budget); self.prune_skipped(); } /// Forget the skipped runs no cached group is left in: nothing can offer them again. fn prune_skipped(&mut self) { // Most cursors never skip, and this runs on every hand-out. if self.record.skipped.is_empty() { return; } let state = self.state.read(); self.record.skipped.prune(state.lookup.len(), |run| { !state.lookup.range(run.clone()).any(|(_, slot)| slot.is_read()) }); } /// Owe again the groups this cursor skipped as stale under a budget narrower than /// `budget`: only a widening reverses a stale verdict it has not reported. Costs the /// skipped runs, never the cache, and nothing when the budget did not widen. fn reoffer(&mut self, budget: Duration) { for run in self.record.skipped.widened(budget) { self.reowe(run); } } /// Owe `run` of skipped groups again, judged anew, withdrawing a stale drop of it not /// reported yet; a reported one stands. fn reowe(&mut self, run: ops::RangeInclusive) { let span = *run.start()..run.end().saturating_add(1); self.record.written_off.remove(span.clone()); if self.drops.is_some() { let old: Vec<_> = self .record .pending .overlapping(span.clone()) .filter(|(_, pending)| pending.hole.stale_budget().is_some()) .map(|(range, _)| range.start.max(span.start)..range.end.min(span.end)) .collect(); for range in old { self.record.pending.remove(range); } } for owed in self.filter.admitted.overlapping(run) { self.filter.owed.insert(owed); } } // A helper to automatically apply Dropped if the state is closed without an error. fn poll(&self, waiter: &kio::Waiter, f: F) -> Poll> where F: Fn(&kio::Ref<'_, TrackState>) -> Poll>, { let polled = self.state.poll(waiter, |state| { let res = f(state); if res.is_pending() { // The end can settle with no write: when the grace for a missing group passes. state.watch_tail(waiter); if self.filter.window.is_reaching() { state.watch_starts(waiter); } } res }); Poll::Ready(match ready!(polled) { Ok(res) => res, // We try to clone abort just in case the function forgot to check for terminal state. Err(state) => Err(state.outcome.aborted().cloned().unwrap_or(Error::Dropped)), }) } /// Poll for the track's declared end, or the error it was cut short with. /// /// Through a [`kio::Watch`]: the end and the abort are plain fields of the track, so an /// unchanged version proves the answer unchanged. [`Self::poll`]'s tail watch is no loss /// here: a pending answer means no end, and the tail only waits once one is declared. fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { let watch = self .finished .get_or_insert_with(|| Box::new(kio::Watch::new(self.state.clone()))); Poll::Ready(match ready!(watch.poll(waiter, |state| state.poll_finished())) { Ok(res) => res, Err(state) => Err(state.outcome.aborted().cloned().unwrap_or(Error::Dropped)), }) } /// Whether a group read in `walk` order would park again at once; see [`idle`]. fn parked(&self, walk: idle::Walk, waiter: &kio::Waiter) -> bool { idle::Idle::parked(&self.idle, walk, waiter) } /// Take the groups skipped since the last call, for the owner to meter. fn take_stale(&mut self) -> stats::Content { std::mem::take(&mut self.stale) } /// The groups this reader reads, and its cap; see [`Coverage`]. fn coverage(&self) -> Coverage { Some((self.filter.window.groups()?.clone(), self.end_key)) } /// This subscriber's clamped drift budget and the live edge to measure against, /// resolved once per poll. /// /// `servable` bounds the edge: the caller passes the same window it reads from, so a /// group is only ever judged against content that could actually be served in its /// place. Read fresh each poll, so a mid-stream [`Control::update`] applies /// to the very next group, and shared across every candidate that poll walks off, so /// discarding a backlog of N groups costs one scan rather than N. Only ever /// [`Poll::Ready`]; the track ending surfaces as the error the caller was going to /// get anyway. fn poll_drift(&self, servable: &Servable, waiter: &kio::Waiter) -> Poll> { let mut max_delay = Duration::default(); let _ = self.subscription.poll_ref(waiter, |subscription| { max_delay = subscription.budget(); Poll::<()>::Pending }); let coverage = self.coverage(); self.poll(waiter, |state| { let bound = state.max_age_bound(); Poll::Ready(Ok(Drift { budget: clamp_max_delay(max_delay, bound), bounded: bound.is_some_and(|bound| max_delay >= bound), edge: state.drift_edge(servable), coverage: coverage.clone(), })) }) } /// Pending while a group in `groups` that the live feed still owes is within the drift /// budget, so a read in key order waits for it; see `TrackState::owed`. A budget /// with no deadline waits until the hole resolves: the group lands, or the track learns /// it won't. fn poll_owed(&self, groups: ops::Range, drift: &Drift, waiter: &kio::Waiter) -> Poll> { if groups.is_empty() { return Poll::Ready(Ok(())); } let window = &self.filter.window; self.poll(waiter, |state| { // An unjudgeable old gap cannot stand in for a fresh gap above the floor. let floor = groups.start.max(state.cache_floor(drift.budget)); if floor >= groups.end { return Poll::Ready(Ok(())); } let Some(owed) = state.owed(floor..groups.end, window) else { return Poll::Ready(Ok(())); }; if state.poll_drifted(owed, &drift.edge.servable, drift.budget, None, waiter) { return Poll::Ready(Ok(())); } // The ledger may stop owing it before the budget gives up on it. if let Some(folds) = state.arrivals.folds_at(owed, state.arrival_grace()) { state.cache.watch_wall(folds, waiter); } // Parked on the track too: the group landing, its drop, or a group landing // between it and the candidate, which can convict it sooner. Poll::Pending }) } /// Whether the drift budget says to skip `group`, against a [`Drift`] already resolved /// for this poll. fn poll_stale(&self, group: &group::Consumer, drift: &Drift, waiter: &kio::Waiter) -> Poll> { self.poll(waiter, |state| { Poll::Ready(Ok(state.is_stale(group.start, &drift.edge, drift.budget))) }) } /// Record `key` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. fn hand_out(&mut self, key: u64, ordered: bool) { if ordered { self.record.delivered.forget_below(key); } self.prune_skipped(); self.record.delivered.insert(key); let mut delivered = std::mem::take(&mut self.record.delivered); self.forget_past_budget(&mut delivered); self.record.delivered = delivered; self.retire_written_off(); } /// Forget the gaps in `delivered` below each group past this cursor's budget now, /// whether this track holds it or not: one landing there would be written off. A group /// skipped as stale is not forgotten, since a wider budget owes it again, and neither is /// one a widening still owes, which a newest-first re-offer has yet to judge. /// /// One edge for every group judged, so forgetting N gaps never resolves it N times. fn forget_past_budget(&self, delivered: &mut Delivered) { if delivered.above_gap().is_none() { return; } let owed = self.filter.owed.first().map(|run| *run.start()); let skipped = self.record.skipped.lowest().into_iter().chain(owed).min(); let max_delay = self.subscription.read().budget(); let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); let edge = state.drift_edge(&self.servable); let now = state.cache.pool().now_wall(); while let Some(oldest) = delivered.above_gap() { // Either clock, as `drifted` judges a group: without a successor, neither. let Some(arrival) = state.arrival(oldest, &edge.servable) else { break; }; let shared = if now.saturating_duration_since(arrival.arrived) >= budget { arrival.group.start } else if let Some(successor) = state.successor(oldest, &edge.servable) && let Some(live) = state.live_above(oldest, &edge) && live.behind(successor.group.timestamp(), budget) { successor.group.start.min(live.key) } else { break; }; // Every delivered run below this successor shares its verdict. Retire // them in one split, including the highest run, rather than once per gap. let last = delivered .last_before(shared) .expect("the oldest run is below its successor"); if let Some(skipped) = skipped && skipped < last { delivered.forget_below(skipped); break; } delivered.forget_below(last); } } /// Hand `group` out whole from its first frame, expiring against its budget. /// /// A group missing its head reads it as [`Error::Lagged`], which a route copy recovers /// from another. fn deliver(&self, group: group::Consumer) -> group::Consumer { let key = group.start; group.with_expiry(Arc::new(GroupExpiry { state: self.state.weak(), cache: self.cache.clone(), reading: Reading::new(self.state.clone()), subscription: self.subscription.consume(), servable: self.published.consume(), key, superseded: Superseded::default(), })) } /// Look up each time start the window can't resolve from the cache, once, and take in /// the answers, parking `waiter` on the lookups still out. /// /// Indexed by time, so a pass costs O(L log L) in the lookups out rather than O(L²). fn look_up(&mut self, waiter: &kio::Waiter) { // The first span looking a time up sets how far its lookup reaches. let mut lookups = BTreeMap::new(); for (time, reach) in self.filter.window.lookups() { lookups.entry(time).or_insert(reach); } if lookups.is_empty() && self.filter.lookups.iter().all(|lookup| lookup.join.is_none()) { return; } let asking = &asking(&self.subscription); let fills = self.state.read().fills.clone(); let asked: BTreeSet = self.filter.lookups.iter().map(|lookup| lookup.time).collect(); for (&time, &reach) in &lookups { if asked.contains(&time) { continue; } let ask = fill::Ask { start: Position::group(0), end: reach, frame_end: None, asking: *asking, time: Some(time), fetch: false, }; let (joins, _) = fills.lock().join(&fills, &ask); self.filter.lookups.push(Lookup { time, join: joins.into_iter().next(), }); } let mut moved = false; for lookup in &mut self.filter.lookups { // The window stops looking once replaced; with no join, nothing can answer. An // answer taken in below stops only the spans looking up that time. if !lookups.contains_key(&lookup.time) { lookup.join = None; continue; } // What the reader asks with still reaches the lookups it waits on. if let Some(join) = &mut lookup.join { join.update(asking); } let answer = match &lookup.join { None => None, Some(join) => match join.outcome().poll(waiter, |outcome| { outcome.first_accepted().map_or(Poll::Pending, Poll::Ready) }) { Poll::Ready(Ok(group)) => Some(group), // Ended unanswered: its handler couldn't tell. Poll::Ready(Err(closed)) => closed.first_accepted(), Poll::Pending => continue, }, }; let state = self.state.read(); lookup.join = None; moved |= self.filter.window.found(lookup.time, answer, &*state); } if moved { self.filter.revision += 1; self.update_servable(); } } /// Ask fills for what this cursor misses from `from`, and take in what they answered, /// parking `waiter` on the fills still out. Whatever it gives up goes to the drop cursor. /// `taken` is what its subscriber handed out, which an eviction never makes a miss. /// /// A live reader that takes each group as it arrives never misses one, so most probes /// find nothing to ask and hold no fill: those skip the fills, the subscription, and /// the answers entirely, paying only for the judgement that the new groups arrived. fn probe(&mut self, from: u64, drift: &Drift, waiter: &kio::Waiter) { self.look_up(waiter); let taken = &self.record.delivered; let (window, cap, revision) = (&self.filter.window, self.end_key, self.filter.revision); let misses = self.misses.get_or_insert_with(Box::default); let mut given_up = Vec::new(); misses.retain((from, revision, cap), window.missable_groups(), || { below(window.missable(from), cap) }); // One lock for the judgement and the fills it asks, since every subscriber of the // track probes through it. let (asks, fills) = { let state = self.state.read(); let asks = misses.judge( &state, (from, revision, cap), window.missable_groups(), window.named(), window.end(), |span| { let end = cap.map_or(*span.end(), |cap| (*span.end()).min(cap.saturating_sub(1))); if cap == Some(0) || *span.start() > end { return RangeInclusiveSet::new(); } window .missable(*span.start()) .take_while(|run| *run.start() <= end) .map(|run| *run.start()..=(*run.end()).min(end)) .collect() }, taken, drift, &mut given_up, waiter, ); // Only an ask needs the fills. let fills = (!asks.is_empty()).then(|| state.fills.clone()); (asks, fills) }; // Nothing to ask, no fill to hear from, and no gap to time: no answer can arrive. if fills.is_some() || !misses.is_idle() { let asking = asking(&self.subscription); match &fills { Some(fills) => misses.ask(&self.state, fills, &asks, &asking, &mut given_up), // What the reader asks with still reaches the fills it joined. None => misses.update(&asking), } loop { let retry = misses.resolve(&self.state, asking.max_delay, &mut given_up, waiter); if retry.is_empty() { break; } let fills = self.state.read().fills.clone(); misses.ask(&self.state, &fills, &retry, &asking, &mut given_up); } } // Only what a fill still owes can expire. if misses.is_waiting() { misses.expire(&self.state.read(), drift, &mut given_up, waiter); } if self.drops.is_some() { for (run, stale) in given_up { DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(stale)); } } // A refusal taken in may move a time end walking history past it, widening what // is missed: the read goes again to ask for it. if !self.filter.window.is_settled() && self.sync() { waiter.waker().wake_by_ref(); } } /// The next group a fill answered for this cursor in its order, cached and not handed /// out yet. fn next_filled(&mut self) -> Option { let misses = self.misses.as_mut().filter(|misses| misses.has_filled())?; let order = self.subscription.read().order; let mut given_up = Vec::new(); let group = misses.next_filled(&self.state.read(), order, &mut given_up); if self.drops.is_some() { for (run, stale) in given_up { DropCursor::give_up(&mut self.record, run.clone(), Hole::missed(stale)); } } group } /// What a group read consults of the fills: the lowest group still owed, whether a filled /// one waits to be taken, and whether any is owed at all. fn fill_view(&self) -> (Option, bool, bool) { let Some(misses) = &self.misses else { return (None, false, false); }; (misses.lowest_wanted(), misses.has_filled(), misses.is_waiting()) } /// Whether a fill still owes this cursor a group, so the end of the feed is not its end. fn awaits_fill(&self) -> bool { self.misses.as_ref().is_some_and(|misses| misses.is_waiting()) || self.filter.window.lookups().next().is_some() || self.heads.as_ref().is_some_and(|heads| !heads.fetching.is_empty()) } /// Move past the group `peeked` found and hand it out: the exact group, even if its /// producer evicted it since, so eviction cannot advance the reader past a gap. fn take(&mut self, peeked: Peeked) -> group::Consumer { let Peeked { group, arrival } = peeked; let Some(taken) = arrival else { return self.advance_key(group); }; self.advance_arrival(group.start, taken); self.record.skipped.remove(group.start); // Taking is the cache access a peek is not. group.cache_refresh(); self.deliver(group) } fn poll_peek_recv(&mut self, waiter: &kio::Waiter) -> Poll>> { self.sync(); if let Some(err) = &self.filter.refused { return Poll::Ready(Err(err.clone())); } self.admit(); // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.reoffer(drift.budget); self.probe(self.floor(), &drift, waiter); if let Some(misses) = &mut self.misses { misses.pick_up(); } self.poll_recv_with(drift, waiter) } /// [`Self::poll_peek_recv`] once synced, judging against `drift`. fn poll_recv_with(&mut self, mut drift: Drift, waiter: &kio::Waiter) -> Poll>> { self.poll_heads(waiter); self.expire_heads(&drift, waiter); loop { // Re-offer what a widening newly covers ahead of the arrival walk, in the // subscription's order: it is the oldest thing still owed. A group the walk has // yet to reach comes again from it, and the subscriber's delivered set drops the // repeat. let owed = match self.next_owed() { Some(producer) => Some((producer.consume(), Taken::Owed)), // A fill answered what the walk never reaches: backfill is not in it. None => match self.next_filled() { Some(producer) => Some((producer.consume(), Taken::Filled)), None => self.next_head().map(|group| (group, Taken::Head)), }, }; let (consumer, taken) = match owed { Some((consumer, taken)) => (consumer, taken), None => { let floor = self.floor(); let index = Cell::new(self.index); let polled = self.poll(waiter, |state| { state.poll_recv_group(&index, floor, &self.filter.window) }); self.index = index.get(); // A moved window moved the servable edge `drift` was judged against. if polled.is_pending() && self.watch_time() { self.admit(); drift = ready!(self.poll_drift(&self.servable, waiter))?; self.expire_heads(&drift, waiter); continue; } let Some((producer, found_index)) = ready!(polled)? else { // The feed ended, but a fill still owes a group. Its answer wakes us. if self.awaits_fill() { return Poll::Pending; } return Poll::Ready(Ok(None)); }; let consumer = producer.consume(); // The first group to arrive resolves a relative floor, which may land // above it, and a later one may settle where a time end falls. if !self.filter.window.is_settled() { self.sync(); self.admit(); // The snapshot saw the pending window, which serves everything. if !self.filter.window.is_pending() { drift = ready!(self.poll_drift(&self.servable, waiter))?; } if consumer.start < self.floor() { self.index = found_index + 1; continue; } } // Pass a group beyond the cap or outside the ranges, holding nothing: a // widening finds it in the cache again. Keep scanning so an in-range group // that arrived behind it still flows. if !self.admits(consumer.start) { self.index = found_index + 1; continue; } (consumer, Taken::Walk(found_index + 1)) } }; // Drop a group the drift budget has given up on and keep scanning, so one // poll walks a whole backlog off rather than handing it out group by group. if ready!(self.poll_stale(&consumer, &drift, waiter))? { self.advance_arrival(consumer.start, taken); self.write_off_stale(consumer.start, consumer.content(), &drift); continue; } if self.lacks_head(&consumer) { self.advance_arrival(consumer.start, taken); self.ask_head(consumer.start, waiter); continue; } return Poll::Ready(Ok(Some(Peeked { group: consumer, arrival: Some(taken), }))); } } /// Write off group `key`, which the drift budget gave up on, as dropped `Old`: /// skipped for a wider budget to offer again, unless the budget is at its bound. fn write_off_stale(&mut self, key: u64, content: stats::Content, drift: &Drift) { self.stale.add(content); if self.drops.is_some() && let Some(end) = key.checked_add(1) { self.record .pending .insert(key..end, Pending::own(Hole::stale(drift.budget))); } match drift.bounded { // A re-offered group stale at the bound is final. true => self.record.skipped.remove(key), false => self.skip(key, drift.budget), } // Whatever the window is now: narrowed to a bounded one later, it accounts for this // group by it, even once the group left the cache. if let Some(end) = key.checked_add(1) { self.record.written_off.insert(key..end); // Settled by the next retire unless still skipped. self.record.skipped.release(key); } } /// Move the arrival cursor past a group it read: from the walk, to `index`; from what /// a widening owes, out of the owed spans. fn advance_arrival(&mut self, key: u64, taken: Taken) { match taken { Taken::Walk(index) => self.index = index, Taken::Owed => self.filter.owed.remove(key..=key), Taken::Filled => { if let Some(misses) = &mut self.misses { misses.take_filled(key); } } Taken::Head => { if let Some(heads) = &mut self.heads { heads.filled.remove(&key); } } } } /// Whether `group` is one this cursor hands out only from its first frame, which it /// lacks: the window is bounded, so it reads whole groups as a fetch would. fn lacks_head(&self, group: &group::Consumer) -> bool { self.filter.window.end().is_some() && group.first_frame() > 0 } /// Ask a fill for the head of `key`, as a fetch from frame 0 would, taking in an /// answer already at hand. fn ask_head(&mut self, key: u64, waiter: &kio::Waiter) { let asking = self.subscription.read(); let (priority, max_delay) = (asking.priority, asking.delay.max); drop(asking); let request = FetchOptions { options: group::Fetch { priority, max_delay, frame_start: 0, }, end: None, }; // The cursor judges the answer against its window when it hands it out. let mut fetching = Fetching::cached_or_queued(self.state.clone(), key, request, stats::Scope::default()).unjudged(); let heads = self.heads.get_or_insert_default(); match kio::Task::poll(&mut fetching, waiter) { Poll::Pending => { heads.fetching.insert(key, fetching); } Poll::Ready(Ok(group)) => { heads.filled.insert(key, group); } Poll::Ready(Err(_)) => self.give_up_heads(std::iter::once(key..key.saturating_add(1))), } } /// Take in the heads answered, parking `waiter` on those still out, once per read. A /// head the reader's bounds no longer admit is let go: a widening offers its group again. fn poll_heads(&mut self, waiter: &kio::Waiter) { let Some(mut heads) = self.heads.take() else { return }; let floor = self.floor(); let admitted = |key: &u64| *key >= floor && self.admits(*key); heads.fetching.retain(|key, _| admitted(key)); heads.filled.retain(|key, _| admitted(key)); let mut given_up = Vec::new(); heads .fetching .retain(|key, fetching| match kio::Task::poll(fetching, waiter) { Poll::Pending => true, Poll::Ready(Ok(group)) => { heads.filled.insert(*key, group); false } Poll::Ready(Err(_)) => { given_up.push(*key..key.saturating_add(1)); false } }); self.heads = Some(heads); self.give_up_heads(given_up); } /// Write off the heads still asked whose group went stale within this cursor's window, /// so a bounded read never waits on a fill for a group it would not hand out. The cursor /// judges what it hands out, so a head fill does not ([`Fetching::unjudged`]). fn expire_heads(&mut self, drift: &Drift, waiter: &kio::Waiter) { let Some(heads) = &self.heads else { return }; if heads.fetching.is_empty() { return; } let asked: Vec = heads.fetching.keys().copied().collect(); let servable = self.servable.clone(); let Poll::Ready(Ok(stale)) = self.poll(waiter, |state| { let stale: Vec<(u64, stats::Content)> = asked .iter() .filter_map(|&key| { // Parked first, so a group landing above while this judges wakes it. let _ = state.cache.wakes().watch_landing(key, waiter); // Judged whether or not the cache still holds the headless copy: an evicted // one goes stale all the same, and counts as a group of unknown size. (key < state.cache_floor(drift.budget) || state.poll_drifted(key, &servable, drift.budget, None, waiter)) .then(|| { let content = match state.lookup.get(&key) { Some(slot) => slot.group.consume().content(), None => stats::Content { groups: 1, ..Default::default() }, }; (key, content) }) }) .collect(); Poll::Ready(Ok(stale)) }) else { return; }; for (key, content) in stale { if let Some(heads) = &mut self.heads { heads.fetching.remove(&key); } self.write_off_stale(key, content, drift); } } /// Give up the groups whose head no fill can serve, as a refused miss is. fn give_up_heads(&mut self, given_up: impl IntoIterator>) { let mut given_up = given_up.into_iter().peekable(); if given_up.peek().is_none() { return; } let misses = self.misses.get_or_insert_with(Box::default); let mut runs = Vec::new(); for run in given_up { misses.give_up(run, &mut runs); } // A head the cursor found missing belonged to a group that existed. if self.drops.is_some() { for (run, _) in runs { DropCursor::give_up(&mut self.record, run, Hole::drop(StreamError::NotFound)); } } } /// The next head answered and not handed out yet, in the subscription's order. fn next_head(&self) -> Option { let heads = self.heads.as_ref()?; let mut filled = heads.filled.values(); match self.subscription.read().order { Order::Ascending => filled.next(), Order::Descending => filled.next_back(), } .cloned() } /// Pass groups another copy already handed out, without delivering or metering them. fn skip_ordered(&mut self, floor: u64) { if floor <= self.next_key { return; } idle::Idle::disarm(&mut self.idle); self.next_key = floor; // A group delivered on another copy is not a stale skip on this one. if !self.seek_pending.is_empty() { self.seek_pending = self.seek_pending.split_off(&floor); } self.forget_heads_below(floor); } /// Let go of the heads asked below `next`, which an ordered read never goes back for: /// one still out would keep the read waiting on a fill it no longer polls. fn forget_heads_below(&mut self, next: u64) { if let Some(heads) = &mut self.heads { heads.fetching = heads.fetching.split_off(&next); heads.filled = heads.filled.split_off(&next); } } /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. fn poll_peek_next(&mut self, waiter: &kio::Waiter) -> Poll>> { let next = self.next_key; self.sync(); if let Some(err) = &self.filter.refused { return Poll::Ready(Err(err.clone())); } let floor = next.max(self.floor()); // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.probe(floor, &drift, waiter); if let Some(misses) = &mut self.misses { misses.pick_up(); } self.poll_next_with(floor, drift, waiter) } /// [`Self::poll_peek_next`] once synced, from `floor`, judging against `drift`. fn poll_next_with( &mut self, mut floor: u64, mut drift: Drift, waiter: &kio::Waiter, ) -> Poll>> { let end = self.end_key; self.poll_heads(waiter); // Where the gap below the candidate starts: groups stepped over as stale are cached. let mut from = floor; let group = loop { let window = &self.filter.window; let polled = self.poll(waiter, |state| state.poll_next_in_range(floor, end, window)); // A moved window moved the servable edge `drift` was judged against. if polled.is_pending() && self.watch_time() { drift = ready!(self.poll_drift(&self.servable, waiter))?; continue; } let Some(producer) = ready!(polled)? else { // The feed ended, but a fill still owes a group. Its answer wakes us. if self.awaits_fill() { return Poll::Pending; } // Deliberately no flush of `seek_pending` here: only a delivery commit // may count a conviction. This `None` can be an artifact of a floor // that will lower again. A conviction never committed is dropped // uncounted with the cursor. return Poll::Ready(Ok(None)); }; // In order means a group a fill still owes below this one comes first. Its // answer, or giving it up, wakes us. if self .misses .as_ref() .and_then(|misses| misses.lowest_wanted()) .is_some_and(|wanted| wanted < producer.start) { return Poll::Pending; } let group = producer.consume(); // The first group to arrive resolves a relative floor, which may land above it, and // a later one may settle where a time end falls. if !self.filter.window.is_settled() { self.sync(); // The snapshot saw the pending window, which serves everything. if !self.filter.window.is_pending() { drift = ready!(self.poll_drift(&self.servable, waiter))?; } from = from.max(self.floor()); if group.start < self.floor() || !self.filter.window.contains(group.start) { floor = floor.max(self.floor()).max(group.start.saturating_add(1)); continue; } } // Skip a group the budget has given up on and keep scanning, so one poll // walks a whole backlog off rather than handing it out group by group. if ready!(self.poll_stale(&group, &drift, waiter))? { // Not counted yet: a conviction is not permanent (the budget can widen, // the edge can be evicted), so the group may still be delivered. // Re-snapshot on every re-examination so the eventual count reflects the // group's latest observed content. self.seek_pending.insert(group.start, (group.content(), drift.budget)); floor = group.start.saturating_add(1); continue; } // In order means a group the live feed still owes below this one comes first, // until the budget gives up on it. Its landing, its drop, or that verdict wakes us. ready!(self.poll_owed(from..group.start, &drift, waiter))?; // A group missing its head comes from the fill asked for it, in its turn. let group = match self.lacks_head(&group) { false => group, true => { let key = group.start; if self .heads .as_ref() .is_none_or(|heads| !heads.fetching.contains_key(&key) && !heads.filled.contains_key(&key)) && self.misses.as_ref().is_none_or(|misses| misses.gone(key).is_none()) { self.ask_head(key, waiter); } match self.heads.as_ref().and_then(|heads| heads.filled.get(&key)) { Some(head) => head.clone(), // Given up: past it, as past any group no fill can serve. None if self.misses.as_ref().is_some_and(|misses| misses.gone(key).is_some()) => { floor = key.saturating_add(1); continue; } None => return Poll::Pending, } } }; // A conviction the budget walked back: the group is handed over after all, // so it must never reach the stale count. self.seek_pending.remove(&group.start); break group; }; Poll::Ready(Ok(Some(Peeked { group, arrival: None }))) } fn advance_key(&mut self, group: group::Consumer) -> group::Consumer { self.next_key = group.start + 1; // The reader holds the group itself now, as on the arrival path. if let Some(misses) = &mut self.misses { misses.take_filled(group.start); } self.forget_heads_below(self.next_key); // The delivery commits everything the seek stepped over to reach this group. self.commit_seek_stale(self.next_key); group.cache_refresh(); self.deliver(group) } /// Count the key path's convictions below `committed` into [`Self::stale`], /// exactly once each. /// /// A conviction only becomes a real skip once a delivery commits past it. Until then /// the entry waits in [`Self::seek_pending`], where a delivered group removes itself /// (see [`Self::poll_next_with`]). `committed` must be the key watermark (one /// past the last delivery), which only rises. The seek's floor is NOT that: it /// includes the local floor, which can be lowered again, and a conviction it flushed could /// then be delivered after all. fn commit_seek_stale(&mut self, committed: u64) { let keep = self.seek_pending.split_off(&committed); for (key, (content, budget)) in std::mem::replace(&mut self.seek_pending, keep) { self.stale.add(content); // Given up for good now, so it is as reported as a write-off is: a key-order // reader tells its caller the group will not come. if self.drops.is_some() && let Some(end) = key.checked_add(1) { self.record.pending.insert(key..end, Pending::own(Hole::stale(budget))); } } } } /// A cloneable handle to a subscriber's delivery preferences. /// /// This updates the same subscription as the owning [`Subscriber`] without /// borrowing its read cursor, so callers can change delivery priority, the max delay /// budget, or group bounds while another task is waiting for groups. #[derive(Clone)] pub struct Control { subscription: Preferences, /// A plain cursor's servable window, republished when the ranges are replaced, so a /// group it handed out is judged against them before the cursor next reads. `None` /// for a reader that follows a route, which [`Self::mirror`] serves instead. republish: Option, /// A reader that follows a route, whose copies take the new preferences at once. mirror: Option, /// What [`Self::refresh`] asks. target: Target, } /// The track a reader's refresh asks: its own, or for a front's reader, the copy serving it. #[derive(Clone)] enum Target { Plain(kio::ConsumerWeak), Front(kio::ConsumerWeak), } impl Target { fn request(&self) -> Refreshing { match self { Self::Plain(state) => TrackState::request(state), Self::Front(logical) => Refreshing::front(logical), } } } /// What a [`Control`] needs to republish a plain cursor's [`Published`] window. #[derive(Clone)] struct Republish { state: kio::ConsumerWeak, published: kio::ProducerWeak, } impl Republish { /// Resolve the ranges in `preferences` as the cursor will, and publish them under the /// cap the cursor holds then, unless the cursor already published them or newer. fn publish(&self, preferences: &Preferences) { let Some(published) = self.published.produce() else { return; }; let (ranges, min, revision) = { let current = preferences.read(); ( current.ranges().collect::>(), current.delay.min, current.revision, ) }; if published.read().revision >= revision { return; } let state = self.state.read(); // The update was refused before it got here if the track can't resolve it. let Ok(window) = Window::keyed(ranges, min, state.latest_live(), &*state) else { return; }; drop(state); let runs = Servable::of(&window, &Arc::default()); if let Ok(mut current) = published.write() && current.revision < revision { *current = Published::new(runs, revision, current.cap); } } } impl Control { /// This subscriber's current preferences. pub fn subscription(&self) -> Subscription { self.subscription.read().subscription() } /// Replace this subscriber's preferences, updating the producer's aggregate. /// /// Returns [`Error::Closed`] if the track already ended; the update is /// meaningless at that point and can usually be ignored. pub fn update(&self, subscription: Subscription) -> Result<()> { self.subscription.update(subscription)?; if let Some(republish) = &self.republish { republish.publish(&self.subscription); } if let Some(mirror) = &self.mirror { mirror.sync(); } Ok(()) } /// Ask for a group newer than the newest held, as [`Subscriber::refresh`] does. pub fn refresh(&self) -> Refreshing { self.target.request() } } impl Subscriber { /// Count the payload of the groups taken from now on only as the caller delivers it, /// through the returned [`stats::Bill`], rather than as it is read. A reader that drops /// some of what it reads, such as a gateway's jitter buffer, takes it before its first /// group, so what it drops counts nothing. pub fn bill(&mut self) -> stats::Bill { self.stats.bill() } /// The track's [`Info`], resolved when the subscription was established. /// /// Free, unlike [`Consumer::info`]: subscribing already waited for the info /// (SUBSCRIBE_OK on the wire), so a subscriber always has it. pub fn info(&self) -> &Info { &self.info } /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { &self.name } /// The broadcast this track belongs to, as reached through the handle it came from. /// Its [`path`](broadcast::Info::path) is what a catalog's relative `broadcast` /// references resolve against. pub fn broadcast(&self) -> &broadcast::Info { &self.broadcast } /// Attribute the groups the drift budget skipped since the last read. fn count_stale(&mut self, meter: &stats::Meter) { // An untagged subscriber leaves the count where it is. if meter.is_tracked() { meter.stale(self.take_stale()); } } /// The groups the drift budget skipped since the last call; see [`Cursor::take_stale`]. pub(super) fn take_stale(&mut self) -> stats::Content { match &mut self.inner { Inner::Plain(cursor) => cursor.take_stale(), Inner::Resume(resume, _) => resume.take_stale(), } } /// Read this subscription's track cache without sharing its cursor. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn consume(&self) -> Consumer { let state = match &self.inner { Inner::Plain(cursor) => cursor.state.weak(), Inner::Resume(_, logical) => logical.state.weak(), }; Consumer::new(self.name.clone(), state) } /// Create a handle for updating this subscriber's delivery preferences. pub fn control(&self) -> Control { match &self.inner { Inner::Plain(cursor) => Control { subscription: cursor.subscription.clone(), republish: Some(Republish { state: cursor.state.weak(), published: cursor.published.weak(), }), mirror: None, target: self.target(), }, Inner::Resume(resume, _) => Control { subscription: resume.subscription().clone(), republish: None, mirror: Some(resume.mirror()), target: self.target(), }, } } fn target(&self) -> Target { match &self.inner { Inner::Plain(cursor) => Target::Plain(cursor.state.weak()), Inner::Resume(_, logical) => Target::Front(logical.state.weak()), } } /// The next group in arrival order. /// /// Returns each group it delivers exactly once, in the order it landed on the wire, /// which may be out of key due to network reordering or loss. Use /// [`Self::ordered`] if you only want groups whose key is higher than any /// previously returned. /// /// Once means once per key: a group the publisher aborts and re-creates, or that /// a second route delivers too, is not handed out again. Once a group is past the /// budget below, the subscriber stops expecting every group before it, so a gap that /// old is never filled, which holds because a timed track refuses a group whose start /// rewinds. The exception is a cached group it skipped as stale, which a wider budget /// offers again. /// /// Groups are semi-reliable, and the [`Delay::max`] budget is the other /// thing (alongside eviction and a moving start) that decides which of them arrive: /// one that has drifted further behind the live edge than the budget tolerates is /// skipped rather than handed over, so a single read walks off a whole backlog. The /// default is [`Duration::ZERO`], which takes the live /// edge and writes the rest off; raise it to read history. [`Self::with_range`] and /// [`Subscription::ranges`] are filters, not exemptions: backfill needs a budget that /// covers it. [`Consumer::fetch_group`] is the way to ask for one old group outright. /// Across same-instance route copies, the lowest next buffered group is delivered /// first; a lower group that arrives later is still delivered in arrival order. /// The budget remains attached to a returned group: if it stalls while newer data /// advances, its pending frame read ends with [`Error::Old`]. /// /// Honors the group range set by [`Self::with_range`]: a group beyond the cap is /// passed without blocking in-range groups that arrive behind it, and holds nothing /// here. Raising the cap, or widening the ranges, offers the groups it newly covers /// that the cache still holds, newest first unless [`Order::Ascending`]. Widening the /// budget offers the cached groups it skipped as stale and now covers the same way. /// Once the track finishes, a reader with nothing left in range ends. /// /// A reader whose [`Subscription::ranges`] all end ends too, once every group in them /// was handed out, dropped, skipped as stale, or given up as unfillable, rather than /// waiting for an update to widen them: a reader that wants more keeps a range open. A /// [`Self::with_range`] cap never ends a read, since it may rise again, and a group it /// keeps out stays owed. A track an origin serves from whichever of its routes is up ends /// the same way, once no route it left may still deliver a group in range. /// Later reads return `Ok(None)` immediately too. Widening the ranges with /// [`Self::update`] resumes reading the newly covered groups without replaying any. /// /// A group one of [`Subscription::ranges`] starts or ends inside reads only the frames /// they cover there. One missing the frames asked for reads [`Error::Lagged`] rather /// than starting short. /// /// Returns `Ok(Some(group))` when a group is available, `Ok(None)` when the track is /// finished or the ranges are read, or the error when the track has been aborted. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_recv_group(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_group(false, waiter) } /// [`Self::poll_recv_group`], handing out the next group only if its key lies in /// `admit`: one outside stays next, and the read is `Pending` without parking on it. #[cfg(feature = "async")] pub(crate) fn poll_recv_group_in( &mut self, admit: (ops::Bound, ops::Bound), waiter: &kio::Waiter, ) -> Poll>> { self.poll_group_in(false, admit, waiter) } /// The next group in arrival order, or in key order for `ordered`, skipping any /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { self.poll_group_in(ordered, (ops::Bound::Unbounded, ops::Bound::Unbounded), waiter) } /// [`Self::poll_group`], refusing a next group whose key lies outside `admit`. fn poll_group_in( &mut self, ordered: bool, admit: (ops::Bound, ops::Bound), waiter: &kio::Waiter, ) -> Poll>> { let res = self.poll_group_unsettled(ordered, admit, waiter); // What the read settled is billed now, and pays what it can of its readers' debt. The // group it hands out stays above the fold until its reader accounts for it. let handed = match &res { Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; match self.settle_record(handed) { Ok(()) => res, Err(err) => Poll::Ready(Err(err)), } } fn poll_group_unsettled( &mut self, ordered: bool, admit: (ops::Bound, ops::Bound), waiter: &kio::Waiter, ) -> Poll>> { if self.record().failed() { return Poll::Ready(Err(Error::Lagged)); } // Ahead of the meter: a read that parks again counts only what a peek skipped. let walk = idle::Walk::new(ordered); match &self.inner { Inner::Plain(cursor) if cursor.parked(walk, waiter) => { if cursor.stale != stats::Content::default() && self.stats.is_tagged() { self.count_stale(&self.stats.meter()); } return Poll::Pending; } // A resumed read takes its copies' skips only on a full read, which counts them. Inner::Resume(resume, _) if resume.parked(walk, waiter) => return Poll::Pending, _ => {} } match self.poll_peek_group(ordered, waiter) { Poll::Ready(Ok(Some(reserved))) if admit.contains(&reserved.key()) => { Poll::Ready(self.take(reserved).map(Some)) } res => { // What the peek skipped for age counts even when nothing is handed out. self.count_stale(&self.stats.meter()); match res { Poll::Ready(Ok(None)) => Poll::Ready(self.ended()), // Refused: it stays next, so nothing behind it is either. Poll::Ready(Ok(Some(_))) => Poll::Pending, res => res.map(|res| res.map(|_| None)), } } } } /// Reserve the next group not yet handed out, without moving past it: [`Self::take`] /// hands out that exact group, even if its producer evicted it meanwhile. pub(super) fn poll_peek_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { match &mut self.inner { Inner::Plain(cursor) => { let walk = idle::Walk::new(ordered); if cursor.parked(walk, waiter) { return Poll::Pending; } let mut idle = idle::Idle::start(&mut cursor.idle, waiter); let polled = loop { let peeked = match ordered { true => cursor.poll_peek_next(idle.waiter()), false => cursor.poll_peek_recv(idle.waiter()), }; let peeked = match peeked { Poll::Pending => { // A read that only skipped since its last hand-out settles its write-offs too. cursor.retire_written_off(); break match !self.parks && cursor.is_exhausted(&[], 0) { true => Poll::Ready(Ok(None)), false => Poll::Pending, }; } Poll::Ready(Err(err)) => break Poll::Ready(Err(err)), Poll::Ready(Ok(None)) => break Poll::Ready(Ok(None)), Poll::Ready(Ok(Some(peeked))) => peeked, }; if !cursor.record.delivered.contains(peeked.group.start) { break Poll::Ready(Ok(Some(Reserved::Plain(peeked)))); } // A replay advances only this copy's cursor, as an ordinary read would. cursor.take(peeked); }; idle.finish(&mut cursor.idle, walk, polled.is_pending()); polled } Inner::Resume(resume, logical) => resume .poll_peek_group(ordered, self.parks, &logical.record.delivered, waiter) .map(|res| res.map(|candidate| candidate.map(|candidate| Reserved::Resume(Box::new(candidate))))), } } /// Whether this route copy's bounded window has nothing left for the logical readers that /// handed out `logical`: what this copy took itself, wrote off, or gave up counts too. /// It judges from its resume point `from`: the logical reader judges what lies under it. pub(super) fn exhausted_for(&mut self, logical: &[&Delivered], from: Option, waiter: &kio::Waiter) -> bool { match &mut self.inner { Inner::Plain(cursor) => cursor.is_exhausted(logical, from.unwrap_or(0)), // A copy that is another front's logical track judges across its own copies. Inner::Resume(resume, own) => { let mut taken = Vec::with_capacity(logical.len() + 1); taken.push(&own.record.delivered); taken.extend_from_slice(logical); resume.exhausted(&taken, waiter) } } } /// Hand out the group a [`Self::poll_peek_group`] on this subscriber reserved. pub(super) fn take(&mut self, reserved: Reserved) -> Result { let meter = self.stats.meter(); let ordered = reserved.ordered(); let res = match (&mut self.inner, reserved) { (Inner::Plain(cursor), Reserved::Plain(peeked)) => Ok(cursor.take(peeked)), (Inner::Resume(resume, _), Reserved::Resume(candidate)) => resume.take(*candidate), _ => unreachable!("a reservation is taken from the subscriber that made it"), }; if let Ok(group) = &res { self.hand_out(group.start, ordered); } self.count_stale(&meter); res.map(|group| group.with_meter(meter)) } /// The error the track's declared end carries, if it does not end cleanly. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn end_error(&self) -> Option { match &self.inner { Inner::Plain(cursor) => cursor.state.read().outcome.end().and_then(|end| end.error.clone()), Inner::Resume(resume, _) => resume.declared_error(), } } /// Whether this reader ran out at an end declared with an error, rather than an abort. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn ended_with_error(&self) -> bool { self.ended::<()>().is_err() } /// What a reader that ran out of groups gets: the end of stream, or the error the track /// ended with once this reader's range reaches that end. fn ended(&self) -> Result> { let end = match &self.inner { Inner::Plain(cursor) => cursor.state.read().end_error(), Inner::Resume(resume, _) => resume.end_error(), }; match end { Some((fin, err)) if self.reaches(fin) => Err(err), _ => Ok(None), } } /// Whether this reader's range reaches an end at the exclusive final key `fin`: a /// range that stops short of it, past the cap or past every range as resolved, whatever /// its anchors, never reaches the end. pub(crate) fn reaches(&self, fin: u64) -> bool { self.bounds().end.is_none_or(|cap| cap >= fin) && fin.checked_sub(1).is_none_or(|last| self.next_in_range(last).is_some()) } /// Pass groups another route copy handed out, preserving the arrival cursor and bounds. pub(super) fn skip_ordered(&mut self, floor: u64) { match &mut self.inner { Inner::Plain(cursor) => cursor.skip_ordered(floor), Inner::Resume(resume, logical) => { // What it handed out moves, which nothing it parked on notifies. if !logical.record.delivered.covers(0..floor) { resume.disarm(); } // Its retention decisions also need the containing reader's past prefix. // This records what is no longer owed, without handing a group out. logical.record.delivered.forget_below(floor); } } } /// Record `key` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. fn hand_out(&mut self, key: u64, ordered: bool) { match &mut self.inner { Inner::Plain(cursor) => cursor.hand_out(key, ordered), Inner::Resume(resume, logical) => { let delivered = &mut logical.record.delivered; if ordered { delivered.forget_below(key); } delivered.insert(key); resume.forget_past_budget(delivered); } } } /// Forget the gaps in `delivered` below each group past this subscriber's budget now: /// one landing there would be written off. pub(super) fn forget_past_budget(&self, delivered: &mut Delivered) { match &self.inner { Inner::Plain(cursor) => cursor.forget_past_budget(delivered), Inner::Resume(resume, _) => resume.forget_past_budget(delivered), } } /// How many runs of handed-out groups this subscriber remembers. #[cfg(test)] pub(super) fn delivered_runs(&self) -> usize { self.record().delivered.runs() } /// Bill what this reader settled, and pay what it can of the debt its track left its /// readers; [`Error::Lagged`] once that needed a run it was promised. pub(super) fn settle_record(&mut self, handed: Option) -> Result<()> { match &mut self.inner { Inner::Plain(cursor) => { let state = &cursor.state; match cursor.misses.as_deref_mut() { Some(misses) => cursor.record.settle(|| state.read().cache.clone(), misses, handed), None => cursor.record.settle(|| state.read().cache.clone(), &mut (), handed), } } Inner::Resume(_, logical) => { let state = &logical.state; logical.record.settle(|| state.read().cache.clone(), &mut (), handed) } } } /// What this reader settled. fn record(&self) -> &Record { match &self.inner { Inner::Plain(cursor) => &cursor.record, Inner::Resume(_, logical) => &logical.record, } } /// Whether the pool ended this reader, needing a run it was promised. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn lagged(&self) -> bool { self.record().failed() } /// What this reader settled, for its publisher to record what its wire was told. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn record_mut(&mut self) -> &mut Record { match &mut self.inner { Inner::Plain(cursor) => &mut cursor.record, Inner::Resume(_, logical) => &mut logical.record, } } /// How many runs of groups written off as stale this subscriber remembers. #[cfg(test)] fn written_off_runs(&self) -> usize { match &self.inner { Inner::Plain(cursor) => cursor.record.written_off.iter().count(), Inner::Resume(..) => 0, } } /// Whether an unread cached group needs this subscription, without moving its cursor. pub(super) fn has_unread_group(&self, unread: &Unread<'_>) -> bool { let cursor = match &self.inner { Inner::Plain(cursor) => cursor, Inner::Resume(resume, _) => return resume.has_unread_group(unread), }; let budget = cursor.subscription.read().budget(); let state = cursor.state.read(); // A finite budget skips the cache below its floor at once, so it is not unread. let budget = clamp_max_delay(budget, state.max_age_bound()); let floor = cursor.floor().max(state.cache_floor(budget)).max(unread.start); let eligible = |key: u64| { key >= floor && super::subscription::before_end(key, unread.end) && cursor.filter.window.contains(key) && !unread.delivered.contains(key) }; // What a widening owes, the walk has yet to take in, or lifting the local cap would: // a group past it is unread, not gone. let owed = cursor.owed(&Admitted::new(&cursor.filter.window, cursor.floor(), None)); let last = unread.end.map(|end| end.checked_sub(1)); let owed = || { owed.iter().any(|run| { let end = match last { Some(Some(last)) => (*run.end()).min(last), Some(None) => return false, None => *run.end(), }; *run.start() <= end && state .lookup .range(*run.start()..=end) .any(|(key, slot)| eligible(*key) && slot.is_read()) }) }; state.readable() && (state.arrivals_from(cursor.index).any(|entry| { eligible(entry.key) && state .lookup .get(&entry.key) .is_some_and(|slot| slot.stamp == entry.stamp && slot.is_read()) }) || owed()) } /// The keys that landed on this copy since the last call, below `below`. pub(super) fn late(&mut self, below: u64) -> Vec { match &mut self.inner { Inner::Plain(cursor) => cursor.late(below), Inner::Resume(resume, _) => resume.late(below), } } /// Report the groups that landed below where this key-order reader had read, which it /// never hands out, as given up `Old`. A group it handed out or judged stale is already /// settled, so the record blocks it. fn note_late(&mut self) { let mut late = match &mut self.inner { Inner::Plain(cursor) => cursor.late(cursor.next_key), Inner::Resume(resume, logical) => resume.late(logical.record.delivered.floor()), }; // A copy's window can be wider than this reader's ranges and bounds, so a late key they // leave out is no gap of this reader's, as the plain path's window already clips it. if matches!(self.inner, Inner::Resume(..)) { late.retain(|key| !self.runs_in_range(*key, key.saturating_add(1)).is_empty()); } let record = match &mut self.inner { Inner::Plain(cursor) => &mut cursor.record, Inner::Resume(_, logical) => &mut logical.record, }; for key in late { if let Some(end) = key.checked_add(1) { DropCursor::give_up(record, key..end, Hole::drop(StreamError::Old)); } } } /// Record `key` as served, so no cut or drop of it is reported afterwards. fn settle(&mut self, key: u64) { if let Some(end) = key.checked_add(1) { self.settle_range(key..end); } } /// Record `keys` as served or reported, so no cut or drop of them is reported again. fn settle_range(&mut self, keys: ops::Range) { let record = match &mut self.inner { Inner::Plain(cursor) => &mut cursor.record, Inner::Resume(_, logical) => &mut logical.record, }; record.tell(keys, Report::Settled); } /// The key cursor behind [`Ordered`], which owns the only public door to it. pub(super) fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_group(true, waiter) } /// Read this track's groups in key order instead of arrival order. /// /// Consumes the subscriber, so one handle carries exactly one cursor: an /// arrival-order [`Subscriber`] or a key-order [`Ordered`], never both at once. /// The two advance independently, and interleaving them produces a group stream that /// is neither, which is why the choice is a handle rather than a method. pub fn ordered(self) -> Ordered { Ordered { inner: self, held: None, gaps: BTreeMap::new(), at: 0, started: false, idle: None, } } /// Whether `other` was cloned from this subscriber (shares the same underlying state). pub fn is_clone(&self, other: &Self) -> bool { match (&self.inner, &other.inner) { (Inner::Plain(a), Inner::Plain(b)) => a.state.same_channel(&b.state), (Inner::Resume(a, _), Inner::Resume(b, _)) => a.subscription().same_channel(b.subscription()), _ => false, } } /// Poll for where the source's feed starts, raised to this cursor's floor, once /// resolved: the first group a publisher tells its peer it serves. A feed starting /// below the floor serves the floor's group too, and a fill this cursor waits on starts it /// lower still. `None` when the source declares none and no fill is out. /// /// Where a reader that waits on each group in key starts when [`Self::floor`] names /// none: groups arrive newest first, so the first to arrive is not the oldest served. /// /// Pending too while a time start waits on the groups held to place it, since where its /// range starts is unknown until then, unless the floor sits at or past that time, as a /// pre-lite-07 wire's start does: the group containing it is out of range then. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_start(&mut self, waiter: &kio::Waiter) -> Poll> { let cursor = match &mut self.inner { Inner::Plain(cursor) => cursor, Inner::Resume(resume, _) => return resume.poll_start(waiter), }; let start = ready!(Consumer::poll_state_start(&cursor.state, waiter)); cursor.sync(); if cursor.filter.window.is_waiting_above(cursor.min_key) { // Registered before syncing again, so a change placing it after this wakes it. A // closed track places it at once, since nothing more can arrive. let closed = cursor.state.poll(waiter, |_| Poll::<()>::Pending).is_ready(); cursor.sync(); if !closed && cursor.filter.window.is_waiting_above(cursor.min_key) { return Poll::Pending; } } let start = start.map(|start| start.max(cursor.floor())); let filling = cursor.misses.as_ref().and_then(|misses| misses.lowest()); Poll::Ready(match (start, filling) { (Some(start), Some(filling)) => Some(start.min(filling)), (start, filling) => start.or(filling), }) } /// The lowest group this subscriber's ranges name, as resolved here when it last read, /// raised to the start of [`Self::with_range`]: `None` while one reaches back to the /// oldest group still around, or a relative start still waits on the first group from /// the live feed. /// /// Where a reader that waits on each group in key starts, rather than at whatever /// arrives first, which newest-first delivery makes the latest. pub fn floor(&self) -> Option { let start = match &self.inner { Inner::Plain(cursor) => cursor.filter.window.start(), Inner::Resume(resume, _) => resume.floor(), }; Some(start?.max(self.bounds().start)) } /// Each of this subscriber's ranges keyed in `keys` as resolved here, in that order: its /// first group and exclusive end, `None` while open, or `None` for one whose start still /// waits on the live feed or a lookup, or that isn't held. One empty as given, or that only /// resolves empty, ends at or below its start. /// /// A publisher tells its peer where a relative or time start landed, which the peer /// can't work out from the groups it receives. Settles the ranges once for the batch, so /// asking for k of them costs O(k log n) past that. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn resolved_keys(&mut self, keys: impl IntoIterator) -> Vec> { match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); keys.into_iter() .map(|key| cursor.filter.window.resolved_key(key)) .collect() } Inner::Resume(resume, _) => resume.resolved_keys(keys), } } /// This subscriber's ranges keyed in `keys` while their time starts wait on the groups /// held to place them, or on its route copies for a reader following them: each one's /// time, and the exclusive end it resolved to. `None` for one that resolved. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn waiting_keys(&mut self, keys: impl IntoIterator) -> Vec)>> { match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); keys.into_iter() .map(|key| cursor.filter.window.waiting_key(key)) .collect() } Inner::Resume(resume, _) => resume.waiting_keys(keys), } } /// This subscriber's range keyed `key` as resolved here; see [`Self::resolved_keys`]. #[cfg(test)] #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn resolved_range(&mut self, key: u64) -> Option { self.resolved_keys([key]).pop().flatten() } /// The lowest group at or above `from` this subscriber's ranges and /// [`Self::with_range`] floor cover, past the starts the publisher declared skipped /// (see [`Producer::skip_groups`]) and those given up for good, `None` when none does: /// the groups between are never read, so a reader waiting on each group in key need not /// wait for them. A wire before lite-07 forwards a skip as such a drop. `Some` from there /// while a relative anchor waits on the first group, which can't say yet. pub fn next_in_range(&self, from: u64) -> Option { let from = from.max(self.bounds().start); match &self.inner { Inner::Plain(cursor) => { let state = cursor.state.read(); first_open(cursor.filter.window.runs(from), |key| state.hole_end(key, true)) } Inner::Resume(resume, _) => resume.next_in_range(from), } } /// Whether the publisher declared every start in `groups` skipped, never to be produced /// (see [`Producer::skip_groups`], which appending does between groups), so no group can /// arrive there: true for an empty range. Unlike [`Self::next_in_range`], this ignores the /// subscriber's ranges. A copy knows only the skips its wire forwarded. /// /// What tells a reader that nothing lies between two groups, or before a track's first. /// It knows only the skips recorded so far, so `false` means "not known skipped": a skip /// still in flight reads `false`, as does one a local producer's cache expired below its /// edge. A reader that asks once treats such a group as one with something before it. pub fn is_skipped(&self, groups: ops::Range) -> bool { let Some(last) = groups.end.checked_sub(1).filter(|last| *last >= groups.start) else { return true; }; let run = std::iter::once(groups.start..=last); match &self.inner { Inner::Plain(cursor) => { let state = cursor.state.read(); first_open(run, |key| state.hole_end(key, false)).is_none() } Inner::Resume(resume, _) => resume.is_skipped(run), } } /// The start of the group before `start` on the track, as a lite-07 GROUP's Previous Start /// names it: just below the run of starts the publisher declared skipped directly under /// `start`, else the group just below it, else `None`. No group starts between the two. /// A copy knows only the skips its wire forwarded. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn previous_start(&self, start: u64) -> Option { match &self.inner { Inner::Plain(cursor) => cursor.state.read().previous_start(start), Inner::Resume(resume, _) => resume.previous_start(start), } } /// Whether the track's cache holds the live feed's group at `key`, still serving it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn holds(&self, key: u64) -> bool { match &self.inner { Inner::Plain(cursor) => cursor.state.read().holds_group(key), Inner::Resume(resume, _) => resume.holds(key), } } /// The groups from `from` up to the exclusive `end` that this subscriber's ranges and /// [`Self::with_range`] bounds name, lowest first: the groups it is owed, without the /// ones between disjoint ranges. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn runs_in_range(&mut self, from: u64, end: u64) -> Vec> { let bounds = self.bounds(); let from = from.max(bounds.start); let end = bounds.end.map_or(end, |cap| cap.min(end)); let runs = match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); cursor.filter.window.runs(from).collect::>() } Inner::Resume(resume, _) => resume.runs(from), }; runs.into_iter() .take_while(|run| *run.start() < end) .map(|run| *run.start()..run.end().saturating_add(1).min(end)) .collect() } /// Whether `accounted` holds every group from `from` on that this subscriber's ranges /// and [`Self::with_range`] bounds name, once every range has a resolved end: `false` /// while one stays open, counts back from a group not seen yet, or ends at a time not /// settled yet. /// /// A publisher that accounts for each group it serves or drops learns from it that a /// bounded subscription has nothing more to deliver. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn covered(&mut self, from: u64, accounted: &impl Accounted) -> bool { let bounds = self.bounds(); let from = from.max(bounds.start); match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); cursor.filter.window.covered_by(from, bounds.end, accounted) } Inner::Resume(resume, _) => resume.covered(from, bounds.end, accounted), } } /// This subscriber's own ranges keyed in `keys`, in that order, as resolved here: `None` /// for one empty, still waiting, or not held, and for every range of a front's reader, /// whose copies resolve them. pub(super) fn resolved_ranges(&mut self, keys: impl IntoIterator) -> Vec> { match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); keys.into_iter() .map(|key| cursor.filter.window.resolved_key(key)) .collect() } Inner::Resume(..) => keys.into_iter().map(|_| None).collect(), } } /// Why this subscriber's ranges can't be served from its track, the error its reads /// fail with; `None` for a front's reader, whose copies answer for it. pub(super) fn refused(&mut self) -> Option { match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); cursor.filter.refused.clone() } Inner::Resume(..) => None, } } /// The ranges of `subscription` resolved against this track's cache now, by index. pub(super) fn resolve_ranges(&self, subscription: &Subscription) -> Vec> { match &self.inner { Inner::Plain(cursor) => { let state = cursor.state.read(); Window::new( &subscription.ranges, subscription.delay.min, state.latest_live(), &*state, ) .map(|window| window.resolved()) .unwrap_or_default() } Inner::Resume(..) => Vec::new(), } } /// Poll for a run of keys this subscription gave up without serving, with the /// error it was given up with, so a publisher can tell its peer not to wait for them. /// /// Stale skips are recorded only once this has been polled, so a reader that never /// forwards drops keeps none. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.poll_drop_probing(true, waiter) } /// [`Self::poll_drop`] without judging the live edge again: only the runs a read since /// the last judgement gave up, such as a receive that probed after a drop poll found /// nothing. That judgement parked `waiter` on everything that could change it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_queued_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { self.poll_drop_probing(false, waiter) } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, Hole)> { // What a publisher told its wire since is billed now. A reader the pool ended reports // nothing more: its reads fail with the reason. if self.settle_record(None).is_err() { return Poll::Pending; } match &mut self.inner { Inner::Plain(cursor) => cursor.poll_drop(probe, waiter), // Each copy reports to its own route; the logical record keeps one route's run from // going out over what another's already settled. Inner::Resume(resume, logical) => loop { if let Some(next) = logical.record.next_pending() { return Poll::Ready(next); } let (range, hole) = std::task::ready!(resume.poll_drop(&logical.record.delivered, probe, waiter)); logical.record.pending.insert(range, Pending::own(hole)); }, } } /// Poll for the track's cache reflecting its live feed (or the track ending), with the /// largest position it holds; `None` for a track with nothing yet. A front's logical /// track answers for the route serving it. See [`Producer::set_idle`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_live(&mut self, waiter: &kio::Waiter) -> Poll> { let cursor = match &mut self.inner { Inner::Plain(cursor) => cursor, Inner::Resume(resume, _) => return resume.poll_live(waiter), }; let res = cursor.state.poll(waiter, |state| match state.readable() { true => Poll::Ready(state.largest()), false => Poll::Pending, }); match res { Poll::Ready(Ok(largest)) => Poll::Ready(largest), Poll::Ready(Err(state)) => Poll::Ready(state.largest()), Poll::Pending => Poll::Pending, } } /// The track's declared exclusive end, as [`Self::poll_finished`] reports it, without /// registering a wake. A track that never declared one answers without its lock. pub(super) fn declared_end(&mut self) -> Option { match &mut self.inner { Inner::Plain(cursor) => match cursor.cache.end_declared() { true => cursor.state.read().declared(), false => None, }, Inner::Resume(..) => match self.poll_finished(&kio::Waiter::noop()) { Poll::Ready(Ok(end)) => Some(end), _ => None, }, } } /// The track's declared exclusive end position, or the cause on an abort. See /// [`Producer::finish_at`] for what the position means. /// /// Resolves as soon as the end is known, which may be ahead of the live edge /// when the producer finished via [`Producer::finish_at`]. This reports the declared /// end, not that every group has arrived. A local session close without a declared /// end returns [`Error::Closed`]. Read [`Self::poll_recv_group`] (or /// [`Ordered::poll_next_group`]) until it yields `None` to observe the track fully drained. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { match &mut self.inner { Inner::Plain(cursor) => cursor.poll_finished(waiter), Inner::Resume(resume, _) => resume.poll_finished(waiter), } } /// Limit subsequent reads to these group keys, returning `self` for chaining. /// /// `2..=5` includes groups 2 through 5; `2..5` excludes group 5, and `..0` is the /// empty range. Both bounds are assigned, in either direction: an omitted start is /// group 0 and an omitted end removes the cap. Lowering the floor or raising the cap /// offers the groups it newly covers that the cache still holds and this reader never /// handed out, in its [`Order`]; a group is never handed out twice. Groups past the cap /// stay cached, and lowering the cap below the cursor waits until it rises again or /// the track finishes. [`Self::poll_largest`] reports within these bounds. /// /// A local filter, not a request: it tells the publisher nothing, so groups outside it /// are still delivered and simply not returned. To change what is requested, set /// [`Subscription::ranges`] via [`Self::update`]. See [Local cursor vs wire /// preference](Self#local-cursor-vs-wire-preference). One exception: on a track served /// across route changes, lowering the floor asks the serving route again for the groups /// it newly covers within those ranges, since the route that delivered them may be gone. pub fn with_range(mut self, groups: impl RangeBounds) -> Self { self.set_range(groups); self } /// [`Self::with_range`] on a reader held in place. /// Park past the end of a bounded window rather than end the read there, as a wire /// publisher does: its peer reads the end of a subscription's stream as the track's, /// unless the wire ends a covered subscription itself. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn parked(mut self) -> Self { self.parks = true; self } pub(crate) fn set_range(&mut self, groups: impl RangeBounds) { let (start, end) = super::subscription::key_bounds(groups); match &mut self.inner { Inner::Plain(cursor) => { cursor.min_key = start; cursor.end_key = end; cursor.update_servable(); } Inner::Resume(resume, logical) => resume.set_range(start, end, &logical.record.delivered), } self.largest.bounds_moved(); } /// The local bounds `largest` reports within. fn bounds(&self) -> Bounds { match &self.inner { Inner::Plain(cursor) => Bounds { start: cursor.min_key, end: cursor.end_key, }, Inner::Resume(resume, _) => resume.bounds(), } } /// This subscriber's current preferences. pub fn subscription(&self) -> Subscription { self.control().subscription() } /// This subscriber's preferences, read by key without materializing them. pub(super) fn preferences(&self) -> &Preferences { match &self.inner { Inner::Plain(cursor) => &cursor.subscription, Inner::Resume(resume, _) => resume.subscription(), } } /// Replace this subscriber's delivery preferences. /// /// Stored verbatim; the publisher's max age window is applied to the aggregate, not /// here (see [`Producer::subscription`]). Replaced ranges resolve again here, a /// relative anchor counting back from the latest group now, and widening them offers /// the cached groups they newly cover, even ones that arrived before the groups /// already read. Returns [`Error::Closed`] /// if the track already ended; the update is meaningless at that point and can usually /// be ignored. pub fn update(&mut self, subscription: Subscription) -> Result<()> { let channel = match &self.inner { Inner::Plain(cursor) => &cursor.subscription, Inner::Resume(resume, _) => resume.subscription(), }; channel.update(subscription)?; match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); } // The copies take the change now, not on the next read, so it reaches their // sources even while nobody reads. Inner::Resume(resume, _) => resume.mirror().sync(), } // A reader the pool ended reports it on its next read. let _ = self.settle_record(None); Ok(()) } /// Apply `edit` to this subscriber's preferences, costing the ranges it names: only the /// ranges it adds resolve, and every other keeps where it resolved; see [`Edit`]. /// /// The keys are the caller's. The ranges the subscription started with, or that /// [`Self::update`] last gave, are keyed by position. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn edit(&mut self, edit: Edit) -> Result<()> { let channel = match &self.inner { Inner::Plain(cursor) => &cursor.subscription, Inner::Resume(resume, _) => resume.subscription(), }; channel.edit(edit)?; match &mut self.inner { Inner::Plain(cursor) => { cursor.sync(); } Inner::Resume(resume, _) => resume.mirror().sync(), } let _ = self.settle_record(None); Ok(()) } /// Poll for the track's largest location within [`Self::with_range`]: the current one, then each newer one, and `None` once the track ends. /// /// The largest is the newest group at the newest frame (0 while empty), as /// the source reported it and its live feed delivered since, so nothing reports until /// the track's feed is live. A largest past the cap or below the floor reports nothing: /// the cache does not stand in for it, so a cap moved below the largest stays pending. /// Moving the bounds wakes a parked poll, which reports the largest once it is within /// them and newer than the last report. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_largest(&mut self, waiter: &kio::Waiter) -> Poll> { if self.largest.ended { return Poll::Ready(None); } let bounds = self.bounds(); let poll = self.poll_largest_within(bounds, waiter); // Only a parked poll waits on the bounds; a finished one is not woken by them. match poll { Poll::Pending => waiter.register(&mut self.largest.parked), Poll::Ready(next) => { self.largest.parked = kio::WaiterList::new(); self.largest.ended = next.is_none(); } } poll } fn poll_largest_within(&mut self, bounds: Bounds, waiter: &kio::Waiter) -> Poll> { let seen = &mut self.largest.position; match &mut self.inner { Inner::Plain(cursor) => { let state = &cursor.state; let cache = state.read().cache.clone(); poll_largest( &cache, seen, bounds, waiter, |waiter| state.poll(waiter, |state| largest_live(state, waiter)).map(|_| ()), |waiter| state.poll(waiter, |state| largest_ended(state, waiter)).map(|_| ()), ) } Inner::Resume(resume, _) => resume.poll_largest(seen, bounds, waiter), } } /// Whether `position`, the largest, ended there: its group finished or closed, or its /// source said so. An aborted group stays open, since it can be recreated or fetched. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn largest_finished(&self, position: Position) -> bool { match &self.inner { Inner::Plain(cursor) => cursor.state.read().cache.largest_finished(position), Inner::Resume(resume, _) => resume.largest_finished(position), } } /// The track's largest location known now, ignoring [`Self::with_range`], without waiting /// or moving [`Self::poll_largest`] on: past what the cache holds once the source /// reported a newer one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn peek_largest(&mut self) -> Option { let mut seen = None; match &mut self.inner { Inner::Plain(cursor) => cursor.state.read().cache.take_largest(&mut seen, Bounds::default()), Inner::Resume(resume, _) => match resume.poll_largest(&mut seen, Bounds::default(), &kio::Waiter::noop()) { Poll::Ready(largest) => largest, Poll::Pending => None, }, } } /// The parts of `run` this subscriber's track still holds as a received stale run: one /// a new upstream subscription superseded is gone (see [`Producer::supersede_stale`]). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn stale_held(&self, run: ops::Range) -> RangeSet { match &self.inner { Inner::Plain(cursor) => cursor .state .read() .drops .overlapping(&run) .filter(|(_, hole)| hole.stale_budget().is_some()) .map(|(held, _)| held.start.max(run.start)..held.end.min(run.end)) .collect(), // A front's reader checks its own routes' copies. Inner::Resume(..) => RangeSet::new(), } } /// Bumped whenever a run [`Self::stale_held`] reported may no longer be held, so a caller /// rechecks the runs it holds only after a change. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn stale_revision(&self) -> u64 { match &self.inner { Inner::Plain(cursor) => cursor.state.read().stale_revision, Inner::Resume(..) => 0, } } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn latest_key(&self) -> Option { match &self.inner { Inner::Plain(cursor) => cursor.state.read().max_key, Inner::Resume(resume, _) => resume.latest(), } } /// How many keys just below `key` the publisher declared it never produced; see /// [`Producer::skip_groups`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn skipped_below(&self, key: u64) -> u64 { match &self.inner { Inner::Plain(cursor) => cursor.state.read().skipped_below(key), Inner::Resume(resume, _) => resume.skipped_below(key), } } /// Ask the publisher for a group starting after the newest one this node holds, judged /// by the track's [`Refresh`], resolving once the request is sent. /// /// The request is made now. It is sent once the newest group is old enough, when /// whatever serves this node's copy reads it: the session forwarding it upstream, or a /// local publisher waiting in [`Demand::poll_refresh`]. Concurrent requests coalesce into one, /// and all of them resolve when it is sent. Dropping the future before then withdraws /// this caller's request; one already sent stays sent. It resolves at once when there is /// nothing to send: a track that does not refresh, a ratio the newest group is under, a /// track with no group yet (whose first is already asked for), or a request already /// sent. It also resolves when a newer group lands first, which answers it. It errors /// if the track closes before then. A front's reader asks the copy serving it, and asks /// the next one if that copy closes first. /// /// The answer arrives like any other group: the first group this subscriber receives /// that starts after the newest held when it was called. A group already on its way when /// the request reached the publisher answers it, so across hops "made after this call" /// is best effort; a reader that needs that puts its own marker in the frame. pub fn refresh(&self) -> Refreshing { self.target().request() } /// Ask for a new group as `ask`, a request nobody withdraws; a front's reader asks the /// copy serving it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn ask(&self, ask: Ask) { match &self.inner { Inner::Plain(cursor) => TrackState::ask(cursor.state.read(), ask), Inner::Resume(_, logical) => { let routes = logical.state.read().routes.clone(); if let Some(copy) = routes.and_then(|routes| routes.serving()) { copy.ask(ask); } } } } } /// A [`Subscriber::refresh`] request, resolving once it is sent. Dropping it before then /// withdraws the request. /// /// Owns its handles, so it borrows no subscriber: a reader keeps reading groups while it waits. #[must_use = "dropping it withdraws the request"] pub struct Refreshing { ticket: Option, /// A front's reader follows its logical track's route instead; see [`Failover`]. failover: Option>, park: kio::Park, } /// A front reader's request outlives the copy it asked: once that copy closes while the /// logical track is open, it asks the copy serving next, so only the logical track closing /// rejects it. struct Failover { logical: kio::ConsumerWeak, routes: super::resume::Consumer, /// The route generation of the copy last asked. asked: u64, /// The request to that copy, itself a front's when fronts are chained. copy: Option, /// How that copy's request ended: the track's end once no route serves another. closed: Error, } impl Failover { fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { loop { if self.logical.poll_closed(waiter).is_ready() { self.copy = None; let err = self.logical.read().outcome.aborted().cloned(); return Poll::Ready(Err(err.unwrap_or(Error::Dropped))); } if let Some(copy) = &mut self.copy { match copy.poll(waiter) { Poll::Pending => return Poll::Pending, Poll::Ready(Ok(())) => return Poll::Ready(Ok(())), Poll::Ready(Err(err)) => { self.copy = None; self.closed = err; continue; } } } // The copy asked closed, or none served: ask the next one the route serves. let (generation, copy, terminal) = self.routes.answering(); if generation != self.asked { self.asked = generation; if let Some(copy) = copy { let request = copy.request(); // The new copy has nothing to send, such as no group yet: its join asks. if request.is_done() { return Poll::Ready(Ok(())); } self.copy = Some(request); } continue; } if terminal { return Poll::Ready(Err(std::mem::replace(&mut self.closed, Error::Dropped))); } ready!(self.routes.poll_changed(generation, waiter)); } } } /// A caller's claim on a pending request: [`Refreshing`] without the parked waiter. struct Ticket { state: kio::ConsumerWeak, subscriptions: kio::Lock, epoch: u64, } impl Refreshing { fn done() -> Self { Self { ticket: None, failover: None, park: kio::Park::default(), } } fn waiting(ticket: Ticket) -> Self { Self { ticket: Some(ticket), failover: None, park: kio::Park::default(), } } /// A request on the logical track `logical`, asking the copy its routes serve. fn front(logical: &kio::ConsumerWeak) -> Self { let Some(routes) = logical.read().routes.clone() else { return Self::done(); }; let (asked, copy, _) = routes.answering(); let copy = copy.map(|copy| copy.request()).filter(|copy| !copy.is_done()); // A logical track already closed still rejects, as a plain track does. if copy.is_none() && !logical.is_closed() { return Self::done(); } Self { ticket: None, failover: Some(Box::new(Failover { logical: logical.clone(), routes, asked, copy, closed: Error::Dropped, })), park: kio::Park::default(), } } fn is_done(&self) -> bool { self.ticket.is_none() && self.failover.is_none() } /// Ready once the request is sent, or ended without needing to be: answered by a newer /// group. Errors if the track closes first. pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { Self::poll_request(&mut self.ticket, &mut self.failover, waiter) } fn poll_request( slot: &mut Option, failover: &mut Option>, waiter: &kio::Waiter, ) -> Poll> { let Some(front) = failover else { return Self::poll_ticket(slot, waiter); }; let result = front.poll(waiter); if result.is_ready() { *failover = None; } result } fn poll_ticket(slot: &mut Option, waiter: &kio::Waiter) -> Poll> { let Some(ticket) = slot else { return Poll::Ready(Ok(())); }; if ticket.subscriptions.lock().requested(ticket.epoch, waiter) { *slot = None; return Poll::Ready(Ok(())); } if ticket.state.poll_closed(waiter).is_ready() { let err = ticket.state.read().outcome.aborted().cloned().unwrap_or(Error::Dropped); // The request can no longer be sent, so dropping the ticket withdraws it. *slot = None; return Poll::Ready(Err(err)); } Poll::Pending } } impl Future for Refreshing { type Output = Result<()>; fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { let this = self.get_mut(); let waiter = this.park.hold(cx); Refreshing::poll_request(&mut this.ticket, &mut this.failover, waiter) } } impl Drop for Ticket { fn drop(&mut self) { let wakes = self.subscriptions.lock().withdraw(self.epoch); for mut wakes in wakes { wakes.wake(); } } } /// A [`Subscriber`] that reads groups in key order. /// /// Created by [`Subscriber::ordered`], which consumes the subscriber, so a track is read /// one way or the other and never both. Every group it returns has a higher key /// than the last, so a late arrival (network reordering, or a gap the cache filled after /// the fact) is reported as a [gap](Item::Gap) rather than delivered out of turn. /// /// # Age and skipping /// /// [`Delay::max`] applies as this cursor reads, exactly as it does on the /// arrival cursor: a group is skipped once its *reach*, where its immediate successor /// begins, is that far behind the newest frame on the track. Nothing weaker convicts it, /// because the reach is the only proof that *every* frame it could still hold is past the /// budget: a group's own timestamps say where it starts presenting, not where it stops, /// and an unstamped successor leaves it unbounded and therefore kept. A backlog inside /// the budget is still delivered whole, as a burst in order, which is what a decoder /// reading a gap-free key needs. /// /// The same budget follows a group already handed out: once newer content pulls that far /// ahead, a read still waiting on it ends with [`Error::Old`] and the cursor moves on. /// /// # Gaps /// /// A missing group is waited for only while the track still owes it: on a copy a /// subscription feeds, a group that may still land, reordered behind its successor, is /// read first, until the same budget gives up on it. Without a budget (`None`, which a /// history or VOD read uses) it waits until the hole resolves. A group the track knows will never land costs nothing: /// one the publisher dropped, one that arrived and was since evicted or aborted, or one /// below where the feed starts. A locally produced track owes /// nothing, so there the cursor seeks to the lowest cached group in range. /// /// An unlimited or long budget only avoids blocking where the feed signals its gaps: /// moq-lite, whose publisher sends `SUBSCRIBE_DROP` for every group it won't deliver, or an /// ordered moq-transport FETCH. A moq-transport SUBSCRIBE signals no gaps, so there a group /// lost in transit can hold the ordered reader at its hole until the track ends. /// /// A group this reader gives up on is not lost silently: it comes back as an /// [`Item::Gap`], so a reader that paces on what was sent counts the loss instead of /// waiting for a group that will never come. That is every group the publisher or a relay /// dropped, the transport evicted, the budget aged out, a fetch could not find, or that /// landed after the read passed its key (reported as `Old`). A gap carries the keys it /// spans and is returned once, after any group the read still owes below it and before /// the next group above it, or when the track ends. Keys the publisher declared skipped /// (see [`Producer::skip_groups`]) were never groups, so they are no gap; /// [`Self::is_skipped`] answers for them. /// /// The budget is updatable mid-stream through [`Self::control`]. [`Duration::ZERO`] (the /// default) keeps only what nothing newer has superseded, so a consumer that stalls and /// resumes rejoins the live edge instead of replaying at 1x. pub struct Ordered { inner: Subscriber, /// A group read and not yet returned, because a gap below it goes first. held: Option>, /// The gaps found and not yet returned, as `start => (end, error)`. They never overlap, and /// a drop found while others wait goes in order among them. gaps: BTreeMap, /// One past the highest key returned, as a group or the end of a gap. at: u64, /// Whether the drop record exists. It holds only what happened after it began, so it /// starts before the first read. started: bool, /// The read that last parked, answered again until something it parked on fires. idle: Option>, } /// What an [`Ordered`] read returns next. pub enum Item { /// A group, with a higher key than any group returned before. Group(group::Consumer), /// Groups this reader gave up on, which it will not return. Gap { /// The keys given up, as group starts. range: ops::Range, /// Why: what a read of those groups would have ended with. error: Error, }, } impl Item { /// The group, or `None` for a gap. pub fn group(self) -> Option { match self { Self::Group(group) => Some(group), Self::Gap { .. } => None, } } } impl Ordered { /// The track's [`Info`], resolved when the subscription was established. pub fn info(&self) -> &Info { self.inner.info() } /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { self.inner.name() } /// The broadcast this track belongs to, as reached through the handle it came from. pub fn broadcast(&self) -> &broadcast::Info { self.inner.broadcast() } /// The next group with a higher key than any previously returned, or a gap this reader /// gave up on (see [Gaps](Self#gaps)). /// /// Honors the group range set by [`Subscriber::with_range`] before [`Subscriber::ordered`]: /// a group past the cap stays in the producer's cache and becomes eligible again if /// the cap rises or is removed. Ranges that all end end the read once they are /// accounted for, as for [`Subscriber::poll_recv_group`]. /// Later reads return `Ok(None)` immediately too; widening the ranges with /// [`Self::update`] resumes reading above the last key returned. /// /// Returns `Ok(Some(item))` when a group or gap is available, `Ok(None)` when the track is /// finished (including an abort after its declared end settled, see [`Producer::abort`]) /// or the ranges are read, or the error when the track has been aborted short of its end. /// Every gap is returned before the end, even a clean one. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { if idle::Idle::parked(&self.idle, (), waiter) { return Poll::Pending; } let mut idle = idle::Idle::start(&mut self.idle, waiter); let polled = self.poll_item(idle.waiter()); idle.finish(&mut self.idle, (), polled.is_pending()); polled } fn poll_item(&mut self, waiter: &kio::Waiter) -> Poll>> { // Gaps are recorded only once polled for, so before the first read. if !self.started { self.started = true; self.find_gap(true, waiter); } let read = self.read(waiter); // After the read, which judged what it walked past: a group it handed out leaves only // what that queued, and anything else judges the live edge for the gaps it parks on. self.find_gap(!matches!(read, Poll::Ready(Ok(Some(_)))), waiter); match read { Poll::Ready(Ok(Some(group))) => { let key = group.start; match self.lowest_gap() { // The gap lies wholly below it, or around it: the part below goes first. Some((at, span)) if span.start < key => { // What lies above the group, or above the run when it ends before it, waits. let resume = if span.end <= key { span.end } else { key.saturating_add(1) }; let below = span.start..span.end.min(key); let item = self.take_gap(at, below, resume); self.held = Some(Box::new(group)); Poll::Ready(Ok(Some(item))) } // A group handed out is no gap, even one a drop also named, as a cut group is. lowest => { if let Some((at, span)) = lowest && span.start == key { self.cut_gap(at, span.start, key.saturating_add(1)); } self.at = self.at.max(key.saturating_add(1)); Poll::Ready(Ok(Some(Item::Group(group)))) } } } // What the read still owes lies above the gap, so the gap goes first. Poll::Pending => match self.next_gap() { Some(item) => Poll::Ready(Ok(Some(item))), None => Poll::Pending, }, // Every gap is returned before the end. Poll::Ready(end) => match self.lowest_gap() { Some((at, span)) => { let resume = span.end; Poll::Ready(Ok(Some(self.take_gap(at, span, resume)))) } None => Poll::Ready(end.map(|_| None)), }, } } /// The next group in key order: the one held behind a gap, if the ranges still cover it. fn read(&mut self, waiter: &kio::Waiter) -> Poll>> { if let Some(group) = self.held.take() { let key = group.start; if !self.inner.runs_in_range(key, key.saturating_add(1)).is_empty() { return Poll::Ready(Ok(Some(*group))); } // A range moved off it. With nothing in range above, it waits for the range to widen // again, and the read waits or ends as it does for capped ranges; otherwise the read // passes it. if self.inner.runs_in_range(key, u64::MAX).is_empty() { self.held = Some(group); } } let read = self.inner.poll_next_group(waiter); if let Poll::Ready(Ok(Some(group))) = &read { // Handed out, so no later cut or drop of it is a gap. self.inner.settle(group.start); } read } /// The lowest run of a gap the ranges cover now, as the gap's key in `gaps` and the keys of /// it that run covers, since the ranges may have moved since the gap was found. A gap they /// exclude waits, in case they widen again. fn lowest_gap(&mut self) -> Option<(u64, ops::Range)> { // The fields borrow apart, so the map is walked in place and the walk stops at the // first gap the ranges cover. let Self { inner, gaps, .. } = self; gaps.iter().find_map(|(start, (end, _))| { let runs = inner.runs_in_range(*start, *end); Some((*start, runs.into_iter().next()?)) }) } /// Take `given` from the gap `lowest_gap` found, as an item. What lies from `resume` on stays /// a gap, to be clipped and returned later, as does what lies below `given`. fn take_gap(&mut self, at: u64, given: ops::Range, resume: u64) -> Item { let error = self.cut_gap(at, given.start, resume); self.give(given, error) } /// Remove `from..resume` from the gap held at `at`, returning its error. What lies below /// `from`, which the ranges leave out, and from `resume` on stay gaps, to be clipped and /// returned later. fn cut_gap(&mut self, at: u64, from: u64, resume: u64) -> Error { let (end, error) = self.gaps.remove(&at).expect("the lowest gap is held"); if at < from { self.gaps.insert(at, (from, error.clone())); } if end > resume { self.gaps.insert(resume, (end, error.clone())); } error } /// Look for every drop recorded, in whatever order they come. fn find_gap(&mut self, judge: bool, waiter: &kio::Waiter) { self.inner.note_late(); loop { let polled = match judge { true => self.inner.poll_drop(waiter), false => self.inner.poll_queued_drop(waiter), }; let Poll::Ready((range, hole)) = polled else { return; }; match hole { Hole::Drop { error, .. } => { // A final drop replaces a stale one still waiting, rather than overlap it. self.forget_gaps(range.clone()); self.gaps.insert(range.start, (range.end, error.into())); } // A skip was never a group: nothing was given up, and it replaces a drop // still waiting. Hole::Skip => self.forget_gaps(range), } } } /// Remove `skipped` from the gaps waiting, splitting those it only partly covers. fn forget_gaps(&mut self, skipped: ops::Range) { // The gaps never overlap, so their ends rise with their starts: walk down from the // highest one that starts below the skip's end, and stop at the first that ends before it. let hit: Vec = self .gaps .range(..skipped.end) .rev() .take_while(|(_, (end, _))| *end > skipped.start) .map(|(start, _)| *start) .collect(); for start in hit { let (end, error) = self.gaps.remove(&start).expect("the gap was just found"); if start < skipped.start { self.gaps.insert(start, (skipped.start, error.clone())); } if end > skipped.end { self.gaps.insert(skipped.end, (end, error)); } } } /// The lowest gap, once nothing the read still owes lies below it. fn next_gap(&mut self) -> Option { let (at, span) = self.lowest_gap()?; let ready = span.start <= self.at || self.inner.next_in_range(self.at).is_none_or(|owed| owed >= span.start); let resume = span.end; ready.then(|| self.take_gap(at, span, resume)) } /// A gap as an item, which the read has now moved past. fn give(&mut self, range: ops::Range, error: Error) -> Item { self.at = self.at.max(range.end); // Told once: no later drop of these keys is a gap again. self.inner.settle_range(range.clone()); Item::Gap { range, error } } /// Narrow subsequent reads to these group keys, as [`Subscriber::with_range`] does: a /// read in key order never goes back, so lowering the floor below a group it already /// returned offers nothing. #[cfg(test)] #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn with_range(mut self, groups: impl RangeBounds) -> Self { self.set_range(groups); self } /// [`Self::with_range`] on a reader held in place. #[cfg(test)] fn set_range(&mut self, groups: impl RangeBounds) { // A moved range notifies nothing a parked read waits on. idle::Idle::disarm(&mut self.idle); self.inner.set_range(groups); } /// Create a handle for updating this subscriber's delivery preferences without /// borrowing the read cursor. pub fn control(&self) -> Control { self.inner.control() } /// Ask for a group newer than the newest held; see [`Subscriber::refresh`]. pub fn refresh(&self) -> Refreshing { self.inner.refresh() } /// This subscriber's current preferences. pub fn subscription(&self) -> Subscription { self.inner.subscription() } /// Replace this subscriber's delivery preferences. /// /// Returns [`Error::Closed`] if the track already ended. pub fn update(&mut self, subscription: Subscription) -> Result<()> { idle::Idle::disarm(&mut self.idle); self.inner.update(subscription) } /// The exclusive end position, once the track declares its end. /// /// See [`Subscriber::poll_finished`]: this reports the declared end, not that every group /// has arrived. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { self.inner.poll_finished(waiter) } /// The largest location within [`Subscriber::with_range`]; see [`Subscriber::poll_largest`]. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_largest(&mut self, waiter: &kio::Waiter) -> Poll> { self.inner.poll_largest(waiter) } /// Whether the publisher declared every start in `groups` skipped; see /// [`Subscriber::is_skipped`]. pub fn is_skipped(&self, groups: ops::Range) -> bool { self.inner.is_skipped(groups) } /// Whether `other` reads the same underlying track state. pub fn is_clone(&self, other: &Self) -> bool { self.inner.is_clone(&other.inner) } } /// A requested track being written before any subscriber sees it; see [`Request::stage`]. /// /// Write through it as through the [`Producer`] it derefs to. Subscribers waiting on the /// request resolve only at [`Self::commit`], with everything written by then. Dropping it /// uncommitted rejects the request with [`Error::Dropped`]. #[must_use = "a staged track is rejected unless committed"] pub(crate) struct Staged { // Taken only by `commit`. producer: Option, // The request's fetch claim, held until the track is published like an unanswered // request's: a cache miss meanwhile waits rather than racing its own handler. _dynamic: Requests, } impl Staged { /// Publish the track: every waiting subscriber resolves now, with what was written. pub(crate) fn commit(mut self) -> Producer { let producer = self.producer.take().expect("taken only by commit"); if let Ok(mut state) = producer.state.write() { state.commit(); } producer } } impl ops::Deref for Staged { type Target = Producer; fn deref(&self) -> &Producer { self.producer.as_ref().expect("taken only by commit") } } impl ops::DerefMut for Staged { fn deref_mut(&mut self) -> &mut Producer { self.producer.as_mut().expect("taken only by commit") } } impl Drop for Staged { fn drop(&mut self) { let Some(producer) = self.producer.take() else { return; }; // Aborted like the producer it derefs to, so a group written through it ends too. let _ = producer.abort(Error::Dropped); } } /// A subscriber asked for a track this broadcast doesn't have yet. /// /// Yielded by [`poll_requested_track`](crate::broadcast::Requests::poll_requested_track), /// or created up front with [`broadcast::Producer::reserve_track`](crate::broadcast::Producer::reserve_track). /// Subscribers block until the request is /// resolved: call [`accept`](Self::accept) to serve it with a [`Producer`], or /// [`reject`](Self::reject) to fail them. Dropping it without either rejects with /// [`Error::Dropped`]. /// /// Concurrent requests for one name are coalesced, so exactly one of these exists per /// name at a time. pub struct Request { name: Arc, // The parent broadcast's info, threaded into the [`Producer`] on accept. broadcast: Arc, state: kio::Producer, // The previous subscription that was combined, used to detect changes. prev_subscription: Option, // Shared with the accepted [`Producer`] and every [`Requests`]: its `Drop` is the // teardown, and it stays inert until a producer is minted. alive: Arc, // A requested track is served on demand, so it counts as fetch-capable from // birth: a consumer's cache-miss `fetch_group` waits to be served instead of // racing the producer (e.g. a relay) into creating its own handler. Released // when the request is accepted or dropped; by then the relay holds its own. _dynamic: Requests, // Ingress stats scope, threaded into the accepted [`Producer`]. Empty (no-op) // unless this request was reserved on a tagged broadcast. stats: stats::Scope, // Where the accepted track's start is pending; see [`Self::request_start`]. requested_start: RequestedStart, // Whether the accepted track starts idle; see [`Self::idle`]. idle: bool, // A track the producer reserved, listed by its broadcast once accepted. list: Option, } /// Whether a [`Request`] asked for its accepted track's start to be pending. enum RequestedStart { /// The track starts as [`Request::accept`] leaves it. Unset, /// Pending at this floor, `None` for the live edge, until the source resolves it. Pending(Option), } impl Request { pub(super) fn new(broadcast: Arc, name: impl Into>) -> Self { let name = name.into(); let state = TrackState::spawn(broadcast.clone()); let alive = Alive::new(name.clone(), state.clone()); let dynamic = Requests::new(name.clone(), state.clone(), alive.clone()); Self { name, broadcast, state, prev_subscription: None, alive, _dynamic: dynamic, stats: stats::Scope::default(), requested_start: RequestedStart::Unset, idle: false, list: None, } } /// List the track in its broadcast once accepted: the producer reserved it. pub(super) fn listed(mut self, list: broadcast::ListOnAccept) -> Self { self.list = Some(list); self } /// Serve the track from a front's routes, read straight from their copies; see /// [`super::resume`]. Applied at once, so a fetch made before [`Self::accept`] already /// reaches the copy whose info is still in flight. pub(super) fn routes(self, routes: super::resume::Consumer) -> Self { if let Ok(mut state) = self.state.write() { state.routes = Some(routes); } self } /// Accept the track with its start pending at `floor`, the first group its source was /// asked for (`None` for the live edge), until the source says where its feed starts /// with [`Producer::start_at`]. /// /// Nothing below `floor` arrives, as with [`Producer::start_at`], and groups are read /// as they arrive, but a relay serving the track waits for the resolved start before /// declaring its own. Applied atomically with [`Self::accept`], before any reader can /// see the track. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn request_start(mut self, floor: impl Into>) -> Self { self.requested_start = RequestedStart::Pending(floor.into()); self } /// Accept the track idle, as a copy that no live feed fills yet: readers get nothing /// from it until the feed goes live. Applied atomically with [`Self::accept`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn idle(mut self) -> Self { self.idle = true; self } /// Attach an ingress stats scope, which meters every group the track takes, including /// fills answered before accept. Set by a tagged [`broadcast::Producer::reserve_track`]. pub(super) fn with_stats(mut self, scope: stats::Scope) -> Self { if let Ok(mut state) = self.state.write() { state.ingress = scope.clone(); } self.stats = scope; self } /// The requested track name. pub fn name(&self) -> &str { &self.name } /// A [`Consumer`] for the eventual track, usable before the request is accepted. pub fn consume(&self) -> Consumer { Consumer::new(self.name.clone(), self.state.weak().consume_weak()) } /// Create a [`Requests`] handle that serves on-demand fetches of uncached /// groups, before [`Self::accept`] is even called. A relay creates one to fetch /// past groups from upstream while (or instead of) serving a live subscription. pub fn requests(&self) -> Requests { Requests::new(self.name.clone(), self.state.clone(), self.alive.clone()) } /// Watch subscriber demand without keeping the track alive. pub fn demand(&self) -> Demand { Demand::new(self.name.clone(), self.state.weak()) } /// Join the eventual track's demand, for a request made through its broadcast. pub(super) fn join(&self, subscription: Option<&Subscription>) -> (kio::Consumer, Option) { let preferences = subscription.map(|value| Preferences::register(self.state.read(), value.clone())); (self.state.consume(), preferences) } /// Read the eventual track without joining its demand. pub(super) fn peek(&self) -> kio::ConsumerWeak { self.state.weak().consume_weak() } /// Mark this request as taken by a dynamic handler, which alone decides its answer. pub(super) fn claim(self) -> Self { if let Ok(mut state) = self.state.write() { state.claim(); } self } /// Reject only while no consumer needs this pending track. Demand and the check /// share one lock, so demand returning after `Demand::wanted` went to `None` wins the race, and the /// close under that lock stops a later consumer attaching to a dead request. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn reject_unwanted(&self, err: Error) -> bool { match self.state.write_unwanted() { kio::Unwanted::Idle(guard) => { commit_abort(guard, err, None).abort(); true } kio::Unwanted::Closed => true, kio::Unwanted::Wanted => false, } } /// Serve the request with the given track, resolving every waiting subscriber. /// /// The name is taken from [`Self::name`]; `info` supplies the remaining knobs /// (`None` for the defaults). If the track was already aborted, the returned /// [`Producer`] is inert: writes fail with the abort error, as if it had been /// aborted immediately after accepting. pub fn accept(self, info: impl Into>) -> Producer { self.stage(info).commit() } /// Mint the track's [`Producer`] without resolving any subscriber yet, so content /// written through the returned [`Staged`] is there when the first reader looks. /// /// For a relay that already holds groups of the track: write them, then /// [`Staged::commit`]. Dropping the [`Staged`] uncommitted rejects the request with /// [`Error::Dropped`]. pub(crate) fn stage(self, info: impl Into>) -> Staged { let info = info.into().unwrap_or_default(); // A closed state means the track was aborted under us. Mirror `reject` and // tolerate it: the Producer we hand back simply can't write. let open = match self.state.write() { Ok(mut state) => { state.stage(info.clone()); if let RequestedStart::Pending(floor) = self.requested_start { state.set_start(Start::Pending(floor)); } if self.idle { state.set_idle(); } true } Err(_) => false, }; // Accepting the request creates the track producer: count it as one ingress // subscription (closed when the last handle drops). No-op when untagged. self.alive.publish(Some(&self.stats)); // An aborted track already left the table, so it is never listed. if let Some(list) = self.list.filter(|_| open) { self.alive.list(list.list(&info)); } Staged { producer: Some(Producer { name: self.name, info, broadcast: self.broadcast, state: self.state, prev_subscription: None, alive: self.alive, }), _dynamic: self._dynamic, } } /// Reject the request, waking all waiting subscribers with `err`. pub fn reject(self, err: Error) { if let Ok(mut state) = self.state.write() { state.reject(err, None); } } /// Reject the request as an upstream refusal that asked to be tried again after `retry`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn refuse(self, err: Error, retry: Duration) { if let Ok(mut state) = self.state.write() { state.reject(err, Some(retry)); } } /// The delivery preferences aggregated across everyone waiting on this request, /// or `None` if nobody is waiting. Useful for sizing the track before accepting. pub fn subscription(&self) -> Option { let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); snapshot_subscription(&subs, bound) } /// Arm `waiter` for the next change to the demand of everyone waiting, without reading /// it; see [`Producer::watch_subscription`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn watch_subscription(&self, waiter: &kio::Waiter) { watch_combined(&self.state, waiter); } /// The aggregate [`subscription`](Self::subscription) once it changes, or `None` once /// nobody is waiting. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_subscription_changed(&mut self, waiter: &kio::Waiter) -> Poll> { let state = self.state.read(); let (subs, bound) = (state.subscriptions.clone(), state.demand_bound()); drop(state); poll_combined_changed(&subs, bound, &mut self.prev_subscription, waiter) } pub(super) fn weak(&self) -> TrackWeak { TrackWeak { name: self.name.clone(), state: self.state.weak(), } } } #[cfg(test)] use futures::FutureExt; #[cfg(test)] #[expect(missing_docs, reason = "test-only assertion helpers")] impl Subscriber { pub fn assert_group(&mut self) -> group::Consumer { kio::wait(|waiter| self.poll_recv_group(waiter)) .now_or_never() .expect("group would have blocked") .expect("would have errored") .expect("track was closed") } pub fn assert_no_group(&mut self) { assert!( kio::wait(|waiter| self.poll_recv_group(waiter)) .now_or_never() .is_none(), "recv_group would not have blocked" ); } pub fn assert_not_closed(&mut self) { assert!( kio::wait(|waiter| self.poll_finished(waiter)).now_or_never().is_none(), "should not be closed" ); } pub fn assert_closed(&mut self) { assert!( kio::wait(|waiter| self.poll_finished(waiter)).now_or_never().is_some(), "should be closed" ); } pub fn assert_is_clone(&self, other: &Self) { assert!(self.is_clone(other), "should be clone"); } pub fn assert_not_clone(&self, other: &Self) { assert!(!self.is_clone(other), "should not be clone"); } } #[cfg(test)] mod test { use super::*; use crate::frame; use crate::test_tracing::count_drop_warnings; use std::time::Duration; /// Mint a track for tests with a default parent broadcast, since tracks are /// normally born from a [`broadcast::Producer`]. `None` is an untimed track with no /// publisher max age. fn track_producer(name: impl Into>, info: impl Into>) -> Producer { let info = info.into().unwrap_or_else(|| Info::default().with_max_age(None)); Producer::new(Arc::new(broadcast::Info::default()), name, info) } #[test] fn drop_changes_follow_split_coalesced_and_recreated_runs() { let producer = track_producer("test", Info::default()); let indexed = || { let state = producer.state.read(); let mut changed: Vec<_> = state.drops_changes.since(0).collect(); changed.sort_by_key(|run| run.start); let current: Vec<_> = state.drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(changed, current, "one change entry per current drop run"); }; producer.drop_groups(10..20, StreamError::Old); producer.drop_groups(20..30, StreamError::Old); indexed(); producer.drop_groups(15..25, StreamError::Cancel); indexed(); // Still arriving, so the next drop ends it and covers it again. let _recreated = producer.create_group(group::Info { start: 17 }).unwrap(); indexed(); producer.drop_groups(14..26, StreamError::Old); indexed(); assert_eq!( producer.state.read().drops_changes.since(0).collect::>(), vec![10..30] ); } proptest::proptest! { /// The times a track reports unplaced are only those its `containing` can't place, but /// for the ones a copy's reported time starts place, after each step: through groups /// arriving and leaving, and every way keys are given up. A time it leaves out lies below /// the newest start reported past an earlier range's time, which may place it once no held /// group precedes it. #[test] fn unplaced_holds_only_what_containing_cant_place( ops in proptest::collection::vec((0u8..6, 0u64..40, 1u64..8), 0..24), wire: bool, starts in proptest::collection::vec((0u64..48, 0u64..48), 0..3), ) { let producer = track_producer("test", Info::default()); if wire { producer.anchored(); producer.time_starts(starts.clone()); } let reported: BTreeSet = starts.iter().map(|(time, _)| *time).collect(); let past = starts.iter().filter(|(asked, start)| asked < start).map(|(_, start)| *start).max(); let check = |producer: &Producer| -> std::result::Result<(), proptest::test_runner::TestCaseError> { let state = producer.state.read(); let unplaced = state.unplaced().expect("a track always tells"); for time in (0..64).chain([u64::MAX - 1, u64::MAX]) { let inside = unplaced.runs.iter().any(|run| run.contains(&time)); let unknown = state.containing(time) == Containing::Unknown; if wire && reported.contains(&time) { proptest::prop_assert!(!unknown, "a reported start places {}", time); } else if inside { proptest::prop_assert!(unknown, "{} is unplaced but placed", time); } else if unknown { proptest::prop_assert!(wire && past.is_some_and(|past| time < past), "{} is left out", time); } } Ok(()) }; check(&producer)?; for (op, start, len) in ops { let groups = start..start + len; match op { 0 => producer.drop_groups(groups, StreamError::Cancel), 1 => producer.skip_groups(groups), 2 => drop(producer.create_group(group::Info { start })), 3 => producer.modify().unwrap().forget_drops(groups), // Evict the first group held from `start`, as expiry does. 4 => { let held = producer.state.read().lookup.range(start..).next().map(|(&key, _)| key); if let Some(key) = held { producer.modify().unwrap().evict_group(key, Error::Evicted); } } // A group at the last key that can start one, past every time but the last. _ => drop(producer.create_group(group::Info { start: u64::MAX - 1 })), } check(&producer)?; } } /// The coalesced drop set answers coverage as a walk of the drop runs would, through /// every way a run is given up, split, replaced, or forgotten. #[test] fn dropped_cover_matches_a_walk_of_the_drops( ops in proptest::collection::vec((0u8..4, 0u64..40, 1u64..8), 1..40), probes in proptest::collection::vec((0u64..48, 0u64..48), 16), ) { let producer = track_producer("test", Info::default()); for (op, start, len) in ops { let groups = start..start + len; match op { 0 => producer.drop_groups(groups, StreamError::Cancel), 1 => producer.drop_groups(groups, StreamError::Old), 2 => producer.skip_groups(groups), _ => producer.modify().unwrap().forget_drops(groups), } let state = producer.state.read(); let mut union = RangeSet::new(); for (run, _) in state.drops.iter() { union.insert(run.clone()); } proptest::prop_assert_eq!(&union, &state.dropped); for &(start, len) in &probes { let keys = start..start + len; let walk = |kind: &dyn Fn(&Hole) -> bool| { if keys.is_empty() { return true; } let mut at = keys.start; for (run, hole) in state.drops.overlapping(&keys) { if !kind(hole) || run.start > at { return false; } at = run.end; } at >= keys.end }; proptest::prop_assert_eq!(state.dropped_cover(keys.clone()), walk(&|_| true)); proptest::prop_assert_eq!(state.never_produced(keys.clone()), walk(&|hole| *hole == Hole::Skip)); } } } } /// A final drop ends a group still arriving, but leaves one that finished here to every /// reader, as a group arriving after its drop replaces it. #[test] fn a_final_drop_leaves_a_finished_group() { let producer = track_producer("test", Info::default()); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); let open = producer.create_group(group::Info { start: 1 }).unwrap(); producer.drop_groups(0..2, StreamError::Cancel); let drops: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(drops, vec![1..2]); assert!(open.is_aborted(), "the group still arriving ends"); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().start, 0); } /// A drop leaves a group a fetched head serves whole, whatever its live copy is doing: a /// final drop still ends a live copy still arriving, and the head serves in its place. #[kio::test] #[cfg(feature = "async")] async fn a_drop_leaves_a_group_a_fetched_head_serves() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut lives = Vec::new(); for key in 0..2 { let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let fetch = consumer.fetch_group(key, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } head.finish().unwrap(); fetch.await.unwrap(); lives.push(live); } lives.pop().unwrap().abort(Error::Cancel).unwrap(); producer.drop_groups(0..1, StreamError::App(5)); producer.give_up(1..2, Hole::stale(Duration::from_millis(500))); assert!(producer.state.read().drops.iter().next().is_none()); assert!(lives[0].is_aborted(), "the live copy still arriving ends"); let mut sub = producer.subscribe(None); assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); for key in 0..2 { assert_eq!(sub.assert_group().start, key); } } /// A live group 0 fed from frame 1, plus a fetched head still arriving that serves it /// whole. #[cfg(feature = "async")] async fn arriving_head(producer: &Producer) -> (group::Producer, group::Producer) { let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); (live, head) } /// A final drop of a group a fetched head is still filling waits on the head: once the /// head dies unfinished, the drop is why the group is gone, so a reader starting there /// sees it rather than stall, even for a drop that abandons the live copy without a cut. #[kio::test] #[cfg(feature = "async")] async fn a_final_drop_outlives_a_head_that_dies() { let producer = track_producer("test", Info::timed()); let (live, head) = arriving_head(&producer).await; producer.drop_groups(0..1, StreamError::Old); assert!(live.is_aborted(), "the live copy still arriving ends"); { let state = producer.state.read(); assert!(state.drops.iter().next().is_none(), "the head still serves the group"); assert!(state.given_up(0).is_none()); } head.abort(Error::Cancel).unwrap(); let state = producer.state.read(); assert!(state.lookup[&0].is_read(), "a reader is handed the drop"); assert!(matches!(state.given_up(0), Some(Error::Stream(StreamError::Old)))); } /// A drop cursor that scanned while the head filled hears the drop once the head dies: /// the head's abort is announced as the cut it leaves. #[kio::test] #[cfg(feature = "async")] async fn a_drop_cursor_hears_a_final_drop_a_dying_head_leaves() { let producer = track_producer("test", Info::timed()); let (_live, head) = arriving_head(&producer).await; let mut sub = producer.subscribe(None); producer.drop_groups(0..1, StreamError::Old); assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); head.abort(Error::Cancel).unwrap(); match sub.poll_drop(&kio::Waiter::noop()) { Poll::Ready((run, hole)) => { assert_eq!(run, 0..1); assert!(matches!(hole.error(), StreamError::Old)); } Poll::Pending => panic!("the drop the head left is reported"), } } /// A live copy that finished wins over a final drop for good, even while a fetched head /// still fills, and even once both copies are gone. #[kio::test] #[cfg(feature = "async")] async fn a_finished_live_copy_wins_over_a_final_drop_a_head_waits_on() { let producer = track_producer("test", Info::timed()); let (live, head) = arriving_head(&producer).await; let live = live.finish().unwrap(); producer.drop_groups(0..1, StreamError::Old); head.abort(Error::Cancel).unwrap(); live.abort(Error::Evicted).unwrap(); let state = producer.state.read(); assert!(state.drops.iter().next().is_none()); assert!(state.given_up(0).is_none()); } /// A fetched head that finishes after a final drop of its group wins over the drop, even /// once it is later freed: no drop is ever recorded beside it. #[kio::test] #[cfg(feature = "async")] async fn a_head_that_finishes_wins_over_a_final_drop() { let producer = track_producer("test", Info::timed()); let (_live, mut head) = arriving_head(&producer).await; producer.drop_groups(0..1, StreamError::Old); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let head = head.finish().unwrap(); { let state = producer.state.read(); assert!(state.drops.iter().next().is_none()); assert!(state.given_up(0).is_none()); } let mut sub = producer.subscribe(None); assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); assert_eq!(sub.assert_group().start, 0); head.abort(Error::Evicted).unwrap(); assert!( producer.state.read().given_up(0).is_none(), "a freed head was still whole" ); } /// Let `duration` pass on the cache pool `producer`'s groups charge into. fn elapse(producer: &Producer, duration: Duration) { producer.broadcast.pool.step(duration); } #[test] fn a_missing_group_uses_the_subscription_grace() { let producer = track_producer("test", None); let budget = Duration::from_millis(50); let _sub = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let feed = producer.consume(); let waiter = kio::Waiter::noop(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert!(feed.poll_group(0, &waiter).is_pending()); assert!(feed.poll_group(u64::MAX, &waiter).is_pending()); elapse(&producer, budget / 2); assert!(feed.poll_group(0, &waiter).is_pending()); elapse(&producer, budget / 2); assert!(matches!(feed.poll_group(0, &waiter), Poll::Ready(Ok(None)))); } #[test] fn an_explicit_drop_does_not_wait_for_an_arrival_gap() { let producer = track_producer("test", Info::default()); let feed = producer.consume(); let waiter = kio::Waiter::noop(); assert!(feed.poll_group(1, &waiter).is_pending()); producer.drop_groups(1..2, StreamError::Old); // The wait ends with the drop's own error. assert!(matches!( feed.poll_group(1, &waiter), Poll::Ready(Err(Error::Stream(StreamError::Old))) )); } /// A bounded replay window for tests whose subject requires every buffered group. fn replay() -> Subscription { Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(30))) } /// Helper: count live cached groups in state. fn live_groups(state: &TrackState) -> usize { state.lookup.len() } /// Helper: the position of the first arrival entry still held, or the next one's. fn first_arrival(state: &TrackState) -> usize { state.arrival.front().map_or(state.next_arrival, |entry| entry.at) } /// Helper: get the key of the first live group in arrival order. fn first_live_key(state: &TrackState) -> u64 { state .arrival .iter() .find(|entry| state.is_current(entry.key, entry.stamp)) .map(|entry| entry.key) .unwrap() } /// An empty group raises the largest at its head, so its first frame names nothing newer. #[cfg(feature = "async")] #[kio::test] async fn largest_streams_groups_and_frames_and_coalesces() { let producer = track_producer("test", Info::default()); let mut subscriber = producer.subscribe(replay()); assert_eq!(producer.largest(), None); let mut group = producer.append_next().unwrap(); assert_eq!( subscriber.largest().await, Some(Position::group(0)), "an empty group is the largest at its head" ); group .write_frame(crate::Timestamp::unscaled(group.start), &b"a"[..]) .unwrap(); assert_eq!(producer.largest(), Some(Position { group: 0, frame: 0 })); assert!( subscriber.largest().now_or_never().is_none(), "the first frame is the head already named" ); group .write_frame(crate::Timestamp::unscaled(group.start), &b"b"[..]) .unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"c"[..]) .unwrap(); assert_eq!(subscriber.largest().await, Some(Position { group: 0, frame: 2 })); let group = group.finish().unwrap(); let producer = producer.finish().unwrap(); drop(group); drop(producer); assert_eq!(subscriber.largest().await, None); } /// Every key below a reached end already raised the largest, so a late group below /// it changes nothing, and the ended stream stays ended. #[test] fn a_late_group_below_the_largest_changes_nothing() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let mut framed = producer.create_group(group::Info { start: 2 }).unwrap(); framed .write_frame(crate::Timestamp::unscaled(framed.start), &b"a"[..]) .unwrap(); framed.finish().unwrap(); producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); producer.finish_at(Position::group(6)).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); // Late content below the end is still admitted; it just isn't the largest. let mut late = producer.create_group(group::Info { start: 4 }).unwrap(); late.write_frame(crate::Timestamp::unscaled(late.start), &b"b"[..]) .unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); let mut fresh = producer.consume(); assert_eq!(fresh.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); } /// An aborted largest group can still be recreated, so the stream stays open past the /// end until a copy of it finishes, and the recreated frames are reported before it ends. #[test] fn a_recreated_largest_group_is_reported_before_the_end() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); first .write_frame(crate::Timestamp::unscaled(first.start), &b"a"[..]) .unwrap(); first.abort(Error::Cancel).unwrap(); producer.finish_at(Position::group(1)).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(0)))); assert!( reader.poll_largest(&noop).is_pending(), "the aborted group can be recreated" ); let mut again = producer.create_group(group::Info { start: 0 }).unwrap(); for payload in [&b"a"[..], b"b"] { again .write_frame(crate::Timestamp::unscaled(again.start), payload) .unwrap(); } assert_eq!( reader.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 1 })) ); again.finish().unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); } /// A copy a transport bound cut short finishes only its own range, so the largest group /// may still grow and the stream stays open past the end. #[kio::test] #[cfg(feature = "async")] async fn a_finished_bounded_copy_keeps_the_largest_open() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let noop = kio::Waiter::noop(); let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); first.write_frame(first.key_time(), &b"a"[..]).unwrap(); first.abort(Error::Cancel).unwrap(); producer.finish_at(Position::group(1)).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(0)))); let fetch = reader.fetch_range( 0, FetchOptions { options: group::Fetch::default(), end: Some(1), }, ); let mut bounded = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); bounded.write_frame(bounded.key_time(), &b"a"[..]).unwrap(); bounded.finish().unwrap(); fetch.await.unwrap(); assert!( reader.poll_largest(&noop).is_pending(), "the bounded copy does not hold the whole group" ); } /// A reader that starts at the largest group while it is still empty is at the live edge: /// it gets that group, which ends with nothing, then the next one. #[test] fn a_live_edge_reader_starts_on_an_empty_largest_group() { let producer = track_producer("test", None); let noop = kio::Waiter::noop(); let mut first = producer.append_next().unwrap(); first .write_frame(crate::Timestamp::unscaled(first.start), &b"a"[..]) .unwrap(); first.finish().unwrap(); let empty = producer.append_next().unwrap(); let mut reader = producer.subscribe(replay()); let Poll::Ready(Some(edge)) = reader.poll_largest(&noop) else { panic!("the empty group is the largest"); }; assert_eq!(edge, Position::group(1)); reader.set_range(edge.group..); empty.finish().unwrap(); let mut next = producer.append_next().unwrap(); next.write_frame(crate::Timestamp::unscaled(next.start), &b"b"[..]) .unwrap(); let mut group = reader.assert_group(); assert_eq!(group.start, 1); assert!( matches!(group.poll_read_frame(&noop), Poll::Ready(Ok(None))), "it ends with nothing" ); let mut group = reader.assert_group(); assert_eq!(group.start, 2); assert!(matches!(group.poll_read_frame(&noop), Poll::Ready(Ok(Some(_))))); } /// A finished track ends the stream while its producer is still held, once the last /// group finishes, and finishing it wakes a parked reader. #[test] fn largest_ends_at_the_declared_end() { use std::sync::atomic::{AtomicBool, Ordering}; struct Flag(AtomicBool); impl futures::task::ArcWake for Flag { fn wake_by_ref(flag: &Arc) { flag.0.store(true, Ordering::SeqCst); } } let producer = track_producer("test", Info::default()); let mut subscriber = producer.subscribe(replay()); let mut group = producer.append_next().unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"a"[..]) .unwrap(); let noop = kio::Waiter::noop(); assert_eq!( subscriber.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 0 })) ); let producer = producer.finish().unwrap(); assert!( subscriber.poll_largest(&noop).is_pending(), "the open group can still grow" ); group .write_frame(crate::Timestamp::unscaled(group.start), &b"b"[..]) .unwrap(); assert_eq!( subscriber.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 1 })) ); let flag = Arc::new(Flag(AtomicBool::new(false))); let waiter = kio::Waiter::new(futures::task::waker(flag.clone())); assert!(subscriber.poll_largest(&waiter).is_pending()); group.finish().unwrap(); assert!( flag.0.load(Ordering::SeqCst), "finishing the last group woke the reader" ); assert_eq!(subscriber.poll_largest(&noop), Poll::Ready(None)); drop(producer); } #[cfg(feature = "async")] #[kio::test] async fn largest_starts_with_the_sources_report() { let producer = track_producer("test", Info::default()); let mut subscriber = producer.subscribe(replay()); producer.report_largest(Position { group: 7, frame: 3 }, false); assert_eq!(subscriber.largest().await, Some(Position { group: 7, frame: 3 })); // A report is a hint about the edge, not a cached group. assert!(subscriber.recv_group().now_or_never().is_none()); // An older report or frame does not move it back. producer.report_largest(Position { group: 7, frame: 1 }, false); let mut _open = producer.create_group(group::Info { start: 2 }).unwrap(); _open .write_frame(crate::Timestamp::unscaled(_open.start), &b"a"[..]) .unwrap(); assert!(subscriber.largest().now_or_never().is_none()); producer.report_largest(Position { group: 8, frame: 0 }, false); assert_eq!(subscriber.largest().await, Some(Position { group: 8, frame: 0 })); } /// Wakes record that a parked poll was woken. struct Flag(std::sync::atomic::AtomicBool); impl futures::task::ArcWake for Flag { fn wake_by_ref(flag: &Arc) { flag.0.store(true, std::sync::atomic::Ordering::SeqCst); } } impl Flag { fn waiter() -> (Arc, kio::Waiter) { let flag = Arc::new(Self(Default::default())); let waiter = kio::Waiter::new(futures::task::waker(flag.clone())); (flag, waiter) } fn take(&self) -> bool { self.0.swap(false, std::sync::atomic::Ordering::SeqCst) } } /// A read that found nothing parks again at once until something it parked on fires, /// but parks anew for another task, which the next group then wakes instead. #[test] fn an_idle_read_parks_for_the_task_polling_it() { for ordered in [false, true] { let producer = track_producer("test", Info::default()); let first = framed(&producer, 0, 1); first.finish().unwrap(); let mut reader = producer.subscribe(replay()); let mut read = |waiter: &kio::Waiter| match ordered { true => reader.poll_group(true, waiter), false => reader.poll_group(false, waiter), }; let (flag, waiter) = Flag::waiter(); assert!(matches!(read(&waiter), Poll::Ready(Ok(Some(_))))); assert!(read(&waiter).is_pending()); assert!(read(&waiter).is_pending(), "nothing fired, so it parks again"); assert!(!flag.take()); let (other, moved) = Flag::waiter(); assert!(read(&moved).is_pending()); let next = framed(&producer, 1, 1); next.finish().unwrap(); assert!(other.take(), "the next group wakes the task that polled last"); assert!(matches!(read(&moved), Poll::Ready(Ok(Some(_))))); } } /// Write group `key` with `frames` frames, left open. fn framed(producer: &Producer, key: u64, frames: usize) -> group::Producer { let mut group = producer.create_group(group::Info { start: key }).unwrap(); for _ in 0..frames { group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); } group } /// Every group a reader hands out with the frame indices it reads, until one would block. /// What an ordered read returns until it ends, as `group 2` or `gap 1..2 (error)`. #[cfg(feature = "async")] fn items(ordered: &mut Ordered) -> Vec { let mut items = Vec::new(); while let Some(next) = ordered.next_group().now_or_never() { match next { Ok(Some(Item::Group(group))) => items.push(format!("group {}", group.start)), Ok(Some(Item::Gap { range, error })) => items.push(format!("gap {range:?} ({error})")), Ok(None) => return items, Err(err) => { items.push(format!("error ({err})")); return items; } } } items.push("pending".to_string()); items } /// An ordered read that expects no gap, as the groups it returns. fn groups_only(polled: Poll>>) -> Poll>> { polled.map(|res| res.map(|item| item.map(|item| item.group().expect("a group, not a gap")))) } fn read_groups( mut next: impl FnMut(&kio::Waiter) -> Poll>>, ) -> Vec<(u64, Vec)> { let noop = kio::Waiter::noop(); let mut groups = Vec::new(); while let Poll::Ready(group) = next(&noop) { let Some(mut group) = group.unwrap() else { break }; let mut frames = Vec::new(); loop { let index = group.index(); match group.poll_read_frame(&noop) { Poll::Ready(Ok(Some(_))) => frames.push(index), Poll::Ready(Ok(None)) => break, res => panic!("group {} read {res:?}", group.start), } } groups.push((group.start, frames)); } groups } /// A range starting and ending inside cached groups reads only the frames it covers, /// in arrival and key order alike, however wide another subscriber made the /// aggregate. #[test] fn a_range_reads_only_its_groups() { let producer = track_producer("test", None); let _groups: Vec<_> = (3..8).map(|key| framed(&producer, key, 8).finish().unwrap()).collect(); let _wide = producer.subscribe(replay()); let range = Range { start: Anchor::Time(4), end: Some(Anchor::Time(5)), }; let bounded = || producer.subscribe(replay().with_ranges([range])); let want = vec![(4, (0..8).collect()), (5, (0..8).collect())]; let mut arrival = bounded(); assert_eq!(read_groups(|waiter| arrival.poll_recv_group(waiter)), want); let mut ordered = bounded().ordered(); assert_eq!(read_groups(|waiter| groups_only(ordered.poll_next_group(waiter))), want); } /// A drop outside a reader's ranges is reported once a group arriving places a time start /// below it, widening its range over the drop, as a scan of the whole window would. #[test] fn a_range_widened_by_a_group_reports_an_earlier_drop() { let producer = track_producer("test", Info::default()); // The range at 0 keeps the floor where it is, so only the window moves. let at = |time| Range { start: Anchor::Time(time), end: Some(Anchor::Time(time)), }; let mut reader = producer.subscribe(replay().with_ranges([at(0), at(8)])); let waiter = kio::Waiter::noop(); assert!(reader.poll_drop(&waiter).is_pending()); producer.drop_groups(5..6, StreamError::Cancel); assert!(reader.poll_drop(&waiter).is_pending(), "outside the range"); let _group = producer.create_group(group::Info { start: 3 }).unwrap(); assert_eq!( reader.poll_drop(&waiter), Poll::Ready((5..6, Hole::drop(StreamError::Cancel))), "the group containing time 8 starts at 3" ); } /// A group missing its head is never served short: reading it is [`Error::Lagged`], /// which a route copy recovers. #[test] fn a_group_missing_its_head_lags() { let producer = track_producer("test", Info::default()); let mut headless = producer.create_group(group::Info { start: 4 }).unwrap(); headless.start_at(2).unwrap(); for _ in 0..3 { headless .write_frame(crate::Timestamp::unscaled(headless.start), &b"x"[..]) .unwrap(); } headless.finish().unwrap(); let noop = kio::Waiter::noop(); let start = Anchor::Time(4); let mut sub = producer.subscribe(replay().with_ranges([Range { start, end: None }])); let Poll::Ready(Ok(Some(mut group))) = sub.poll_recv_group(&noop) else { panic!("no group"); }; assert_eq!(group.index(), 0); assert!(matches!(group.poll_read_frame(&noop), Poll::Ready(Err(Error::Lagged)))); } /// The floor a reader waiting on each group starts at is the higher of its ranges' and /// its local one, so it never waits on a group the local floor withholds. #[test] fn floor_honors_with_range() { let producer = track_producer("test", Info::default()); let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 2)).collect(); let reader = producer.subscribe(replay().with_range(2..)); assert_eq!((reader.floor(), reader.next_in_range(0)), (Some(2), Some(2))); let reader = reader.with_range(4..); assert_eq!((reader.floor(), reader.next_in_range(0)), (Some(4), Some(4))); assert_eq!(reader.next_in_range(5), Some(5)); // Below the ranges' floor, the ranges still decide. let reader = producer.subscribe(replay().with_range(3..)).with_range(1..); assert_eq!((reader.floor(), reader.next_in_range(0)), (Some(3), Some(3))); // A floor reaching back to the oldest group stays unknown. let reader = producer.subscribe(replay()).with_range(4..); assert_eq!(reader.floor(), None); } /// The largest reports only while it is within the bounds: a cap below it or a floor /// above it reports nothing, rather than a cached group standing in for it. #[test] fn largest_reports_only_within_the_bounds() { let producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 2)).collect(); let mut capped = producer.subscribe(replay()); capped.set_range(..3); assert!(capped.poll_largest(&noop).is_pending(), "the largest is past the cap"); // An inclusive cap reports the group it names. let mut ordered = producer.subscribe(replay()).ordered(); ordered.set_range(..=5); assert_eq!( ordered.poll_largest(&noop), Poll::Ready(Some(Position { group: 5, frame: 1 })) ); let mut floored = producer.subscribe(replay()); floored.set_range(8..); assert!( floored.poll_largest(&noop).is_pending(), "the largest is below the floor" ); let _late = framed(&producer, 8, 0); assert_eq!(floored.poll_largest(&noop), Poll::Ready(Some(Position::group(8)))); } /// Moving the bounds wakes a parked reader, which reports the largest once it is within /// them; bounds that leave it where it was report nothing new. #[test] fn largest_follows_the_bounds_as_they_move() { let producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 1)).collect(); let mut reader = producer.subscribe(replay()).ordered(); reader.set_range(..3); let (flag, waiter) = Flag::waiter(); assert!(reader.poll_largest(&waiter).is_pending()); reader.set_range(..); assert!(flag.take(), "lifting the cap woke the reader"); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); assert!(reader.poll_largest(&waiter).is_pending()); reader.set_range(..2); assert!(flag.take(), "lowering the cap woke the reader"); assert!( reader.poll_largest(&noop).is_pending(), "the largest is past the new cap" ); reader.set_range(..); assert!( reader.poll_largest(&noop).is_pending(), "lifting it again leaves the last report standing" ); reader.set_range(3..); assert!( reader.poll_largest(&noop).is_pending(), "a floor below the largest leaves it where it was" ); } /// Once the stream ended, widening the bounds over a largest it never reported does not /// bring it back. #[test] fn largest_stays_ended_when_the_bounds_widen() { let producer = track_producer("test", Info::default()); framed(&producer, 5, 1).finish().unwrap(); let producer = producer.finish().unwrap(); let noop = kio::Waiter::noop(); let mut reader = producer.subscribe(replay()); reader.set_range(..3); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); reader.set_range(..); assert_eq!( reader.poll_largest(&noop), Poll::Ready(None), "the stream already ended" ); } /// A dropped poll leaves no registration behind for a bounds move to wake. #[test] fn largest_forgets_a_dropped_poll() { let producer = track_producer("test", Info::default()); let mut reader = producer.subscribe(replay()); let (flag, waiter) = Flag::waiter(); assert!(reader.poll_largest(&waiter).is_pending()); drop(waiter); reader.set_range(..); assert!(!flag.take(), "the dropped poll was not woken"); } /// A poll that reported is not parked, so moving the bounds afterwards does not wake it. #[test] fn largest_wakes_only_a_parked_poll_on_a_bounds_move() { let producer = track_producer("test", Info::default()); let _group = framed(&producer, 0, 1); let mut reader = producer.subscribe(replay()); let (flag, waiter) = Flag::waiter(); assert_eq!(reader.poll_largest(&waiter), Poll::Ready(Some(Position::group(0)))); reader.set_range(..); assert!(!flag.take(), "the finished poll was not woken"); assert!(reader.poll_largest(&waiter).is_pending()); reader.set_range(..); assert!(flag.take(), "the parked poll was woken"); } /// Bounds that move while the copy is idle keep the last report, so going live again /// does not repeat it. #[test] fn largest_keeps_its_last_report_across_an_idle_move() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); let mut reader = producer.subscribe(replay()); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(2)))); producer.set_idle(); reader.set_range(..); assert!(reader.poll_largest(&noop).is_pending(), "the copy is idle"); producer.set_live(Some(Position::group(2)), None).unwrap(); assert!( reader.poll_largest(&noop).is_pending(), "the same largest is not reported again" ); let _next = framed(&producer, 3, 1); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(3)))); } /// An idle copy that declared an end it has not reached yet still waits for the live /// feed, while one that finished reports its settled largest. #[test] fn largest_waits_for_an_idle_copy_with_a_future_end() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); producer.finish_at(Position::group(10)).unwrap(); producer.set_idle(); let mut reader = producer.subscribe(replay()); assert!(reader.poll_largest(&noop).is_pending(), "the copy is idle"); producer.set_live(Some(Position::group(5)), None).unwrap(); let _answer = framed(&producer, 5, 1); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); let mut finished = track_producer("test", Info::default()); framed(&finished, 0, 1).finish().unwrap(); finished.set_idle(); let finished = finished.finish().unwrap(); let mut reader = finished.subscribe(replay()); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(0)))); } /// An idle copy's largest is what an ended feed left, so nothing reports until the /// route's answer shows, though the cache still holds groups. #[test] fn largest_waits_for_an_idle_copy_to_go_live() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); producer.set_idle(); let mut reader = producer.subscribe(replay()); let (flag, waiter) = Flag::waiter(); assert!(reader.poll_largest(&waiter).is_pending(), "the copy is idle"); producer.set_live(Some(Position::group(5)), None).unwrap(); producer.report_largest(Position::group(5), false); assert!( reader.poll_largest(&waiter).is_pending(), "the answer has not shown yet" ); flag.take(); let _answer = framed(&producer, 5, 1); assert!(flag.take(), "the answer showing woke the reader"); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); } /// A relay copy holding groups 0 to 3 from an ended feed, idle until its route answers. fn idle_copy() -> Producer { let mut producer = track_producer("test", None); for key in 0..4 { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); producer } /// Every group `reader` can take right now, in delivery order. fn ready_groups(reader: &mut Subscriber) -> Vec { let noop = kio::Waiter::noop(); let mut keys = Vec::new(); while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&noop) { keys.push(group.start); } keys } /// A timed track of 500 ms groups starting at 0 through 5 s, each presenting at its start. fn half_second_groups() -> Producer { let mut producer = track_producer("test", Info::timed()); for _ in (0..=5000).step_by(500) { append_half_second(&mut producer); } producer } /// Append the next 500 ms group, presenting at its start. fn append_half_second(producer: &mut Producer) -> u64 { let start = producer.state.read().latest_group.map_or(0, |latest| latest + 500); let mut group = producer.create_group(group::Info { start }).unwrap(); group .write_frame(Timestamp::from_millis(start).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); start } /// A join over `range` holding `min` behind the live edge, with a budget that keeps the cache. fn joining(range: Range, min: impl Into>) -> Subscription { Subscription::default() .with_ranges([range]) .with_delay(Delay::default().with_min(min).with_max(Duration::from_secs(30))) } /// A live join stating a Min Age starts at the newest group that began that far behind the /// newest frame, and plays through to live. #[test] fn a_live_join_reaches_back_by_its_min_age() { let mut producer = half_second_groups(); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!( ready_groups(&mut reader), vec![3000, 3500, 4000, 4500, 5000], "four groups back" ); assert_eq!(append_half_second(&mut producer), 5500); assert_eq!(ready_groups(&mut reader), vec![5500], "no gap to live"); // Measured from the newest frame, not the latest group's start. let producer = half_second_groups(); let mut group = producer.create_group(group::Info { start: 5500 }).unwrap(); group .write_frame(Timestamp::from_millis(5900).unwrap(), &b"x"[..]) .unwrap(); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!(ready_groups(&mut reader), vec![3500, 4000, 4500, 5000, 5500]); } /// A live join stating no Min Age, or zero, starts at the latest group, and a join from the /// oldest group reaches as far back as its budget keeps, whatever its Min Age. #[test] fn a_join_without_a_min_age_starts_where_its_range_says() { let producer = half_second_groups(); let all: Vec = (0..=5000).step_by(500).collect(); for min in [None, Some(Duration::ZERO)] { let mut live = producer.subscribe(joining(Range::live(), min)); assert_eq!(ready_groups(&mut live), vec![5000], "{min:?}"); } for min in [None, Some(Duration::ZERO), Some(Duration::from_secs(2))] { let mut oldest = producer.subscribe(joining(Range::oldest(), min)); assert_eq!(ready_groups(&mut oldest), all, "{min:?}"); } } /// A Min Age past everything held starts at the oldest group held, and never past what the /// budget keeps: a shorter budget starts at the oldest group still fresh. #[test] fn a_min_age_reaches_back_no_further_than_the_cache_or_the_budget() { let producer = half_second_groups(); let all: Vec = (0..=5000).step_by(500).collect(); let mut far = producer.subscribe(joining(Range::live(), Duration::from_secs(60))); assert_eq!(ready_groups(&mut far), all); let budget = Subscription::default().with_ranges([Range::live()]).with_delay( Delay::default() .with_min(Duration::from_secs(2)) .with_max(Duration::from_secs(1)), ); let mut budget = producer.subscribe(budget); assert_eq!(ready_groups(&mut budget), vec![4000, 4500, 5000]); } /// A start naming a time or a count back, or a range with an end, ignores the Min Age. #[test] fn a_named_start_or_an_end_ignores_the_min_age() { let producer = half_second_groups(); let min = Duration::from_secs(2); let cases = [ ( Range { start: Anchor::Time(1000), end: Some(Anchor::Time(2000)), }, vec![1000, 1500, 2000], ), ( Range { start: Anchor::Relative(1), end: None, }, vec![4500, 5000], ), ( Range { start: Anchor::Oldest, end: Some(Anchor::Time(1000)), }, vec![0, 500, 1000], ), ( Range { start: Anchor::Relative(0), end: Some(Anchor::Relative(0)), }, vec![5000], ), ]; for (range, expected) in cases { let mut reader = producer.subscribe(joining(range, min)); assert_eq!(ready_groups(&mut reader), expected, "{range:?}"); } } /// Changing the Min Age later moves nothing already started. #[test] fn an_update_to_the_min_age_moves_no_started_join() { let mut producer = half_second_groups(); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(1))); assert_eq!(ready_groups(&mut reader), vec![4000, 4500, 5000]); reader.update(joining(Range::live(), Duration::from_secs(3))).unwrap(); assert_eq!(ready_groups(&mut reader), Vec::::new()); append_half_second(&mut producer); assert_eq!(ready_groups(&mut reader), vec![5500]); } /// A scale-less track has no media time, so a join reaches back by arrival on this hop's /// clock, as expiry measures it. #[test] fn a_scale_less_join_reaches_back_by_arrival() { let producer = track_producer("test", None); // Group N arrives N seconds in, and the join comes at 5 s. for key in 0..5 { framed(&producer, key, 1).finish().unwrap(); elapse(&producer, Duration::from_secs(1)); } let mut reader = producer.subscribe(joining(Range::live(), Duration::from_millis(2500))); assert_eq!(ready_groups(&mut reader), vec![2, 3, 4]); let mut far = producer.subscribe(joining(Range::live(), Duration::from_secs(10))); assert_eq!(ready_groups(&mut far), vec![0, 1, 2, 3, 4]); framed(&producer, 5, 1).finish().unwrap(); assert_eq!(ready_groups(&mut reader), vec![5]); } /// A copy whose wire keys groups by sequence measures the reach back by where each group's /// first frame presented, not by its key. #[test] fn a_sequence_keyed_copy_reaches_back_by_first_frames() { let producer = track_producer("test", Info::timed()); producer.received(); producer.starts_at_keys(); for key in 0..10 { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(key * 500).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!(ready_groups(&mut reader), vec![5, 6, 7, 8, 9]); let mut far = producer.subscribe(joining(Range::live(), Duration::from_secs(60))); assert_eq!(ready_groups(&mut far), (0..10).collect::>()); } /// A live join reads none of the groups held until one measures the live edge: a copy /// holding only group headers gives nothing to reach back by, so no group is handed out /// before a first frame says where the join starts. #[test] fn a_live_join_reads_nothing_until_a_frame_measures_the_live_edge() { let producer = track_producer("test", Info::timed()); producer.received(); producer.starts_at_keys(); let groups: Vec<_> = (0..10) .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!(ready_groups(&mut reader), Vec::::new(), "nothing to measure by"); for (key, mut group) in (0..).zip(groups) { group .write_frame(Timestamp::from_millis(key * 500).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } let mut got = ready_groups(&mut reader); got.sort_unstable(); assert_eq!(got, vec![5, 6, 7, 8, 9]); } /// A join skips a group the publisher gave up: it starts at the newest group still served /// that began by the target, never at an aborted one that would floor it past that group. #[test] fn a_live_join_reaches_back_past_an_aborted_group() { let mut producer = track_producer("test", Info::timed()); for _ in (0..=2500).step_by(500) { append_half_second(&mut producer); } let aborted = producer.create_group(group::Info { start: 3000 }).unwrap(); aborted.abort(Error::Cancel).unwrap(); let mut group = producer.create_group(group::Info { start: 3500 }).unwrap(); group .write_frame(Timestamp::from_millis(3500).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); for _ in (4000..=5000).step_by(500) { append_half_second(&mut producer); } let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!(ready_groups(&mut reader)[0], 2500); } /// A Min Age that is not a whole number of ticks rounds up, so the join never starts newer /// than it allows. #[test] fn a_live_join_rounds_its_min_age_up_to_whole_ticks() { let producer = track_producer("test", Info::timed()); for start in 0..=2 { let mut group = producer.create_group(group::Info { start }).unwrap(); group .write_frame(Timestamp::from_millis(start).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } let mut reader = producer.subscribe(joining(Range::live(), Duration::from_micros(1500))); assert_eq!(ready_groups(&mut reader), vec![0, 1, 2]); } /// A newest group given up does not move the live edge: the join measures from the content /// still held, so it keeps the buffering prefix. #[test] fn a_live_join_measures_from_groups_still_held() { let producer = half_second_groups(); let aborted = producer.create_group(group::Info { start: 10_000 }).unwrap(); aborted.abort(Error::Cancel).unwrap(); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!(ready_groups(&mut reader)[0], 3000); } /// A lite-07 copy reaches back only to groups it holds, so the join starts at once and /// leaves no unheld prefix for its upstream to be asked: the newest group held starting at /// or before the target, or the oldest one held when none does. #[test] #[cfg(feature = "async")] fn a_received_copy_reaches_back_only_to_groups_it_holds() { let placed = |subscriber: &mut Subscriber| { subscriber .resolved_range(0) .map(|resolved| (resolved.start, resolved.floored)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 5000); let mut short = producer.subscribe(joining(Range::live(), Duration::from_secs(2))); assert_eq!( placed(&mut short), Some((5000, true)), "nothing held reaches back further" ); assert_eq!(drain(&mut short), vec![5000]); write_group(&producer, 3000); let mut held = producer.subscribe(joining(Range::live(), Duration::from_secs(1))); assert_eq!( placed(&mut held), Some((3000, true)), "the newest held at or before 4000" ); } /// A live join that has not started yet reaches back by the Min Age stated when it starts, /// though the update left its ranges as they were. #[test] fn a_min_age_update_reaches_a_join_not_yet_started() { let mut producer = track_producer("test", Info::timed()); let mut reader = producer.subscribe(joining(Range::live(), Duration::from_secs(1))); assert_eq!(ready_groups(&mut reader), Vec::::new()); reader.update(joining(Range::live(), Duration::from_secs(2))).unwrap(); for _ in (0..=5000).step_by(500) { append_half_second(&mut producer); } assert_eq!(ready_groups(&mut reader), vec![3000, 3500, 4000, 4500, 5000]); } /// A relative start taken while the copy is idle counts back from the live feed's /// latest group once the route answers, not from the newest one an ended feed left. #[test] fn a_relative_start_on_an_idle_copy_counts_from_the_live_feed() { let behind = Range { start: Anchor::Relative(1), end: None, }; for (ranges, expected) in [(Range::live(), vec![4]), (behind, vec![3, 4])] { let mut producer = idle_copy(); let mut reader = producer.subscribe(replay().with_ranges([ranges])); assert_eq!(ready_groups(&mut reader), Vec::::new(), "the copy is idle"); assert_eq!(reader.floor(), None, "the start waits for the live feed"); producer.set_live(Some(Position::group(4)), None).unwrap(); framed(&producer, 4, 1).finish().unwrap(); assert_eq!(ready_groups(&mut reader), expected, "{ranges:?}"); } } /// A relative start waits past groups the route resends below its answer: the feed /// goes live, and the start resolves, only once the answer shows. #[test] fn a_relative_start_on_an_idle_copy_waits_for_the_answer_to_show() { let mut producer = idle_copy(); let mut reader = producer.subscribe(replay().with_ranges([Range::live()])); producer.set_live(Some(Position::group(5)), None).unwrap(); framed(&producer, 4, 1).finish().unwrap(); assert_eq!(ready_groups(&mut reader), Vec::::new(), "the answer has not shown"); framed(&producer, 5, 1).finish().unwrap(); assert_eq!(ready_groups(&mut reader), vec![5]); } /// A route answer behind what an idle copy cached, with no Largest or one before the /// newest position the copy holds, is malformed, and the copy's cache is dropped. #[test] fn an_answer_behind_an_idle_copys_cache_is_malformed() { let behind = [None, Some(Position::group(2)), Some(Position { group: 3, frame: 0 })]; let current = [Some(Position { group: 3, frame: 1 }), Some(Position::group(4))]; for (largest, ok) in behind .map(|largest| (largest, false)) .into_iter() .chain(current.map(|largest| (largest, true))) { let mut producer = track_producer("test", None); framed(&producer, 2, 1).finish().unwrap(); framed(&producer, 3, 2).finish().unwrap(); producer.set_idle(); let mut reader = producer.subscribe(replay()); let answered = producer.set_live(largest, None); assert_eq!(answered.is_ok(), ok, "{largest:?}"); if let Err(err) = answered { assert!(matches!(err, Error::MalformedTrack), "{largest:?}"); // The session fails the copy, and its readers get the error, not the cache. producer.abort(err).unwrap(); let noop = kio::Waiter::noop(); assert!( matches!(reader.poll_recv_group(&noop), Poll::Ready(Err(Error::MalformedTrack))), "{largest:?}" ); } } let mut producer = idle_copy(); assert!(matches!( producer.set_live_ahead(Position::group(2), None), Err(Error::MalformedTrack) )); // Largest names the newest object, which an empty newest group does not hold. for (largest, ok) in [(Some(Position::group(2)), true), (None, false)] { let mut producer = track_producer("test", None); framed(&producer, 2, 1).finish().unwrap(); framed(&producer, 3, 0).finish().unwrap(); producer.set_idle(); assert_eq!(producer.set_live(largest, None).is_ok(), ok, "{largest:?}"); } } /// The answer is judged against what the ended feed left, not the next feed's groups /// that land before it: a group in flight at idle is sent again, past the answer. #[test] fn an_idle_copy_judges_its_answer_against_the_ended_feed() { let mut producer = track_producer("test", None); framed(&producer, 2, 1).finish().unwrap(); let _open = framed(&producer, 3, 1); producer.set_idle(); let resent = framed(&producer, 3, 3); producer.set_live(Some(Position { group: 3, frame: 0 }), None).unwrap(); drop(resent); } /// Largest names a moq-transport object by its ID, which a sparse group's frame count /// does not give. #[cfg(feature = "async")] #[test] fn an_idle_copy_judges_a_sparse_group_by_object_id() { let wire = group::Wire { sparse: true, first_object: true, first_id: 0, extensions: false, has_end: false, fetchable: true, last: None, }; for (object, ok) in [(50, false), (100, true)] { let mut producer = track_producer("test", None); let mut group = producer.create_group(group::Info { start: 2 }).unwrap(); group.set_wire(wire.clone()).unwrap(); for id in [0, 100] { let mut frame = group .create_object(frame::Header { info: frame::Info { size: 1, timestamp: Timestamp::unscaled(2), }, fields: Some(Arc::new(super::super::object::Fields { id, properties: Default::default(), })), inflated: None, }) .unwrap(); frame.write(&b"x"[..]).unwrap(); group = frame.finish().unwrap(); } group.finish().unwrap(); producer.set_idle(); let answered = producer.set_live( Some(Position { group: 2, frame: object, }), None, ); assert_eq!(answered.is_ok(), ok, "object {object}"); } } /// A group standing in for a missing Largest says nothing of how far the route's feed /// reaches, so one below the cache does not judge it. #[test] fn a_first_group_below_an_idle_copys_cache_is_not_judged() { let mut producer = idle_copy(); let mut reader = producer.subscribe(replay()); producer.set_live_from(Position::group(0), None); assert_eq!(ready_groups(&mut reader), vec![0, 1, 2, 3]); } /// An end the idle copy has not reached does not make its cache the whole track: until /// the route answers, it serves nothing and anchors no relative start. #[test] fn an_idle_copy_with_a_future_end_waits_for_the_live_feed() { let mut producer = idle_copy(); producer.finish_at(Position::group(10)).unwrap(); let mut live = producer.subscribe(replay().with_ranges([Range::live()])); let mut oldest = producer.subscribe(replay()); assert_eq!(ready_groups(&mut live), Vec::::new(), "the copy is idle"); assert_eq!(live.floor(), None, "the start waits for the live feed"); assert_eq!(ready_groups(&mut oldest), Vec::::new(), "the copy is idle"); producer.set_live(Some(Position::group(4)), None).unwrap(); framed(&producer, 4, 1).finish().unwrap(); assert_eq!(ready_groups(&mut live), vec![4]); assert_eq!(ready_groups(&mut oldest), vec![0, 1, 2, 3, 4]); } /// An idle copy that ends without going live makes its cache the whole track, so a /// relative start that waited counts back from its newest group, not from where the /// earlier feed began. #[test] fn a_relative_start_on_an_idle_copy_that_ends_counts_from_its_newest_group() { let producer = idle_copy(); let mut reader = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!(ready_groups(&mut reader), Vec::::new(), "the copy is idle"); let _producer = producer.finish().unwrap(); assert_eq!(ready_groups(&mut reader), vec![3]); } /// A live join's answer anchors a relative start where it says the feed is, counting back /// over the groups held below it, both for the copy's readers and for the bounds sent /// upstream, even once a newer group lands before the one it named. #[test] fn a_relative_start_counts_from_the_answered_edge() { let producer = track_producer("test", Info::default()); // Counting back counts groups held: two below the edge the answer names. for key in [2, 3] { framed(&producer, key, 1).finish().unwrap(); } let back = Range { start: super::super::subscription::Anchor::Relative(2), end: None, }; producer.report_edge(4); let mut reader = producer.subscribe(replay().with_ranges([back])); let upstream = producer.resolve_subscription(&replay().with_ranges([back])).unwrap(); assert_eq!(upstream.start(), Some(2), "the answer resolves the upstream bounds"); framed(&producer, 5, 1).finish().unwrap(); // The default max age serves only the live edge, so the groups held below it are stale. assert_eq!(ready_groups(&mut reader), vec![5]); assert_eq!(reader.floor(), Some(2), "the answer anchors the start"); } /// A cached observer must not keep an unfinished writer alive or serve its dead copy. #[kio::test] async fn dropped_group_writer_aborts_and_releases_its_cached_copy() { let pool = cache::Pool::unbounded(); let producer = track_producer_pooled("test", pool.clone()); let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); let mut reader = group.consume(); assert!(pool.used() > 0); drop(group); assert!(matches!( reader.poll_read_frame(&kio::Waiter::noop()), Poll::Ready(Err(Error::Dropped)) )); let state = producer.state.read(); assert!(state.lookup.get(&3).unwrap().is_aborted()); assert_eq!(state.lookup.get(&3).unwrap().cache_size(), 0); drop(state); assert_eq!(pool.used(), 0); } #[kio::test] async fn dropped_group_writer_is_absent_and_its_key_can_be_recreated() { let producer = track_producer("test", Info::timed()); let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); drop(group); // The writer gave the group up: a reader gets it, to read why it ended. let mut reader = producer.subscribe(Subscription::default().with_range(0..)); let Poll::Ready(Ok(Some(mut cut))) = reader.poll_recv_group(&kio::Waiter::noop()) else { panic!("the cut group is handed out"); }; assert!(matches!( cut.poll_read_frame(&kio::Waiter::noop()), Poll::Ready(Err(Error::Dropped)) )); let mut replacement = producer .create_group(group::Info { start: 3 }) .expect("the dead copy is replaceable"); replacement .write_frame(replacement.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); replacement.finish().unwrap(); let mut later = producer.subscribe(Subscription::default().with_range(0..)); assert!(matches!( later.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Ok(Some(_))) )); } #[kio::test] async fn dropped_group_writer_preserves_the_committed_resume_position() { let producer = track_producer("test", Info::timed()); let consumer = producer.consume(); let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); let handle = group.handle(); let _partial = group .create_frame(frame::Info { size: 2, timestamp: Timestamp::ZERO, }) .unwrap(); assert_eq!(handle.frame_count(), 2); drop(_partial); assert_eq!(handle.resume_frame(), Some(1)); assert_eq!(consumer.resume_position(), Some(Position { group: 3, frame: 1 })); } /// No group holds the reserved key, so its fetch fails at once rather than parking /// until every handler goes. #[test] #[cfg(feature = "async")] fn the_reserved_key_misses_with_a_handler() { let producer = track_producer("test", None); let dynamic = producer.requests(); let consumer = producer.consume(); let fetch = consumer.fetch_group(u64::MAX, None).now_or_never(); assert!(matches!(fetch, Some(Err(Error::NotFound)))); assert!(dynamic.requested_groups().now_or_never().is_none(), "nothing to fill"); } #[test] fn evict_expired_groups() { let producer = track_producer("test", Info::timed()); // Create 3 groups at time 0. producer.append_next().unwrap().finish().unwrap(); // seq 0 producer.append_next().unwrap().finish().unwrap(); // seq 1 producer.append_next().unwrap().finish().unwrap(); // seq 2 { let state = producer.state.read(); assert_eq!(live_groups(&state), 3); assert_eq!(first_arrival(&state), 0); } // Advance time past the pool's LRU window. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Append a new group to trigger eviction. producer.append_next().unwrap().finish().unwrap(); // seq 3 // Groups 0, 1, 2 are expired but seq 3 (the live edge) is kept. Their arrival // entries no longer resolve, so the leading ones are trimmed and the queue now // starts at seq 3's position. { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); assert_eq!(first_live_key(&state), 3); assert_eq!(first_arrival(&state), 3); assert!(!state.lookup.contains_key(&0)); assert!(!state.lookup.contains_key(&1)); assert!(!state.lookup.contains_key(&2)); assert!(state.lookup.contains_key(&3)); } } /// A group whose frames outlive the pool's LRU window is aged out when the next group starts, but /// a subscriber that already drained it must still see the clean end of group. Otherwise a /// track with long groups (a per-minute rollup, say) fails its readers at every boundary. #[kio::test] #[cfg(feature = "async")] async fn aging_out_a_finished_group_keeps_the_clean_end() { let producer = track_producer("test", Info::timed()); let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); let mut consumer = group.consume(); group .write_frame(Timestamp::from_millis(0).unwrap(), b"hello".as_slice()) .unwrap(); assert_eq!(consumer.next_frame().await.unwrap().unwrap().size, 5); // The group stays open well past the LRU window, then the next period starts. elapse(&producer, cache::DEFAULT_EXPIRY * 2); group.finish().unwrap(); let _next = producer.create_group(group::Info { start: 1 }).unwrap(); assert!(consumer.next_frame().await.unwrap().is_none()); } /// An actively-read group is not expired out from under its reader: every frame /// read restarts the retention clock. A group nobody reads still ages out on /// schedule, so reclamation stays intact. #[kio::test] #[cfg(feature = "async")] async fn active_reader_survives_expiry() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); // A finished group with one frame per step of the read loop below. let mut group = producer.create_group(0u64.into()).unwrap(); for _ in 0..10 { group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } group.finish().unwrap(); let mut reading = subscriber.assert_group(); // A sibling written at the same time that nobody ever reads. producer.create_group(1u64.into()).unwrap().finish().unwrap(); // Each step stays well inside the retention window, but the whole read // spans several windows. New groups keep the expiry scan running. for seq in 2..12u64 { elapse(&producer, cache::DEFAULT_EXPIRY / 2); let frame = reading.next_frame().await; assert!( matches!(frame, Ok(Some(_))), "an actively-read group must not expire mid-read (step {seq})" ); producer.create_group(seq.into()).unwrap().finish().unwrap(); } let state = producer.state.read(); assert!(state.lookup.contains_key(&0), "the read group survived"); assert!(!state.lookup.contains_key(&1), "the unread group still expired"); } /// Whole-frame reads served from the prefetch batch must also keep the group /// alive: the batch is filled (and stamped) once per `Prefetch::CAP` frames, /// which bounds frames, not elapsed time, so a slow `read_frame` reader has to /// re-stamp on a time bound between refills. #[kio::test] #[cfg(feature = "async")] async fn slow_prefetch_reader_survives_expiry() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let mut group = producer.create_group(0u64.into()).unwrap(); for _ in 0..20 { group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } group.finish().unwrap(); let mut reading = subscriber.assert_group(); // One whole-frame read per half-window: most are served straight from the // prefetch without locking. New groups keep the expiry scan running. for seq in 1..20u64 { elapse(&producer, cache::DEFAULT_EXPIRY / 2); let frame = reading.read_frame().await; assert!( matches!(frame, Ok(Some(_))), "a slow prefetch reader must not expire mid-read (step {seq})" ); producer.create_group(seq.into()).unwrap().finish().unwrap(); } } /// Receiving a group is itself a cache access: a subscriber that takes /// delivery just before the group would age out still gets to read it a full /// window later. #[kio::test] #[cfg(feature = "async")] async fn delivery_restarts_the_expiry_clock() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); let mut group = producer.create_group(0u64.into()).unwrap(); group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); group.finish().unwrap(); // A second group so seq 0 leaves the protected live edge. producer.create_group(1u64.into()).unwrap().finish().unwrap(); // Deliver just inside the window: the delivery stamps the group. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); let mut reading = subscriber.assert_group(); // Almost another full window passes: far beyond the write, inside the // delivery stamp. The new group runs the expiry scan. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); producer.create_group(2u64.into()).unwrap().finish().unwrap(); let frame = reading.read_frame().await.unwrap(); assert!(frame.is_some(), "a just-delivered group must not expire unread"); } /// Streaming chunks into an in-flight frame is a write access: a straggler /// group (behind the live edge) trickling a large frame across several /// retention windows must not be expired mid-write. #[test] fn streaming_frame_writes_keep_the_group_alive() { let producer = track_producer("test", Info::timed()); let straggler = producer.create_group(0u64.into()).unwrap(); // The live edge moves on, so the straggler is demoted and expirable. producer.create_group(1u64.into()).unwrap().finish().unwrap(); let mut frame = straggler .create_frame(frame::Info { size: 10, timestamp: Timestamp::ZERO, }) .unwrap(); // One chunk per half-window; the whole frame spans several windows. New // groups keep the expiry scan running. for seq in 2..12u64 { elapse(&producer, cache::DEFAULT_EXPIRY / 2); frame.write(bytes::Bytes::from_static(b"x")).unwrap(); producer.create_group(seq.into()).unwrap().finish().unwrap(); } let straggler = frame.finish().unwrap(); straggler.finish().unwrap(); let state = producer.state.read(); assert!( state.lookup.contains_key(&0), "a group streaming a frame survives expiry" ); } /// The wire ingest coalesces its chunk wakes to the poll boundary, so a payload /// whose tail arrives all at once completes without a single `frame_notify`. /// Committing is itself a write access: a group that just finished a frame must /// not be expired by the next track write on its stale frame-open stamp. #[test] fn coalesced_frame_completion_keeps_the_group_alive() { let producer = track_producer("test", Info::timed()); let straggler = producer.create_group(0u64.into()).unwrap(); // The live edge moves on, so the straggler is demoted and expirable. producer.create_group(1u64.into()).unwrap().finish().unwrap(); let mut frame = straggler .create_frame(frame::Info { size: 3, timestamp: Timestamp::ZERO, }) .unwrap(); // The sender stalls past the retention window, then the whole payload lands in // one poll turn: the loop never returns `Pending`, so `notify` is never reached // and `finish` is the only write the charge sees. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); frame.write_deferred(bytes::Bytes::from_static(b"abc")).unwrap(); let straggler = frame.finish().unwrap(); straggler.finish().unwrap(); // A new group runs the expiry scan. producer.create_group(2u64.into()).unwrap().finish().unwrap(); let state = producer.state.read(); assert!( state.lookup.contains_key(&0), "a group whose frame just completed must not expire" ); } /// Re-offering a parked group (once the cap rises) is a delivery: it restarts /// the expiry clock so the subscriber gets to read what it was just handed. #[kio::test] #[cfg(feature = "async")] async fn parked_reoffer_restarts_the_expiry_clock() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); subscriber.set_range(..1); for seq in 0..2u64 { let mut group = producer.create_group(seq.into()).unwrap(); group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); group.finish().unwrap(); } // Group 0 is in range; group 1 is beyond the cap and parks. assert_eq!(subscriber.assert_group().start, 0); subscriber.assert_no_group(); // Just inside the window, the cap rises and the re-offer stamps group 1. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); subscriber.set_range(..2); let mut reading = subscriber.assert_group(); assert_eq!(reading.start, 1); // Almost another full window passes: far beyond the write, inside the // re-offer stamp. The new group runs the expiry scan. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); producer.create_group(2u64.into()).unwrap().finish().unwrap(); let frame = reading.read_frame().await.unwrap(); assert!(frame.is_some(), "a just-re-offered group must not expire unread"); } #[test] fn evict_keeps_max_key() { let producer = track_producer("test", Info::timed()); let _open = producer.append_next().unwrap(); // seq 0 // Advance time past the LRU window. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Append another group; seq 0 is expired and evicted. let _open = producer.append_next().unwrap(); // seq 1 { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); assert_eq!(first_live_key(&state), 1); assert_eq!(first_arrival(&state), 1); } } /// A publisher that jumps far ahead and then keeps publishing below it holds the /// relay to what it caches: the protected high key never pins the dead entries /// behind it, and a live reader past them never walks them again. #[test] fn a_far_ahead_key_pins_nothing_behind_it() { let producer = track_producer("test", Info::timed()); let high = 1 << 40; // Read from the high key on: everything after lands below this reader's floor. let mut subscriber = producer.subscribe(Subscription::default()).with_range(high..); let waiter = kio::Waiter::noop(); producer .create_group(group::Info { start: high }) .unwrap() .finish() .unwrap(); let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the high group is handed out"); }; assert_eq!(group.start, high); let n = 4 * EVICT_SLACK as u64; for key in 0..n { elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); // Below the reader's floor: never handed out. assert!(subscriber.poll_recv_group(&waiter).is_pending()); } let state = producer.state.read(); assert!(state.lookup.contains_key(&high), "the live edge stays"); assert!( state.arrival.len() <= 2 * state.lookup.len() + EVICT_SLACK, "{} arrival entries for {} cached groups", state.arrival.len(), state.lookup.len() ); let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() }; assert_eq!( cursor.index, state.next_arrival, "the reader passed what it is never handed" ); } #[test] fn no_eviction_when_fresh() { let producer = track_producer("test", Info::timed()); producer.append_next().unwrap().finish().unwrap(); // seq 0 producer.append_next().unwrap().finish().unwrap(); // seq 1 producer.append_next().unwrap().finish().unwrap(); // seq 2 { let state = producer.state.read(); assert_eq!(live_groups(&state), 3); assert_eq!(first_arrival(&state), 0); } } #[test] fn consumer_skips_evicted_groups() { let producer = track_producer("test", Info::timed()); producer.append_next().unwrap().finish().unwrap(); // seq 0 let mut consumer = producer.subscribe(None); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer.append_next().unwrap().finish().unwrap(); // seq 1 // Group 0 was evicted. Consumer should get group 1. let group = consumer.assert_group(); assert_eq!(group.start, 1); } /// Mint a track under an origin whose pool has the given wall-clock LRU window. fn track_producer_expiring(name: impl Into>, expiry: impl Into>) -> Producer { track_producer_pooled(name, cache::Pool::new(cache::Config::default().with_expiry(expiry))) } /// Mint a track under an origin caching into `pool`. fn track_producer_pooled(name: impl Into>, pool: cache::Pool) -> Producer { Producer::new( Arc::new(broadcast::Info { pool, ..Default::default() }), name, Info::timed(), ) } /// The write path is not the only thing that runs expiry: a pool sweep reclaims a /// track's idle groups even when the track never writes again, which is the only /// bound on a publisher that stalls with a group still open. #[test] fn pool_sweep_expires_without_a_write() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let producer = track_producer_pooled("test", pool.clone()); let mut stalled = producer.append_next().unwrap(); // seq 0, left open stalled.write_frame(stalled.key_time(), b"x".as_slice()).unwrap(); let _open = producer.append_next().unwrap(); // seq 1, the live edge elapse(&producer, Duration::from_secs(2)); pool.sweep(); assert!( !producer.state.read().lookup.contains_key(&0), "the sweep reclaimed an idle open group with no write behind it" ); } #[test] fn cache_gc_dates_activity_before_expiring_it() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let now = crate::model::clock::now(); assert_eq!( pool.poll(now, &kio::Waiter::noop()), Some(now.saturating_add(Duration::from_millis(500))) ); let producer = track_producer_pooled("test", pool.clone()); let mut group = producer.append_next().unwrap(); group.write_frame(group.key_time(), b"first".as_slice()).unwrap(); let _open = producer.append_next().unwrap(); // Activity written while cleanup was idle gets the new supplied time. let later = now.saturating_add(Duration::from_secs(60)); pool.poll(later, &kio::Waiter::noop()); assert!(producer.state.read().lookup.contains_key(&0)); pool.poll(later.saturating_add(Duration::from_secs(2)), &kio::Waiter::noop()); assert!(!producer.state.read().lookup.contains_key(&0)); } #[test] fn cache_gc_reaches_old_entries_behind_a_fresh_front() { let expiry = Duration::from_secs(1); let pool = cache::Pool::new(cache::Config::default().with_expiry(expiry)); let producer = track_producer_pooled("test", pool.clone()); let now = crate::model::clock::now(); let groups: Vec<_> = (0..EVICT_SCAN * 3).map(|_| producer.append_next().unwrap()).collect(); let _open = producer.append_next().unwrap(); pool.poll(now, &kio::Waiter::noop()); // Keep more than a write scan's worth of leading entries fresh. for group in &groups[..EVICT_SCAN * 2] { group.handle().cache_refresh(); } pool.poll(now.saturating_add(expiry * 2), &kio::Waiter::noop()); let state = producer.state.read(); for key in 0..EVICT_SCAN * 2 { assert!(state.lookup.contains_key(&(key as u64)), "fresh front survives"); } for key in EVICT_SCAN * 2..EVICT_SCAN * 3 { assert!(!state.lookup.contains_key(&(key as u64)), "old tail is reclaimed"); } } /// One sweep drains a whole idle backlog, not a rotating window of it: a quiet /// track has no writes left to revisit the rest of the queue with, so a bounded /// pass would leave the oldest groups parked for a backlog's length in windows. #[test] fn pool_sweep_drains_a_deep_backlog() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let producer = track_producer_pooled("test", pool.clone()); // Comfortably more than one write-driven scan window (EVICT_SCAN). let backlog = 4 * EVICT_SCAN; let mut _open = Vec::new(); for _ in 0..backlog { let mut group = producer.append_next().unwrap(); group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); _open.push(group); } let _open = producer.append_next().unwrap(); // the live edge, always protected elapse(&producer, Duration::from_secs(2)); pool.sweep(); let state = producer.state.read(); let stale = (0..backlog as u64).filter(|seq| state.lookup.contains_key(seq)).count(); assert_eq!(stale, 0, "one sweep reclaimed the whole idle backlog"); } #[test] fn pool_expiry_controls_eviction() { // A shorter LRU window on the pool evicts sooner than the default. let producer = track_producer_expiring("test", Duration::from_secs(1)); let _open = producer.append_next().unwrap(); // seq 0 // Past the pool's window but well within cache::DEFAULT_EXPIRY. elapse(&producer, Duration::from_secs(2)); let _open = producer.append_next().unwrap(); // seq 1 // Seq 0 is gone because the pool only keeps idle groups for 1s. let state = producer.state.read(); assert_eq!(live_groups(&state), 1); assert_eq!(first_live_key(&state), 1); } #[test] fn small_frame_write_expires_idle_siblings() { let producer = track_producer_expiring("test", Duration::from_secs(1)); producer.append_next().unwrap().finish().unwrap(); // seq 0 let mut live = producer.append_next().unwrap(); // seq 1 elapse(&producer, Duration::from_secs(2)); live.write_frame(live.key_time(), b"x".as_slice()).unwrap(); let expired = !producer.state.read().lookup.contains_key(&0); assert!(expired, "a small frame write runs expiry"); } #[test] fn fresh_expiry_scan_does_not_wake_track_consumers() { use std::sync::atomic::{AtomicBool, Ordering}; let producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); producer.append_next().unwrap().finish().unwrap(); let mut live = producer.append_next().unwrap(); let mut consumer = producer.subscribe(replay()); assert_eq!(consumer.assert_group().start, 0); assert_eq!(consumer.assert_group().start, 1); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(consumer.poll_recv_group(&waiter).is_pending()); // A frame write wakes the readers following the largest, and only them. The live group's // first frame is the head it already raised the largest to, so the second one advances it. let mut follower = producer.subscribe(replay()); let noop = kio::Waiter::noop(); assert_eq!(follower.poll_largest(&noop), Poll::Ready(Some(Position::group(1)))); live.write_frame(live.key_time(), b"x".as_slice()).unwrap(); let advanced = Arc::new(AtomicBool::new(false)); let largest = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(advanced.clone())))); assert!(follower.poll_largest(&largest).is_pending()); live.write_frame(live.key_time(), b"y".as_slice()).unwrap(); assert!( !woken.load(Ordering::SeqCst), "a no-op expiry scan must not wake track consumers" ); assert!(advanced.load(Ordering::SeqCst), "the frame advanced the largest"); } #[test] fn streaming_frame_write_expires_idle_siblings() { let producer = track_producer_expiring("test", Duration::from_secs(1)); producer.append_next().unwrap().finish().unwrap(); // seq 0 let live = producer.append_next().unwrap(); // seq 1 let mut frame = live .create_frame(frame::Info { size: 1, timestamp: Timestamp::ZERO, }) .unwrap(); elapse(&producer, Duration::from_secs(2)); frame.write(b"x".as_slice()).unwrap(); let expired = !producer.state.read().lookup.contains_key(&0); assert!(expired, "a streamed chunk runs expiry"); } /// A stalled track keeps its newest group and gives up the ones before it by the wall /// clock: `max_age` measures how long ago each group's successor arrived as well as /// media time, so a publisher that stops writing cannot pin its superseded groups. #[test] #[cfg(feature = "async")] fn a_stalled_track_keeps_only_its_newest_group() { let mut producer = track_producer("test", Info::timed().with_max_age(Duration::from_secs(1))); for millis in [0, 10, 20] { append_at(&mut producer, millis); } assert_eq!(drain(&mut producer.subscribe(replay())), vec![0, 1, 2]); // Nothing is written again: media time stands still. elapse(&producer, Duration::from_secs(2)); assert_eq!( drain(&mut producer.subscribe(replay())), vec![2], "the newest group is kept" ); assert_eq!( live_groups(&producer.state.read()), 3, "reclaiming the bytes is the pool's job" ); } /// A peer whose timestamps barely advance never ages a group out on media time. The wall /// clock still does. #[test] #[cfg(feature = "async")] fn frozen_timestamps_still_age_out_by_wall_clock() { let mut producer = track_producer("test", Info::timed()); let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))); append_frozen(&mut producer); elapse(&producer, Duration::from_millis(500)); append_frozen(&mut producer); elapse(&producer, Duration::from_millis(800)); append_frozen(&mut producer); // Group 0's successor arrived 800ms ago, group 1's just now. assert_eq!(drain(&mut producer.subscribe(budget.clone())), vec![0, 1, 2]); elapse(&producer, Duration::from_millis(200)); assert_eq!(drain(&mut producer.subscribe(budget.clone())), vec![1, 2]); elapse(&producer, Duration::from_secs(1)); assert_eq!(drain(&mut producer.subscribe(budget)), vec![2]); } /// Groups that arrive at once are judged by media time, which convicts them before the /// wall clock can. #[test] #[cfg(feature = "async")] fn a_startup_burst_ages_out_by_media_time() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } // Group N reaches N + 1 seconds against the 4s edge: groups 0 and 1 are 3s and 2s // behind, past 1.5s. Their successors arrived just now. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500)))); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } /// A read parked on a superseded group is released by the wall clock alone: nothing is /// written after its successor arrives, so no write could wake it. That holds whether a /// driver collected the pool before the read parked or only after. #[kio::test] #[cfg(feature = "async")] async fn a_parked_read_wakes_at_its_wall_deadline() { for (info, primed) in [Info::default().with_max_age(None), Info::timed()] .into_iter() .flat_map(|info| [(info.clone(), true), (info, false)]) { let producer = track_producer("test", info.clone()); // The first group, at key 0. let at = info.timescale.map_or(Timestamp::unscaled(0), |_| Timestamp::ZERO); if primed { elapse(&producer, Duration::ZERO); } let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); let mut held = producer.append_next().unwrap(); held.write_frame(at, bytes::Bytes::from_static(b"x")).unwrap(); let mut reading = subscriber.assert_group(); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); // The successor never presents a frame, so media time never bounds the held group. let _successor = producer.append_next().unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(reading.poll_read_frame(&waiter).is_pending()); if !primed { elapse(&producer, Duration::ZERO); assert!( woken.swap(false, Ordering::SeqCst), "{info:?}: the first collection never woke the read" ); assert!(reading.poll_read_frame(&waiter).is_pending()); } elapse(&producer, Duration::from_millis(900)); assert!( !woken.load(Ordering::SeqCst), "{info:?} primed={primed}: woken before the deadline" ); elapse(&producer, Duration::from_millis(100)); assert!( woken.load(Ordering::SeqCst), "{info:?} primed={primed}: the wall deadline never fired" ); let result = reading.poll_read_frame(&waiter); assert!(matches!(result, Poll::Ready(Err(Error::Old))), "{info:?}: {result:?}"); } } /// The next group in range passes over the starts the publisher declared skipped, /// whether by appending past them or outright, and those given up for good, so a reader /// waiting on each group in key need not wait for them. #[test] fn the_next_group_in_range_passes_over_skips() { let producer = track_producer("test", Info::default().with_max_age(None)); let subscriber = producer.subscribe(Subscription::default()); producer.append_group(Timestamp::unscaled(0)).unwrap().finish().unwrap(); assert_eq!(subscriber.next_in_range(1), Some(1), "nothing skipped yet"); assert!(!subscriber.is_skipped(1..2)); producer .append_group(Timestamp::unscaled(10)) .unwrap() .finish() .unwrap(); assert_eq!(subscriber.next_in_range(1), Some(10)); assert_eq!(subscriber.next_in_range(11), Some(11)); assert!(subscriber.is_skipped(1..10)); assert!(!subscriber.is_skipped(0..10), "group 0 is held"); assert!(!subscriber.is_skipped(1..11), "group 10 is held"); assert!(subscriber.is_skipped(5..5), "an empty range"); producer.skip_groups(11..20); assert_eq!(subscriber.next_in_range(11), Some(20)); assert!(subscriber.is_skipped(11..20)); assert!(!subscriber.is_skipped(11..21)); // A drop for good is passed over too, as a wire before lite-07 forwards a skip, but // it was produced, so it is no skip. producer.drop_groups(20..30, StreamError::NotFound); assert_eq!(subscriber.next_in_range(20), Some(30)); assert!(!subscriber.is_skipped(20..30)); } /// A group finished whole ages from its own end, which a zero budget waits for too: a /// fetch before the end still gets it, and one after is told it is old. #[kio::test] #[cfg(feature = "async")] async fn a_whole_group_ages_only_once_it_ends() { let producer = track_producer("test", Info::timed().with_max_age(Duration::ZERO)); elapse(&producer, Duration::ZERO); let mut group = producer.append_group(Timestamp::from_millis(0).unwrap()).unwrap(); group .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) .unwrap(); group.finish_at(40).unwrap(); let consumer = producer.consume(); elapse(&producer, Duration::from_millis(39)); let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); assert!(fetched.is_ok(), "before its end: {:?}", fetched.err()); elapse(&producer, Duration::from_millis(1)); let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); } /// A successor whose header landed but whose first frame never does still starts the /// wall clock, so a relay ages the group before it as the origin does. #[kio::test] #[cfg(feature = "async")] async fn a_frameless_successor_ages_its_predecessor() { for info in [Info::default().with_max_age(None), Info::timed()] { let producer = track_producer("test", info.clone()); // The first group, at key 0. let at = info.timescale.map_or(Timestamp::unscaled(0), |_| Timestamp::ZERO); elapse(&producer, Duration::ZERO); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); let mut held = producer.append_next().unwrap(); held.write_frame(at, bytes::Bytes::from_static(b"x")).unwrap(); let mut reading = subscriber.assert_group(); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(reading.poll_read_frame(&waiter).is_pending()); let _successor = producer.create_group(group::Info { start: 1 }).unwrap(); assert!( woken.swap(false, Ordering::SeqCst), "{info:?}: the header never woke the read" ); assert!(reading.poll_read_frame(&waiter).is_pending()); elapse(&producer, Duration::from_millis(900)); assert!(!woken.load(Ordering::SeqCst), "{info:?}: woken before the deadline"); elapse(&producer, Duration::from_millis(100)); assert!(woken.load(Ordering::SeqCst), "{info:?}: the wall deadline never fired"); let result = reading.poll_read_frame(&waiter); assert!(matches!(result, Poll::Ready(Err(Error::Old))), "{info:?}: {result:?}"); } } /// A handed-out group stays judgeable after the cache evicts it, and its successor too: /// under a budget past the cache window, a stream still waiting on it (a FIN never /// acknowledged) is convicted once its successor's arrival plus the budget passes, not /// by whichever newer group the cache still holds. The newest group never is. A /// subscribed group and a fetched one are judged alike. #[test] fn an_evicted_group_is_judged_by_its_evicted_successor() { let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); let budget = cache::DEFAULT_EXPIRY * 2; let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let waiter = kio::Waiter::noop(); let expired = |held: &mut [group::Consumer; 2]| held.each_mut().map(|group| group.poll_expired(&waiter)); // Frozen timestamps, so the wall clock alone convicts. append_frozen(&mut producer); let fetch = group::Fetch::default().with_max_delay(budget); let mut held = [ subscriber.assert_group(), producer .consume() .fetch_group(0, fetch) .now_or_never() .unwrap() .unwrap(), ]; for group in &mut held { assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); } assert_eq!(expired(&mut held), [false; 2], "the newest group"); elapse(&producer, Duration::from_secs(1)); let mut last = append_frozen(&mut producer); // the successor assert_eq!(expired(&mut held), [false; 2]); // The track runs on, a group a second, until a second short of the deadline. for _ in 0..budget.as_secs() - 1 { elapse(&producer, Duration::from_secs(1)); last = append_frozen(&mut producer); } producer.broadcast.pool.sweep(); let state = producer.state.read(); assert!( !state.lookup.contains_key(&0) && !state.lookup.contains_key(&1), "both evicted" ); drop(state); assert_eq!(expired(&mut held), [false; 2], "early"); elapse(&producer, Duration::from_secs(1)); assert_eq!(expired(&mut held), [true; 2], "the successor arrived a budget ago"); // The newest group outlives any budget. let mut newest = subscriber.assert_group(); while newest.start < last { newest = subscriber.assert_group(); } elapse(&producer, budget * 2); assert!(!newest.poll_expired(&waiter), "the newest group"); } /// A recorded successor that failed hands the clock on to the next live group even once /// the cache reclaimed it before the reader looked again, rather than convicting by the /// failed one's arrival. A subscribed group and a fetched one are judged alike. #[test] fn a_reclaimed_failed_successor_hands_the_clock_on() { let mut producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); let budget = cache::DEFAULT_EXPIRY * 2; let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let waiter = kio::Waiter::noop(); let expired = |held: &mut [group::Consumer; 2]| held.each_mut().map(|group| group.poll_expired(&waiter)); append_frozen(&mut producer); let fetch = group::Fetch::default().with_max_delay(budget); let mut held = [ subscriber.assert_group(), producer .consume() .fetch_group(0, fetch) .now_or_never() .unwrap() .unwrap(), ]; for group in &mut held { assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); } let failed = producer.append_next().unwrap(); assert_eq!(expired(&mut held), [false; 2], "the successor is recorded"); failed.abort(Error::Cancel).unwrap(); let replacement = Duration::from_secs(10); elapse(&producer, replacement); append_frozen(&mut producer); producer.broadcast.pool.sweep(); assert!(!producer.state.read().lookup.contains_key(&1), "reclaimed"); elapse(&producer, budget - replacement); assert_eq!( expired(&mut held), [false; 2], "the failed successor arrived a budget ago" ); elapse(&producer, replacement); assert_eq!(expired(&mut held), [true; 2], "the next live one arrived a budget ago"); } /// On an ended track the cache expires idle groups in place rather than reclaiming them, /// so the reads skip them like any aborted group. A recorded successor that aged out that /// way still convicts the group before it once it arrived a budget ago. #[test] fn an_ended_track_judges_by_its_expired_successor() { let pool = cache::Pool::new(cache::Config::default().with_expiry(cache::DEFAULT_EXPIRY)); let mut producer = track_producer_pooled("test", pool.clone()); let budget = cache::DEFAULT_EXPIRY * 2; let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let waiter = kio::Waiter::noop(); append_frozen(&mut producer); let mut held = subscriber.assert_group(); assert!(matches!(held.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(held.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); pool.step(Duration::from_secs(1)); append_frozen(&mut producer); // the successor assert!(!held.poll_expired(&waiter), "the successor is recorded"); let state = producer.state.downgrade(); let producer = producer.finish().unwrap(); drop(producer); // The first sweep dates the groups idle, the second expires them. let idle = cache::DEFAULT_EXPIRY + Duration::from_secs(1); pool.sweep(); pool.step(idle); pool.sweep(); assert_eq!( state.read(|state| state.lookup.get(&1).is_some_and(Slot::is_aborted)), Some(true), "expired in place" ); assert!(!held.poll_expired(&waiter), "early"); pool.step(budget - idle); assert!(held.poll_expired(&waiter), "the successor arrived a budget ago"); } /// Empty groups are handed out like any other and retired by the wall clock like an /// untimed one, whether they finished without a frame or are still open with none. A /// publisher flooding a run of them cannot pin the group before the run either. #[test] #[cfg(feature = "async")] fn a_run_of_empty_groups_ages_out_by_wall_clock() { const RUN: u64 = 1000; let mut producer = track_producer("test", Info::timed()); let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))); let mut subscriber = producer.subscribe(budget.clone()); append_at(&mut producer, 0); let mut open = Vec::new(); for key in 1..=RUN { let group = producer.append_next().unwrap(); match key % 2 { 0 => { group.finish().unwrap(); } _ => open.push(group), } } append_at(&mut producer, 10); assert_eq!(drain(&mut producer.subscribe(budget.clone())).len() as u64, RUN + 2); elapse(&producer, Duration::from_secs(1)); assert_eq!(drain(&mut subscriber), vec![RUN + 1], "the whole run is retired"); assert_eq!(drain(&mut producer.subscribe(budget)), vec![RUN + 1]); } /// Disabling the pool's expiry keeps idle groups until byte pressure reclaims them. #[test] fn disabled_pool_expiry_never_reclaims() { let producer = track_producer_expiring("test", None); producer.append_next().unwrap().finish().unwrap(); // seq 0 elapse(&producer, Duration::from_secs(3600)); producer.append_next().unwrap().finish().unwrap(); // seq 1 let state = producer.state.read(); assert_eq!(live_groups(&state), 2); } #[test] fn max_delay_clamped_to_cache() { let producer = track_producer_capped("test", Info::timed(), Duration::from_secs(2)); // A max delay budget beyond the cache is capped in the aggregate; a group can't be // waited for longer than the cache keeps it. The subscriber's own preference // is stored verbatim, so what it asked for stays readable. let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); assert_eq!(subscriber.subscription().delay.max, Some(Duration::from_secs(10))); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::from_secs(2))); // A budget within the cache is left alone, and ZERO (skip immediately) stays ZERO. subscriber .update(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))) .unwrap(); assert_eq!( producer.subscription().unwrap().delay.max, Some(Duration::from_millis(500)) ); subscriber .update(Subscription::default().with_delay(Delay::default().with_max(Duration::ZERO))) .unwrap(); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::ZERO)); } /// Mint a track under an origin whose max age ceiling is `cap`, so the /// local delivery budget is capped without rewriting the publisher's max age. fn track_producer_capped(name: impl Into>, info: Info, cap: Duration) -> Producer { Producer::new( Arc::new(broadcast::Info { cache_duration: Some(cap), ..Default::default() }), name, info, ) } /// A fill that lands before the track's metadata installs provisional info with no /// publisher max age: the real bound is unknown, and an authored track's zero would /// refuse history an unlimited publisher still serves. #[test] fn a_fill_before_metadata_claims_no_max_age() { let state = TrackState::spawn(Arc::new(broadcast::Info::default())); let _group = state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds::default(), None) .unwrap() .expect("a fresh group"); let state = state.read(); assert_eq!(state.info.as_ref().unwrap().max_age, None); assert_eq!(state.max_age_bound(), None); } /// A refused request stops protecting the live edge a fill cached before the refusal: it /// joins the eviction order, so pool pressure and idle expiry can still reclaim it. #[test] fn a_refusal_releases_the_live_edge_a_fill_cached() { let state = TrackState::spawn(Arc::new(broadcast::Info::default())); let mut guard = state.write().ok().unwrap(); let _group = guard .insert_group_request(0, group::Bounds::default(), None) .unwrap() .expect("a fresh group"); assert!(guard.protects(0)); guard.reject(Error::NotFound, None); assert!(!guard.protects(0)); assert!( guard.evict.iter().any(|(key, _)| *key == 0), "the released live edge is reclaimable" ); } #[test] fn optional_age_keeps_publisher_metadata_and_local_policy_separate() { assert_eq!(Info::default().max_age, Some(Duration::ZERO)); assert_eq!(Info::timed().max_age, None); for age in [None, Some(Duration::ZERO), Some(Duration::from_secs(30))] { let producer = track_producer_capped("optional", Info::timed().with_max_age(age), Duration::from_secs(1)); assert_eq!(producer.info.max_age, age); assert_eq!(producer.subscribe(None).info().max_age, age); assert_eq!( producer.state.read().max_age_bound(), Some(age.unwrap_or(Duration::MAX).min(Duration::from_secs(1))) ); } } #[test] fn origin_cache_duration_clamps_max_age() { // A publisher asking to keep groups for a minute is capped to the origin's 1s // ceiling; a publisher already below the ceiling is left alone (it's a min). let capped = track_producer_capped( "test", Info::timed().with_max_age(Duration::from_secs(60)), Duration::from_secs(1), ); assert_eq!(capped.state.read().max_age_bound(), Some(Duration::from_secs(1))); assert_eq!(capped.subscribe(None).info().max_age, Some(Duration::from_secs(60))); let under = track_producer_capped( "test", Info::timed().with_max_age(Duration::from_millis(500)), Duration::from_secs(1), ); assert_eq!(under.state.read().max_age_bound(), Some(Duration::from_millis(500))); } /// The origin ceiling clamps the media-timestamp budget only; wall-clock /// reclamation belongs to the pool's LRU window, not the ceiling. #[test] fn origin_cache_duration_does_not_wall_evict() { let producer = track_producer_capped( "test", Info::timed().with_max_age(Duration::from_secs(60)), Duration::from_secs(1), ); producer.append_next().unwrap().finish().unwrap(); // seq 0 // Far past the ceiling in wall time, but inside the pool's LRU window. elapse(&producer, Duration::from_secs(2)); producer.append_next().unwrap().finish().unwrap(); // seq 1 let state = producer.state.read(); assert_eq!(live_groups(&state), 2); } #[test] fn max_delay_clamped_via_every_update_path() { let producer = track_producer_capped("test", Info::timed(), Duration::from_secs(2)); let over = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); // The clamp lives in the aggregation, so it applies no matter which entry point // wrote the raw preference. Previously only `Subscriber::update` clamped. let mut subscriber = producer.subscribe(over.clone()); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::from_secs(2))); subscriber.control().update(over.clone()).unwrap(); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::from_secs(2))); subscriber.update(over).unwrap(); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::from_secs(2))); } #[test] fn max_delay_aggregate_clamps_across_subscribers() { let producer = track_producer_capped("test", Info::timed(), Duration::from_secs(2)); // The aggregate takes the max, then clamps once. Equivalent to clamping each // subscriber first, since `min` distributes over `max`. let _a = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let _b = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); assert_eq!(producer.subscription().unwrap().delay.max, Some(Duration::from_secs(2))); } /// The publisher enforces its own max age, so the demand asked of it is the subscribers' /// own, and a request asks for the same before and after its info lands. #[kio::test] #[cfg(feature = "async")] async fn the_aggregate_is_not_clamped_to_the_publisher_max_age() { let broadcast = broadcast::Info { cache_duration: Some(Duration::from_secs(30)), ..Default::default() } .produce(); let mut dynamic = broadcast.requests(); let consumer = broadcast.consume(); let mut waiting = std::pin::pin!( consumer .track("test") .unwrap() .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60)))) ); assert!(futures::poll!(waiting.as_mut()).is_pending()); let request = dynamic.requested_track().await.unwrap(); // Capped by the local ceiling alone. let before = request.subscription().unwrap().delay.max; assert_eq!(before, Some(Duration::from_secs(30))); let producer = request.accept(Info::timed().with_max_age(Duration::from_secs(2))); assert_eq!(producer.subscription().unwrap().delay.max, before); assert_eq!(producer.state.read().max_age_bound(), Some(Duration::from_secs(2))); // A session's tail still waits for a late group no longer than the publisher keeps it, // even for a subscriber that defers to the max age alone. assert_eq!(producer.tail_grace(Duration::from_secs(60)), Duration::from_secs(2)); assert_eq!(producer.tail_grace(Duration::MAX), Duration::from_secs(2)); waiting.await.unwrap(); } /// A ceiling that doesn't lower an unbounded demand leaves it unbounded: a huge one /// (`Some(Duration::MAX)`) must not reach the wire as a value no version can carry. #[kio::test] #[cfg(feature = "async")] async fn a_huge_ceiling_keeps_an_unbounded_demand_unbounded() { let broadcast = broadcast::Info { cache_duration: Some(Duration::MAX), ..Default::default() } .produce(); let mut dynamic = broadcast.requests(); let consumer = broadcast.consume(); let unbounded = Subscription::default().with_delay(Delay::default().with_max(None)); let mut waiting = std::pin::pin!(consumer.track("test").unwrap().subscribe(unbounded)); assert!(futures::poll!(waiting.as_mut()).is_pending()); let request = dynamic.requested_track().await.unwrap(); assert_eq!(request.subscription().unwrap().delay.max, None); request.accept(Info::timed()); waiting.await.unwrap(); } #[test] fn churned_subscribers_do_not_accumulate() { let producer = track_producer("test", Info::timed()); let consumer = producer.consume(); let _steady = producer.subscribe(None); // Departures remove their contribution even when nobody polls the aggregate. for _ in 0..100 { drop(producer.subscribe(None)); drop(consumer.subscribe(None)); } // Only the steady reader remains registered. let subs = producer.state.read().subscriptions.clone(); let len = subs.lock().len(); assert_eq!(len, 1, "departed subscribers accumulated"); } #[test] fn departures_release_demand_after_a_peak() { let mut producer = track_producer("test", Info::timed()); let waiter = kio::Waiter::noop(); let _steady = producer.subscribe(None); assert!(producer.poll_subscription_changed(&waiter).is_ready()); // A departed burst leaves no demand, even when its preferences matched the steady reader. drop((0..1000).map(|_| producer.subscribe(None)).collect::>()); for _ in 0..100 { drop(producer.subscribe(None)); assert!(producer.poll_subscription_changed(&waiter).is_pending()); let subs = producer.state.read().subscriptions.clone(); assert_eq!(subs.lock().len(), 1, "departed demand survived"); } } /// Append a finished group presenting at `millis`, so the track carries a media /// timeline for the drift budget to measure against. fn append_at(producer: &mut Producer, millis: u64) -> u64 { let mut group = producer.append_next().unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); let group = group.finish().unwrap(); group.start } /// Append a finished group presenting at its key in milliseconds: a timeline that barely /// advances, since starts strictly increase and can't stop. fn append_frozen(producer: &mut Producer) -> u64 { let mut group = producer.append_next().unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let group = group.finish().unwrap(); group.start } /// Every group the subscriber can read right now, in delivery order. #[cfg(feature = "async")] fn drain(subscriber: &mut Subscriber) -> Vec { let mut keys = Vec::new(); while let Some(Ok(Some(group))) = subscriber.recv_group().now_or_never() { keys.push(group.start); } keys } /// A rejoined copy judges the route's answer by the groups that survived the leave. /// /// The cancel that idles a copy resets the group in flight. The route then answers past /// it, so the newest group still cached is two behind: a gap, which the cached group /// cannot be measured across. Judged by the group the copy held when it went idle, the /// answer would read as contiguous, and the cached group would reach all the way to the /// answer, fresh to any positive budget. #[test] #[cfg(feature = "async")] fn an_answer_past_a_reset_group_skips_the_cache() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(500).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); producer.set_idle(); open.abort(Error::Cancel).unwrap(); let mut answer = producer.create_group(group::Info { start: 2 }).unwrap(); producer.set_live(Some(Position::group(2)), None).unwrap(); answer .write_frame(Timestamp::from_millis(533).unwrap(), bytes::Bytes::from_static(b"")) .unwrap(); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(400)))); assert_eq!(drain(&mut subscriber), vec![2]); // A one-shot fetch still serves that cache: an unknown reach is no proof // that the requested group exceeded the fetch's budget. let mut fetched = producer .consume() .fetch_group(0, group::Fetch::default().with_max_delay(Duration::from_millis(400))) .now_or_never() .expect("the cache answers") .expect("cached fetch"); assert!(fetched.read_frame().now_or_never().unwrap().unwrap().is_some()); } /// An old gap below a resumed cache's floor never hides a fresh group still owed /// above it: an ordered finite-budget reader must wait for that fresh group. #[test] #[cfg(feature = "async")] fn an_unjudgeable_gap_does_not_hide_a_fresh_ordered_gap() { let mut producer = track_producer("test", None); producer.received(); for key in [0, 4] { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); let mut reader = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(Duration::from_millis(400))) .with_range(0..), ); producer.set_live(Some(Position::group(8)), None).unwrap(); for key in [6, 8] { framed(&producer, key, 1).finish().unwrap(); } assert_eq!(read_now(&mut reader, true), None, "group 5 is still owed"); framed(&producer, 5, 1).finish().unwrap(); assert_eq!(read_now(&mut reader, true), Some(Some(5))); assert_eq!(read_now(&mut reader, true), Some(Some(6))); } /// Cache pressure can make one missing run span the old cache floor. A finite /// reader gives up only the unjudgeable prefix, keeping the fresh suffix's fill. #[test] #[cfg(feature = "async")] fn a_fill_crossing_the_cache_floor_keeps_its_fresh_suffix() { let budget = Duration::from_millis(400); for cached in [false, true] { let mut producer = track_producer("test", Info::default()); let requests = producer.requests(); producer.received(); for key in 0..3 { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); producer.start_at(6).unwrap(); producer.set_live(Some(Position::group(6)), None).unwrap(); framed(&producer, 6, 1).finish().unwrap(); if !cached { for key in 0..3 { evict(&producer, key); } } let mut reader = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_range(0..6), ); let waiter = kio::Waiter::noop(); assert_eq!(read_now(&mut reader, false), None); let Poll::Ready(Ok(fill)) = requests.poll_requested_groups(&waiter) else { panic!("cached: {cached}: missing history must be requested"); }; assert_eq!( (fill.start().group, fill.end()), (3, 6), "only fresh history is fetched" ); assert_eq!(read_now(&mut reader, false), None, "the fill is still owed"); assert_eq!(fill.wanted(), Some(5), "the fresh suffix stays wanted"); for key in 3..6 { let mut group = fill.accept(key, None).unwrap().expect("a fresh group"); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); } let mut got = Vec::new(); loop { match read_now(&mut reader, false) { Some(Some(key)) => got.push(key), Some(None) => break, None => panic!("cached: {cached}: the bounded read still owes history"), } } got.sort_unstable(); assert_eq!(got, [3, 4, 5], "cached: {cached}"); } } /// A budget narrowed while a fill is outstanding raises the cache floor through the /// fill's wanted run. Only the unjudgeable prefix is given up; the fresh suffix stays /// wanted and is delivered once the fill answers. #[test] #[cfg(feature = "async")] fn a_floor_raised_under_an_asked_fill_keeps_its_fresh_suffix() { let mut producer = track_producer("test", Info::default().with_max_age(None)); let requests = producer.requests(); producer.received(); for key in 0..3 { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); producer.start_at(6).unwrap(); producer.set_live(Some(Position::group(6)), None).unwrap(); framed(&producer, 6, 1).finish().unwrap(); for key in 0..3 { evict(&producer, key); } let unlimited = Subscription::default() .with_delay(Delay::default().with_max(None)) .with_range(0..6); let mut reader = producer.subscribe(unlimited.clone()); let waiter = kio::Waiter::noop(); assert_eq!(read_now(&mut reader, true), None); let Poll::Ready(Ok(fill)) = requests.poll_requested_groups(&waiter) else { panic!("missing history must be requested"); }; assert_eq!( (fill.start().group, fill.end()), (0, 6), "unlimited history is fetched whole" ); assert!(reader.poll_drop(&waiter).is_pending()); reader .update(unlimited.with_delay(Delay::default().with_max(Duration::from_millis(400)))) .unwrap(); assert_eq!(read_now(&mut reader, true), None, "the fresh suffix is still owed"); assert_eq!( reader.poll_drop(&waiter), Poll::Ready((0..3, Hole::drop(StreamError::NotFound))), "the unjudgeable prefix is given up" ); assert!(fill.wanted().is_some(), "the fresh suffix stays wanted"); for key in 3..6 { let mut group = fill.accept(key, None).unwrap().expect("a fresh group"); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); } for key in 3..6 { assert_eq!(read_now(&mut reader, true), Some(Some(key))); } assert_eq!(read_now(&mut reader, true), Some(None)); } /// A finite budget skips the cache below the floor at once, so a copy handing over /// waits on none of it; an unlimited read still owes it. #[test] #[cfg(feature = "async")] fn the_cache_below_the_floor_is_unread_only_to_unlimited_reads() { let mut producer = track_producer("test", Info::default().with_max_age(None)); producer.received(); for key in 0..3 { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); producer.set_live(Some(Position::group(6)), None).unwrap(); framed(&producer, 6, 1).finish().unwrap(); let delivered = Delivered::default(); let unread = Unread { start: 0, end: Some(6), delivered: &delivered, }; for (max, owed) in [(Some(Duration::from_millis(400)), false), (None, true)] { let reader = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(max)) .with_range(0..), ); assert_eq!(reader.has_unread_group(&unread), owed, "max delay {max:?}"); } } /// A finite reader selects a group below the floor, then judges it with the lock /// released in between. An eviction in that window must not turn the skip into a delivery. #[test] #[cfg(feature = "async")] fn an_evicted_group_below_the_floor_stays_stale() { let mut producer = track_producer("test", Info::default()); producer.received(); framed(&producer, 0, 1).finish().unwrap(); producer.set_idle(); producer.set_live(Some(Position::group(6)), None).unwrap(); framed(&producer, 6, 1).finish().unwrap(); evict(&producer, 0); let state = producer.state.read(); let edge = state.drift_edge(&Servable::ALL); assert!(state.is_stale(0, &edge, Duration::from_millis(400))); assert!( !state.is_stale(0, &edge, Duration::MAX), "unlimited history still owes it" ); } /// An idle cache's old groups and the holes after them stay owed by unlimited reads. /// A refused run reports every missing key and releases a bounded reader, while /// an open reader keeps receiving the live feed. Both cursor orders use the same rule. #[test] #[cfg(feature = "async")] fn a_resumed_history_read_accounts_for_cached_and_refused_groups() { for ordered in [false, true] { for bounded in [false, true] { let mut producer = track_producer("test", Info::default().with_max_age(None)); let requests = producer.requests(); producer.received(); framed(&producer, 0, 1).finish().unwrap(); producer.set_idle(); let subscription = Subscription::default() .with_delay(Delay::default().with_max(None)) .with_range(( Bound::Included(0), if bounded { Bound::Excluded(3) } else { Bound::Unbounded }, )); let mut reader = producer.subscribe(subscription); let waiter = kio::Waiter::noop(); assert!(reader.poll_drop(&waiter).is_pending()); assert_eq!(read_now(&mut reader, ordered), None, "the copy is idle"); producer.start_at(3).unwrap(); producer.set_live(Some(Position::group(3)), None).unwrap(); framed(&producer, 3, 1).finish().unwrap(); assert_eq!(read_now(&mut reader, ordered), Some(Some(0)), "cached history"); // An arrival-order reader may take the live group before its history. if !bounded && !ordered { assert_eq!(read_now(&mut reader, ordered), Some(Some(3))); } assert_eq!(read_now(&mut reader, ordered), None, "the missing run is owed"); let Poll::Ready(Ok(fill)) = requests.poll_requested_groups(&waiter) else { panic!("the missing history was never requested"); }; assert_eq!((fill.start(), fill.end()), (Position::group(1), 3)); fill.refuse_groups(1..3, Error::Unsupported); assert!( reader.poll_drop(&waiter).is_pending(), "a late live arrival may still answer" ); elapse(&producer, ledger::GRACE); assert_eq!( reader.poll_drop(&waiter), Poll::Ready((1..3, Hole::drop(StreamError::NotFound))) ); assert!(reader.poll_drop(&waiter).is_pending(), "each drop is reported once"); if bounded { assert_eq!(read_now(&mut reader, ordered), Some(None), "nothing remains owed"); } else { if ordered { assert_eq!(read_now(&mut reader, ordered), Some(Some(3))); } framed(&producer, 4, 1).finish().unwrap(); assert_eq!(read_now(&mut reader, ordered), Some(Some(4)), "the feed survives"); } } } } /// The cancel that idles a copy resets the group in flight, which the publisher never /// gave up: no reader hears it dropped, and the next feed sends it again. #[test] #[cfg(feature = "async")] fn a_reset_group_is_not_dropped_and_comes_again() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); producer.set_idle(); open.abort(Error::Cancel).unwrap(); let mut subscriber = producer.subscribe(replay()); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_drop(&waiter).is_pending(), "nothing was given up"); let mut again = producer.create_group(group::Info { start: 1 }).unwrap(); producer.set_live(Some(Position::group(1)), None).unwrap(); again .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert_eq!(drain(&mut subscriber), vec![0, 1]); assert!(subscriber.poll_drop(&waiter).is_pending(), "nothing was given up"); } #[test] #[cfg(feature = "async")] fn real_time_skips_a_backlog_to_the_live_edge() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } // The default budget is REAL_TIME: a subscriber joining a track that already // holds five seconds of history takes the live edge, not the history. This is // the ceiling a takeover backlog runs into. let mut subscriber = producer.subscribe(None); assert_eq!(drain(&mut subscriber), vec![4]); // And it stays caught up: the next group is live when it lands. append_at(&mut producer, 5000); assert_eq!(drain(&mut subscriber), vec![5]); } #[test] #[cfg(feature = "async")] fn real_time_skips_a_backlog_after_catching_up() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); let mut subscriber = producer.subscribe(None); assert_eq!(drain(&mut subscriber), vec![0]); // Catch-up is a subscriber state, not a startup-only choice. A reader can pause // between groups and still needs to shed the backlog that accumulated meanwhile. for second in 1..6 { append_at(&mut producer, second * 1000); } assert_eq!(drain(&mut subscriber), vec![5]); } #[test] #[cfg(feature = "async")] fn a_newer_edge_changes_an_active_catch_up() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); assert_eq!( subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 3 ); // Group 4 was the edge at the first delivery. Advancing twice puts it outside // the budget before the subscriber asks for another group. append_at(&mut producer, 5000); append_at(&mut producer, 10000); assert_eq!(drain(&mut subscriber), vec![5, 6]); } #[test] #[cfg(feature = "async")] fn a_growing_edge_changes_an_active_catch_up() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); append_at(&mut producer, 1000); let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"a")) .unwrap(); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(2)))); assert_eq!( subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0 ); // The edge is still the same group, but its newest presentation moved far // enough that group 1 has fallen out of range. edge.write_frame(Timestamp::from_millis(5000).unwrap(), bytes::Bytes::from_static(b"b")) .unwrap(); assert_eq!(drain(&mut subscriber), vec![2]); } #[test] #[cfg(feature = "async")] fn a_budget_admits_groups_within_it() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } // Two seconds of tolerance keeps the groups presenting within 2s of the live // edge (2s, 3s, 4s) and drops the two below it. Group 1 reaches exactly 2s behind // the edge, and the reach bound is exclusive, so every frame it could hold is // already past the budget. let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(2)))); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } /// A fetch with a budget parks on its deadlines even when it returns a cached group at /// once, and fetches with distinct budgets never share one. Fetches that finish must not /// leave their deadlines indexed for as long as the track lives. #[test] #[cfg(feature = "async")] fn departed_budgets_leave_no_deadlines_behind() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } let consumer = producer.consume(); for i in 0..1_000 { let budget = Duration::from_millis(1_500) + Duration::from_micros(i); let fetch = group::Fetch::default().with_max_delay(budget); let group = consumer.fetch_group(2, fetch).now_or_never().unwrap().unwrap(); assert_eq!(group.start, 2); } let cache = producer.state.read().cache.clone(); let (media, wall) = cache.wakes().deadlines_indexed(); assert!( media <= 32 && wall <= 32, "{media} media and {wall} wall deadlines indexed" ); } #[test] #[cfg(feature = "async")] fn a_budget_reaches_back_over_the_cache() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } // The default zero budget calls every non-latest group stale, so a subscription // that says nothing joins at the live edge. let mut live = producer.subscribe(None); assert_eq!(drain(&mut live), vec![4]); // Two seconds of tolerance covers the groups presenting within 2s of the live // edge, so the same join is handed the head of what it can still use. One bound // decides both what is sent and what is expired, so a subscriber is never sent // history it would discard on arrival. let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(2))); let mut subscriber = producer.subscribe(budget); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } #[test] #[cfg(feature = "async")] fn a_named_start_is_a_floor_not_a_request() { let mut producer = track_producer("test", Info::timed()); for second in 0..5 { append_at(&mut producer, second * 1000); } // The budget is the only thing that asks for data; a named start only bounds how // far back it may reach. Naming group 1 at real time still delivers the live edge // alone, since the zero budget calls everything older stale. let named = Subscription::default().with_start(1); let mut subscriber = producer.subscribe(named); assert_eq!(drain(&mut subscriber), vec![4]); // A budget reaching further back than the floor is cut off at it. let floored = Subscription::default() .with_start(3) .with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut subscriber = producer.subscribe(floored); assert_eq!(drain(&mut subscriber), vec![3, 4]); // A floor below what the budget admits changes nothing. let slack = Subscription::default() .with_start(1) .with_delay(Delay::default().with_max(Duration::from_secs(2))); let mut subscriber = producer.subscribe(slack); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } /// A start past the live edge is a time, which the newest group contains until a later /// one starts: the cursor reads that group, then each one after it. #[test] #[cfg(feature = "async")] fn a_start_past_the_live_edge_reads_the_group_containing_it() { let producer = track_producer("test", Info::timed()); for key in [0, 1000, 2000] { write_group(&producer, key); } let resumed = Subscription::default() .with_start(7000) .with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut subscriber = producer.subscribe(resumed); assert_eq!(drain(&mut subscriber), vec![2000]); for key in [3000, 4000] { write_group(&producer, key); } assert_eq!(drain(&mut subscriber), vec![3000, 4000]); } /// A lite-07 copy holds only part of the track, so it places a time start only once it /// knows which group contains it: one starting there, one before it once the keys between /// are skipped or a later group arrived, or where its publisher reported starting the /// range asked from that time. Meanwhile the reader reads the groups from the time. #[test] #[cfg(feature = "async")] fn a_received_copy_places_a_time_start_once_it_knows() { let from = |time| { Subscription::default() .with_ranges([Range { start: Anchor::Time(time), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let placed = |subscriber: &mut Subscriber| { subscriber .resolved_range(0) .map(|resolved| (resolved.start, resolved.floored)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 1000); // A group between 1000 and the time may still arrive. let mut later = producer.subscribe(from(1500)); let mut answered = producer.subscribe(from(1700)); let mut skipped = producer.subscribe(from(1800)); let mut before = producer.subscribe(from(500)); let mut at = producer.subscribe(from(1000)); assert_eq!(placed(&mut at), Some((1000, true)), "a group starts there"); for subscriber in [&mut later, &mut answered, &mut skipped, &mut before] { assert_eq!(placed(subscriber), None); } assert_eq!(drain(&mut before), vec![1000], "reads from the time meanwhile"); assert_eq!(drain(&mut later), Vec::::new()); // The publisher started the range asked from 1700 at 1000, and the one from 500 at 1000 // too, past it: nothing at or before 500 is served. producer.time_starts([(1700, 1000), (500, 1000)]); assert_eq!(placed(&mut answered), Some((1000, true))); assert_eq!(placed(&mut before), Some((0, false))); assert_eq!(placed(&mut later), None); // Skipped keys between say nothing arrives there. producer.skip_groups(1001..1801); assert_eq!(placed(&mut skipped), Some((1000, true))); assert_eq!(placed(&mut later), Some((1000, true))); // A group past the time says the feed went by it. let mut past = producer.subscribe(from(2500)); assert_eq!(placed(&mut past), None); write_group(&producer, 3000); assert_eq!(placed(&mut past), Some((1000, true))); assert_eq!(drain(&mut past), vec![1000, 3000]); } /// A reader joining a lite-07 copy that another reader already asks from further back adds /// no range of its own, so its publisher reports no start for the reader's time. The start /// of a range asked from earlier, or from the oldest group, places it: past the time, no /// group starts at or before it, and a publisher serving the reader learns where delivery /// starts. At or before the time, a group between may still arrive. #[test] #[cfg(feature = "async")] fn a_received_copy_places_a_time_start_by_an_earlier_range() { let from = |time| { Subscription::default() .with_ranges([Range { start: Anchor::Time(time), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let placed = |subscriber: &mut Subscriber| { subscriber.resolved_ranges([0])[0].map(|resolved| (resolved.start, resolved.floored)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 1000); let mut before = producer.subscribe(from(500)); let mut after = producer.subscribe(from(1500)); assert_eq!(placed(&mut before), None); assert!(before.poll_start(&kio::Waiter::noop()).is_pending()); // Only a range asked from a later time: says nothing of 500. producer.time_starts([(700, 1000)]); assert_eq!(placed(&mut before), None); // The range asked from the oldest group started at 1000. producer.time_starts([(0, 1000)]); assert_eq!(placed(&mut before), Some((0, false))); assert!(before.poll_start(&kio::Waiter::noop()).is_ready()); assert_eq!(drain(&mut before), vec![1000]); assert_eq!(placed(&mut after), None, "a group past 1000 may still arrive"); } /// A group the copy still holds at or before a time start outranks a start reported for an /// earlier range: the publisher expiring it says nothing of whether it contains the time. #[test] #[cfg(feature = "async")] fn a_received_copy_keeps_a_held_group_over_an_earlier_range() { let from = |time| { Subscription::default() .with_ranges([Range { start: Anchor::Time(time), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let placed = |subscriber: &mut Subscriber| { subscriber.resolved_ranges([0])[0].map(|resolved| (resolved.start, resolved.floored)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 2000); let mut reader = producer.subscribe(from(2500)); producer.time_starts([(0, 3000)]); assert_eq!(placed(&mut reader), None, "the held 2000 may contain 2500"); write_group(&producer, 3000); assert_eq!(placed(&mut reader), Some((2000, true))); assert_eq!(drain(&mut reader), vec![2000, 3000]); } /// Once the copy evicts the group it held before a time start, the start of a range asked /// from earlier places the time: no reported start changed, but the waiting reader still /// starts. #[test] #[cfg(feature = "async")] fn a_received_copy_places_a_time_start_once_the_group_before_it_is_evicted() { let from = |time| { Subscription::default() .with_ranges([Range { start: Anchor::Time(time), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let placed = |subscriber: &mut Subscriber| { subscriber.resolved_ranges([0])[0].map(|resolved| (resolved.start, resolved.floored)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 2000); let mut reader = producer.subscribe(from(2500)); producer.time_starts([(0, 3000)]); assert_eq!(placed(&mut reader), None, "the held 2000 may contain 2500"); producer.evict(2000); write_group(&producer, 3000); assert_eq!(placed(&mut reader), Some((0, false))); assert_eq!(drain(&mut reader), vec![3000]); } /// A floor at or past a time start, as a pre-lite-07 wire's start sets, keeps out the group /// containing that time unless it starts there, so where the range starts waits on nothing /// a lite-07 copy has yet to place. Without the floor, it waits. #[test] #[cfg(feature = "async")] fn a_floor_at_a_time_start_waits_on_no_placing() { let from = |time| { Subscription::default() .with_ranges([Range { start: Anchor::Time(time), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); write_group(&producer, 1000); let mut open = producer.subscribe(from(500)); assert!(open.poll_start(&kio::Waiter::noop()).is_pending()); let mut floored = producer.subscribe(from(500)).with_range(500..); assert!(floored.poll_start(&kio::Waiter::noop()).is_ready()); let mut below = producer.subscribe(from(500)).with_range(499..); assert!(below.poll_start(&kio::Waiter::noop()).is_pending()); assert_eq!(drain(&mut floored), vec![1000]); } /// A lite-07 copy holds only part of the track, so a relative start counts back as its /// publisher reported, not over the groups the copy holds, until a group past the latest /// the publisher counted from moves the live edge. #[test] #[cfg(feature = "async")] fn a_received_copy_counts_a_relative_start_back_as_its_publisher_did() { let back = |back| { Subscription::default() .with_ranges([Range { start: Anchor::Relative(back), end: None, }]) .with_delay(Delay::default().with_max(None)) }; let placed = |subscriber: &mut Subscriber| subscriber.resolved_range(0).map(|resolved| resolved.start); let producer = track_producer("test", Info::timed()); producer.received(); producer.anchored(); for key in [1, 2, 9] { write_group(&producer, key); } producer.report_edge(9); producer.relative_starts([(2, 7, 9)]); let mut reported = producer.subscribe(back(2)); assert_eq!(placed(&mut reported), Some(7), "the publisher's start, not group 1"); let mut unreported = producer.subscribe(back(1)); assert_eq!(placed(&mut unreported), Some(2), "counted over the groups held"); // A reported start is where delivery began, which may sit past the group counted, so a // relative end counts over the groups held. let mut ended = producer.subscribe( Subscription::default() .with_ranges([Range { start: Anchor::Time(0), end: Some(Anchor::Relative(2)), }]) .with_delay(Delay::default().with_max(None)), ); assert_eq!(ended.resolved_range(0).and_then(|resolved| resolved.end), Some(2)); // A newer group moved the live edge, so a later reader counts back from it. write_group(&producer, 10); let mut later = producer.subscribe(back(2)); assert_eq!(placed(&mut later), Some(2)); assert_eq!( placed(&mut reported), Some(7), "an earlier reader stays where it started" ); } #[test] #[cfg(feature = "async")] fn a_late_lower_group_within_the_budget_is_delivered() { let producer = track_producer("test", Info::timed()); for (key, millis) in [(5, 1000), (6, 2000), (7, 3000)] { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); group.finish().unwrap(); } let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(5)))); assert_eq!(drain(&mut subscriber), vec![5, 6, 7]); // Arriving below everything already delivered is not what makes content stale: // the budget is the only gate, and this straggler's timestamp is within it. A // consumer that needs key order reorders (or drops) it itself. let mut late = producer.create_group(group::Info { start: 4 }).unwrap(); late.write_frame(Timestamp::from_millis(500).unwrap(), bytes::Bytes::from_static(b"late")) .unwrap(); late.finish().unwrap(); assert_eq!(drain(&mut subscriber), vec![4]); } #[test] #[cfg(feature = "async")] fn drift_is_measured_in_presentation_time_not_arrival_time() { let mut producer = track_producer("test", Info::timed()); // A relay ingesting a backlog creates every group at once, so arrival time says // they are all equally fresh. Their timestamps say otherwise, which is the whole // point of measuring in presentation time. for second in 0..4 { append_at(&mut producer, second * 1000); } let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500)))); assert_eq!(drain(&mut subscriber), vec![1, 2, 3]); } #[test] #[cfg(feature = "async")] fn a_stamped_successor_expires_an_unstamped_group() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let _open = producer.append_next().unwrap(); // seq 0 stalls before its first frame append_at(&mut producer, 1000); // seq 1 proves the live feed moved on // The candidate needs no timestamp of its own: its reach is where its stamped // successor begins, which the zero budget already puts out of range. assert_eq!(drain(&mut subscriber), vec![1]); } #[kio::test] #[cfg(feature = "async")] async fn a_handed_out_group_expires_while_its_first_frame_is_stalled() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let _writer_1 = producer.append_next().unwrap(); let mut stalled = subscriber.recv_group().await.unwrap().expect("stalled group"); let pending = kio::clock::sim::spawn(async move { stalled.read_frame().await }); kio::clock::sim::yield_now().await; assert!( !pending.is_finished(), "the empty live edge still waits for its first frame" ); elapse(&producer, Duration::from_secs(1)); append_at(&mut producer, 1000); // The producer never finished it, so even an empty prefix is not the group. let result = pending.await.unwrap(); assert!(matches!(result, Err(Error::Old)), "the held group fails: {result:?}"); } /// A first timestamp on a *newer* group can convict a held one, so the held reader has /// to be woken by it. The conviction needs a group beyond the held one's successor: /// a group is bounded by where its successor begins, so the successor itself never /// proves it stale. #[kio::test] #[cfg(feature = "async")] async fn a_handed_out_group_wakes_when_a_newer_group_gets_its_first_timestamp() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let mut old = producer.append_next().unwrap(); old.write_frame(old.key_time(), bytes::Bytes::from_static(b"old")) .unwrap(); // Group 1 bounds group 0's reach at 1s. append_at(&mut producer, 1000); let mut held = subscriber.recv_group().await.unwrap().expect("old group"); assert!(held.read_frame().await.unwrap().is_some()); let mut live = producer.append_next().unwrap(); let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); kio::clock::sim::yield_now().await; assert!(!pending.is_finished(), "the newer group has no timestamp yet"); // 2s edge against group 0's 1s reach: a full second past the budget. live.write_frame( Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"live"), ) .unwrap(); kio::clock::sim::yield_now().await; assert!(pending.is_finished(), "the new presentation edge wakes the held reader"); // Drained but never finished, so the budget fails the group. let result = pending.await.unwrap(); assert!(matches!(result, Err(Error::Old)), "the held group fails: {result:?}"); } /// An appended group wakes only the parked reads whose expiry it crosses, plus the /// newest, whose successor it is. A 2s audio backlog at 400 groups/s parks ~800 reads, /// and waking every one per append pins a publisher's runtime. #[test] fn an_append_wakes_only_the_parked_reads_it_expires() { struct Count(std::sync::atomic::AtomicUsize); impl std::task::Wake for Count { fn wake(self: Arc) { self.wake_by_ref(); } fn wake_by_ref(self: &Arc) { self.0.fetch_add(1, Ordering::SeqCst); } } const FRAME: u64 = 2500; // micros, one Opus frame per group const PARKED: u64 = 64; let producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(200)))); // Left open, so every read parks instead of ending. let _open: Vec<_> = (0..PARKED) .map(|i| { let mut group = producer.append_next().unwrap(); group .write_frame( Timestamp::from_micros(i * FRAME).unwrap(), bytes::Bytes::from_static(b"x"), ) .unwrap(); group }) .collect(); let counts: Vec<_> = (0..PARKED) .map(|_| Arc::new(Count(std::sync::atomic::AtomicUsize::new(0)))) .collect(); let waiters: Vec<_> = counts .iter() .map(|count| kio::Waiter::new(std::task::Waker::from(count.clone()))) .collect(); let mut held: Vec<_> = waiters .iter() .map(|waiter| { let Poll::Ready(Ok(Some(mut group))) = subscriber.poll_recv_group(waiter) else { panic!("every group is cached"); }; assert!(matches!(group.poll_read_frame(waiter), Poll::Ready(Ok(Some(_))))); group }) .collect(); for (group, waiter) in held.iter_mut().zip(&waiters) { assert!(group.poll_read_frame(waiter).is_pending(), "nothing is stale yet"); } for count in &counts { count.0.store(0, Ordering::SeqCst); } // Read `i` can reach its successor's start, `(i + 1) * FRAME`, so an edge at 210ms // puts reads 0..=3 a full 200ms behind and leaves every later one in budget. let mut next = producer.append_next().unwrap(); next.write_frame( Timestamp::from_micros(210_000).unwrap(), bytes::Bytes::from_static(b"x"), ) .unwrap(); let woken: Vec<_> = (0..PARKED) .filter(|&i| counts[i as usize].0.load(Ordering::SeqCst) > 0) .collect(); assert_eq!( woken, vec![0, 1, 2, 3, PARKED - 1], "only the expired reads and the newest wake" ); for (i, (group, waiter)) in held.iter_mut().zip(&waiters).enumerate() { let result = group.poll_read_frame(waiter); if i < 4 { assert!( matches!(result, Poll::Ready(Err(Error::Old))), "read {i} expired: {result:?}" ); } else { assert!(result.is_pending(), "read {i} is in budget: {result:?}"); } } } /// A parked read wakes only once the edge reaches its deadline: its successor's start /// plus its budget. A first frame below the edge moves nothing, and a new edge short of /// the deadline cannot convict it, so neither wakes it. #[test] #[cfg(feature = "async")] fn a_parked_read_wakes_only_once_the_edge_reaches_its_deadline() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); let mut head = producer.append_next().unwrap(); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach let mut between = producer.append_next().unwrap(); append_at(&mut producer, 3000); // the edge let mut held = subscriber .recv_group() .now_or_never() .unwrap() .unwrap() .expect("head group"); assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); between .write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!( !woken.load(Ordering::SeqCst), "a first frame below the edge moves neither the reach nor the edge" ); let mut beyond = producer.append_next().unwrap(); beyond .write_frame(Timestamp::from_millis(4000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!(!woken.load(Ordering::SeqCst), "a new edge short of the deadline"); // The edge's own later frame reaches the deadline. beyond .write_frame(Timestamp::from_millis(11_000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!( woken.load(Ordering::SeqCst), "the edge reaching the deadline wakes the read" ); let result = next.as_mut().poll(&mut cx); assert!( matches!(result, Poll::Ready(Err(Error::Old))), "the head is stale: {result:?}" ); } /// A batch must notify expiry at its highest PTS even when its decode-order tail is lower. #[test] #[cfg(feature = "async")] fn a_reordered_batch_wakes_the_group_pts_drift_deadline() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); let mut head = producer.append_next().unwrap(); head.write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::new()) .unwrap(); append_at(&mut producer, 1000); let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::new()) .unwrap(); let mut held = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); let mut frames = crate::frame::Buffer::<2>::new(); for ms in [11_000, 10_000] { frames .push(crate::frame::Frame::new( Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new(), )) .unwrap(); } edge.write_frames(&mut frames).unwrap(); assert!( woken.load(Ordering::SeqCst), "the batch's highest PTS reaches the deadline" ); assert!(matches!(next.as_mut().poll(&mut cx), Poll::Ready(Err(Error::Old)))); head.finish().unwrap(); edge.finish().unwrap(); } #[test] #[cfg(feature = "async")] fn an_adopted_high_pts_wakes_the_group_drift_deadline() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); let mut old = producer.append_next().unwrap(); old.write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::new()) .unwrap(); append_at(&mut producer, 1000); let mut edge = producer.append_next().unwrap(); edge.start_at(2).unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::new()) .unwrap(); let mut held = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); let mut prefix = group::Info { start: 2 }.produce(); for ms in [1500, 11_000] { prefix .write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) .unwrap(); } let head = prefix.consume().handoff().head().unwrap(); edge.handle().adopt_head(&head); assert_eq!(edge.handle().latest(), Some(Timestamp::from_millis(11_000).unwrap())); assert!( woken.load(Ordering::SeqCst), "the adopted prefix reaches the drift deadline" ); assert!(matches!(next.as_mut().poll(&mut cx), Poll::Ready(Err(Error::Old)))); old.finish().unwrap(); edge.finish().unwrap(); prefix.finish().unwrap(); } /// An aborted edge hands the edge to a group between it and the successor, and neither /// the abort nor that group's frames touch the track, so the read must still wake. #[test] #[cfg(feature = "async")] fn a_parked_read_watches_its_edge_abort() { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(5)))); let mut head = producer.append_next().unwrap(); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach let mut between = producer.append_next().unwrap(); let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); let edge = edge.finish().unwrap(); let mut held = subscriber .recv_group() .now_or_never() .unwrap() .unwrap() .expect("head group"); assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); edge.abort(Error::Cancel).unwrap(); // Starts between its neighbors, then runs past the deadline. for millis in [1500, 10_000] { between .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } assert!( woken.load(Ordering::SeqCst), "the edge's abort, then a new edge below it, lost the wakeup" ); let result = next.as_mut().poll(&mut cx); assert!( matches!(result, Poll::Ready(Err(Error::Old))), "the head is stale: {result:?}" ); } /// An aborted successor hands the reach to the next group, which sits below the edge /// and so is watched only because the read re-selects its successor. #[test] #[cfg(feature = "async")] fn a_parked_read_watches_the_replacement_for_an_aborted_successor() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let mut head = producer.append_next().unwrap(); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); let successor = producer.append_next().unwrap(); let mut replacement = producer.append_next().unwrap(); append_at(&mut producer, 20_000); // the edge let mut held = subscriber .recv_group() .now_or_never() .unwrap() .unwrap() .expect("head group"); assert_eq!(held.start, 0, "an unstamped successor leaves the reach unbounded"); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); successor.abort(Error::Cancel).unwrap(); assert!( woken.load(Ordering::SeqCst), "the successor's abort wakes the parked read" ); woken.store(false, Ordering::SeqCst); assert!( next.as_mut().poll(&mut cx).is_pending(), "the unstamped replacement leaves the reach unbounded" ); replacement .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!( woken.load(Ordering::SeqCst), "the replacement's first frame bounds the reach" ); let result = next.as_mut().poll(&mut cx); assert!( matches!(result, Poll::Ready(Err(Error::Old))), "the head is stale: {result:?}" ); } /// An edge's abort hands the edge back to a lower group whose frames already ran past /// the deadline. No frame write crosses it afterwards, and the abort never touches the /// track, so only the read watching the edge's abort sees the head go stale. #[test] #[cfg(feature = "async")] fn a_parked_read_watches_its_overtaken_edge_abort() { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(5)))); let mut head = producer.append_next().unwrap(); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach // Starts below the edge but presents past the deadline. let mut long = producer.append_next().unwrap(); for millis in [1500, 10_000] { long.write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } long.finish().unwrap(); let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); let edge = edge.finish().unwrap(); let mut held = subscriber .recv_group() .now_or_never() .unwrap() .unwrap() .expect("head group"); assert_eq!(held.start, 0, "the edge keeps the head within budget"); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); edge.abort(Error::Cancel).unwrap(); assert!(woken.load(Ordering::SeqCst), "the edge's abort lost its wakeup"); let result = next.as_mut().poll(&mut cx); assert!( matches!(result, Poll::Ready(Err(Error::Old))), "the head is stale: {result:?}" ); } /// A group's header lands before its first frame, which moves no edge. The first frame /// has to wake a parked read it expires, though the read sits below the group's nearest /// started predecessor. #[test] #[cfg(feature = "async")] fn revealing_a_presented_group_wakes_a_read_it_expires() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let mut head = producer.append_next().unwrap(); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach let mut held = subscriber .recv_group() .now_or_never() .unwrap() .unwrap() .expect("head group"); assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); let mut received = producer.create_group(group::Info { start: 2 }).unwrap(); woken.store(false, Ordering::SeqCst); assert!( next.as_mut().poll(&mut cx).is_pending(), "the frameless group moves no edge" ); received .write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!(woken.load(Ordering::SeqCst), "the first frame wakes the expired read"); let result = next.as_mut().poll(&mut cx); assert!( matches!(result, Poll::Ready(Err(Error::Old))), "the head is stale: {result:?}" ); } /// A route can declare the end, then land every group's header before its first frame. /// Readers reach those groups at once, and the end with them, but nothing counts a /// frameless group as content: the track shows nothing past the first until it starts. #[test] #[cfg(feature = "async")] fn a_frameless_group_does_not_count_as_content() { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60)))); producer.finish_at(Position::group(2)).unwrap(); let mut received: Vec<_> = (0..2) .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let mut first = subscriber.assert_group(); let mut second = subscriber.assert_group(); assert_eq!((first.start, second.start), (0, 1)); assert!(subscriber.recv_group().now_or_never().unwrap().unwrap().is_none()); let waiter = kio::Waiter::noop(); assert!( producer.poll_past(0, &waiter).is_pending(), "a frameless group is not content" ); for group in &mut received { group .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); } assert!( producer.poll_past(0, &waiter).is_ready(), "its first frame makes it content" ); assert!(first.read_frame().now_or_never().unwrap().unwrap().is_some()); assert!(second.read_frame().now_or_never().unwrap().unwrap().is_some()); } /// A parked `poll_past` wakes when a later group starts, not when its header lands. #[test] fn poll_past_wakes_when_a_later_group_starts() { use std::sync::atomic::{AtomicBool, Ordering}; let producer = track_producer("test", Info::timed()); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_past(0, &waiter).is_pending()); let mut group = producer.create_group(group::Info { start: 1 }).unwrap(); woken.store(false, Ordering::SeqCst); assert!( producer.poll_past(0, &waiter).is_pending(), "a frameless header is not content" ); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!(woken.load(Ordering::SeqCst), "its first frame wakes the parked poll"); assert!(producer.poll_past(0, &waiter).is_ready()); } /// A frameless group whose stream ends with no frame, cleanly or reset, ends its readers /// with it, and the track's end follows. #[kio::test] #[cfg(feature = "async")] async fn a_frameless_group_that_ends_empty_ends_its_readers() { for abort in [false, true] { let mut producer = track_producer("test", None); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60)))); producer.finish_at(Position::group(1)).unwrap(); let group = producer.create_group(group::Info { start: 0 }).unwrap(); let mut reading = subscriber.assert_group(); assert_eq!(reading.start, 0); let ended = { let mut next = std::pin::pin!(reading.read_frame()); assert!(futures::poll!(next.as_mut()).is_pending(), "the group is still open"); match abort { true => group.abort(Error::Cancel).unwrap(), false => { group.finish().unwrap(); } } kio::clock::timeout(Duration::from_secs(1), next) .await .expect("the end wakes the reader") }; match abort { true => assert!(matches!(ended, Err(Error::Cancel)), "{ended:?}"), false => assert!(matches!(ended, Ok(None)), "{ended:?}"), } assert!(subscriber.recv_group().now_or_never().unwrap().unwrap().is_none()); } } /// The ordinary live case, at the default real-time budget: 2s GOPs produced one at /// a time and read as they arrive. The budget must take the live edge without /// shortening the group the reader is already on, so every frame of every group is /// delivered. /// /// The reader is parked at its group's end when the successor opens, because the FIN /// and the next group's first frame are separate events and the wire does not order /// them. A zero budget gives up there, and the group fails with [`Error::Old`] since /// its producer never finished it, but only after the reader took every frame. #[kio::test] #[cfg(feature = "async")] async fn real_time_reads_a_live_stream_without_truncating_it() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let gop = |n: u64| { [ Timestamp::from_millis(n * 2000).unwrap(), Timestamp::from_millis(n * 2000 + 1900).unwrap(), ] }; let write = |group: &mut group::Producer, timestamp| { group.write_frame(timestamp, bytes::Bytes::from_static(b"x")).unwrap(); }; let mut open = producer.append_next().unwrap(); write(&mut open, gop(0)[0]); write(&mut open, gop(0)[1]); let mut reading = subscriber.recv_group().await.unwrap().expect("the live group"); let mut read = Vec::new(); for n in 1..5u64 { let key = reading.start; let mut frames = 0; while let Some(res) = reading.read_frame().now_or_never() { match res.expect("no truncation while draining") { Some(_) => frames += 1, None => panic!("group {key} ended early"), } } read.push((key, frames)); let next = { // Parked at the end of the current group: every frame is read and no FIN // has landed. let mut end = std::pin::pin!(reading.read_frame()); assert!(futures::poll!(end.as_mut()).is_pending(), "parked on the FIN"); // The next keyframe opens its group. The verdict is taken here, in the // window before the previous group's FIN arrives. let mut opened = producer.append_next().unwrap(); write(&mut opened, gop(n)[0]); let verdict = futures::poll!(end.as_mut()); open.finish().unwrap(); let res = match verdict { Poll::Ready(res) => res, Poll::Pending => end.await, }; assert!( matches!(res, Err(Error::Old)), "group {key} gives up at the boundary, after its last frame: {res:?}" ); opened }; let mut next = next; write(&mut next, gop(n)[1]); reading = subscriber.recv_group().await.unwrap().expect("the next live group"); open = next; } assert_eq!(read, vec![(0, 2), (1, 2), (2, 2), (3, 2)], "every frame of every group"); } /// Decode-order B-frames cannot rewind the presentation edge and re-admit an old group. #[kio::test] #[cfg(feature = "async")] async fn a_reordered_tail_keeps_the_group_pts_drift_verdict() { for reordered in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(80)))); for ms in [0, 100] { let mut group = producer.append_next().unwrap(); group .write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) .unwrap(); group.finish().unwrap(); } let mut edge = producer.append_next().unwrap(); for ms in [150, 200] { edge.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) .unwrap(); } if reordered { edge.write_frame(Timestamp::from_millis(160).unwrap(), bytes::Bytes::new()) .unwrap(); } edge.finish().unwrap(); // Group 0 can reach 100ms: 100ms behind the 200ms edge, past its 80ms budget. // Group 1 can reach 150ms and stays within the same budget. assert_eq!( subscriber.recv_group().await.unwrap().unwrap().start, 1, "reordered {reordered}" ); } } /// A budget is spent from where the reader stands, not from where its group opened. /// /// A 2s GOP with a 1s budget: the reader has drained to 1900ms when the next group /// opens at 2000ms, so it is 100ms behind the live edge and well inside what it /// asked for. A straggling frame of the old group arriving after the new one opened /// (which is ordinary, the two are separate streams) must still reach it. /// /// Measuring from the group's first frame instead makes the drift 2000ms, so a /// budget shorter than one GOP would drop the tail of every GOP. #[kio::test] #[cfg(feature = "async")] async fn a_budget_is_measured_from_the_readers_position() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); let mut open = producer.append_next().unwrap(); open.write_frame(open.key_time(), bytes::Bytes::from_static(b"key")) .unwrap(); open.write_frame( Timestamp::from_millis(1900).unwrap(), bytes::Bytes::from_static(b"tail"), ) .unwrap(); let mut reading = subscriber.recv_group().await.unwrap().expect("the live group"); assert!(reading.read_frame().await.unwrap().is_some()); assert!(reading.read_frame().await.unwrap().is_some()); let mut end = std::pin::pin!(reading.read_frame()); assert!(futures::poll!(end.as_mut()).is_pending(), "parked at 1900ms"); // The next GOP opens. The reader is 100ms behind it, inside its 1s budget. let mut next = producer.append_next().unwrap(); next.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"key")) .unwrap(); assert!( futures::poll!(end.as_mut()).is_pending(), "a reader inside its budget is not expired by the next group opening" ); // A straggler from the old group, still within the budget. open.write_frame( Timestamp::from_millis(1950).unwrap(), bytes::Bytes::from_static(b"late"), ) .unwrap(); let late = end.await.expect("the straggler is not truncated"); assert_eq!( late.map(|frame| frame.timestamp), Some(Timestamp::from_millis(1950).unwrap()) ); } /// A drained group the budget gives up on fails with [`Error::Old`] on every probe, /// including [`group::Consumer::finished`]: the producer never finished it. #[kio::test] #[cfg(feature = "async")] async fn an_expired_group_stays_old_when_probed_again() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let mut open = producer.append_next().unwrap(); open.write_frame(open.key_time(), bytes::Bytes::from_static(b"a")) .unwrap(); let mut group = subscriber.recv_group().await.unwrap().expect("group"); assert!(group.read_frame().await.unwrap().is_some()); let probes = kio::clock::sim::spawn(async move { let first = group.read_frame().await; let second = group.read_frame().await; let finished = group.finished().await; (first, second, finished) }); kio::clock::sim::yield_now().await; append_at(&mut producer, 1000); let (first, second, finished) = probes.await.unwrap(); assert!(matches!(first, Err(Error::Old)), "the group fails: {first:?}"); assert!(matches!(second, Err(Error::Old)), "and stays failed: {second:?}"); assert!(matches!(finished, Err(Error::Old)), "never finished: {finished:?}"); } #[kio::test] #[cfg(feature = "async")] async fn a_drained_group_finishes_cleanly_after_the_live_edge_advances() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); append_at(&mut producer, 0); let mut group = subscriber.recv_group().await.unwrap().expect("first group"); assert!(group.read_frame().await.unwrap().is_some()); append_at(&mut producer, 1000); assert!(group.read_frame().await.unwrap().is_none()); } #[kio::test] #[cfg(feature = "async")] async fn a_handed_out_partial_frame_expires_while_its_payload_is_stalled() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let source = producer.append_next().unwrap(); let mut writing = source .create_frame(frame::Info { size: 6, timestamp: Timestamp::ZERO, }) .unwrap(); writing.write(bytes::Bytes::from_static(b"old")).unwrap(); let mut group = subscriber.recv_group().await.unwrap().expect("partial group"); let mut frame = group.next_frame().await.unwrap().expect("partial frame"); assert_eq!( frame.read_chunk().await.unwrap(), Some(bytes::Bytes::from_static(b"old")) ); let pending = kio::clock::sim::spawn(async move { frame.read_chunk().await }); kio::clock::sim::yield_now().await; assert!(!pending.is_finished(), "the partial payload is still stalled"); elapse(&producer, Duration::from_secs(1)); append_at(&mut producer, 1000); let result = pending.await.unwrap(); assert!( matches!(result, Err(Error::Old)), "the in-flight frame expires: {result:?}" ); writing.abort(Error::Cancel).unwrap(); } #[test] #[cfg(feature = "async")] fn max_age_bounds_the_budget() { // The publisher only keeps a group around for 500ms, so a subscriber asking to // wait ten seconds for one still gives up at 500ms: the window the publisher // applies to the aggregate, on the subscriber's own side of it. let mut producer = track_producer("test", Info::timed().with_max_age(Duration::from_millis(500))); append_at(&mut producer, 0); append_at(&mut producer, 1000); append_at(&mut producer, 2000); // Group 0 reaches 1s, a full second behind the 2s edge, so the clamped 500ms // budget drops it. An unclamped ten seconds would have kept it. let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); assert_eq!(drain(&mut subscriber), vec![1, 2]); } /// An authored track's default zero max age serves only the live edge: a superseded group is /// stale at once, even to a reader with no budget of its own, while the newest group, a /// snapshot a late joiner bootstraps from, never goes stale. A longer history is the /// publisher's explicit choice. #[test] #[cfg(feature = "async")] fn the_default_max_age_serves_only_the_live_edge() { let history = || Subscription::default().with_delay(Delay::default().with_max(None)); // Timed: a successor at the same instant already supersedes. let mut timed = track_producer("timed", Info::default().with_timescale(Timescale::MILLI)); append_frozen(&mut timed); append_frozen(&mut timed); assert_eq!(drain(&mut timed.subscribe(history())), vec![1]); // Untimed: the wall clock agrees, and an hour on the unchanged newest group still // bootstraps a late joiner. let mut untimed = track_producer("untimed", Info::default()); append_untimed(&mut untimed); append_untimed(&mut untimed); assert_eq!(drain(&mut untimed.subscribe(history())), vec![1]); elapse(&untimed, Duration::from_secs(3600)); assert_eq!(drain(&mut untimed.subscribe(None)), vec![1]); assert_eq!(drain(&mut untimed.subscribe(history())), vec![1]); // An explicit age keeps superseded groups fresh that long; explicit `None` keeps them all. let mut explicit = track_producer( "explicit", Info::default() .with_timescale(Timescale::MILLI) .with_max_age(Duration::from_secs(30)), ); let mut unlimited = track_producer("unlimited", Info::timed()); for millis in [0, 1_000, 31_000] { append_at(&mut explicit, millis); append_at(&mut unlimited, millis); } assert_eq!(drain(&mut explicit.subscribe(history())), vec![1, 2]); assert_eq!(drain(&mut unlimited.subscribe(history())), vec![0, 1, 2]); } #[test] #[cfg(feature = "async")] fn an_explicit_start_gets_no_exemption_from_the_budget() { let mut producer = track_producer("test", Info::timed()); for second in 0..4 { append_at(&mut producer, second * 1000); } // Asking to start at the beginning is a filter, not a request for reliability. // Backfill needs a budget that covers it; without one the live edge still wins. let mut subscriber = producer.subscribe(Subscription::default().with_start(0)); subscriber.set_range(0..); assert_eq!(drain(&mut subscriber), vec![3]); let mut patient = producer.subscribe( Subscription::default() .with_start(0) .with_delay(Delay::default().with_max(Duration::from_secs(10))), ); patient.set_range(0..); assert_eq!(drain(&mut patient), vec![0, 1, 2, 3]); } /// A group's age is where its content *ends*, not where it began. A long group whose /// tail is level with the live edge still owes the reader every frame in it, so /// judging it by its first timestamp would discard exactly the group being filled. #[test] #[cfg(feature = "async")] fn a_long_group_is_not_stale_while_its_tail_reaches_the_edge() { let mut producer = track_producer("test", Info::timed()); // Group 0 spans 0..2000ms; group 1 starts at 2000ms, where group 0 ends. let mut long = producer.append_next().unwrap(); for ms in [0u64, 500, 1000, 1500, 2000] { long.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } long.finish().unwrap(); append_at(&mut producer, 2000); // A budget far shorter than the group's own span still keeps it. let mut sub = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); assert_eq!(drain(&mut sub), vec![0, 1]); } /// The same group, once its tail has fallen behind, is stale like any other. /// The same long group, once its *successor* has itself fallen behind. A frame's /// duration is not on the wire, so group 0's own last timestamp proves nothing about /// where it ends; only group 1's start bounds it. Group 0 is convicted once that bound /// is further behind the edge than the budget allows. #[test] #[cfg(feature = "async")] fn a_long_group_is_stale_once_its_successor_falls_behind() { let mut producer = track_producer("test", Info::timed()); let mut long = producer.append_next().unwrap(); for ms in [0u64, 500, 1000] { long.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } long.finish().unwrap(); // Group 0 reaches at most 3s (where group 1 starts), a full second behind the 4s // edge and so past the budget. Group 1 reaches the edge itself and is kept. append_at(&mut producer, 3000); append_at(&mut producer, 4000); let mut sub = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); assert_eq!(drain(&mut sub), vec![1, 2]); } /// A frameless successor is not content, so the next started group bounds the group /// before it. That bound is sound: the frameless group's first frame is checked to start /// at or below the next group's, so the true reach can only be earlier. #[test] #[cfg(feature = "async")] fn a_frameless_successor_leaves_the_next_started_group_as_the_bound() { let mut producer = track_producer("test", Info::timed()); // Any budget, so only media time judges: the wall clock stands still here. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); append_at(&mut producer, 0); // seq 0 let _writer_1 = producer.append_next().unwrap(); // seq 1 stalls before its first frame append_at(&mut producer, 10_000); // seq 2 append_at(&mut producer, 11_000); // seq 3 // Groups 0 and 1 both reach at most group 2's start, a second behind the edge: stale. assert_eq!(drain(&mut subscriber), vec![2, 3]); } fn untimed_producer() -> Producer { track_producer("test", None) } /// A track that declares no timescale is untimed, whether created or accepted, so a /// timestamped write is refused rather than given an implicit scale. #[test] fn an_undeclared_timescale_is_untimed() { let broadcast = broadcast::Info::new().produce(); let mut created = broadcast.create_track("created", None).unwrap(); let mut accepted = broadcast.reserve_track("accepted").unwrap().accept(None); assert_eq!(Info::default().timescale, None); for track in [&mut created, &mut accepted] { assert!(matches!( track.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); let mut group = track.append_group(Timestamp::unscaled(0)).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), bytes::Bytes::from_static(b"x")) .unwrap(); } } /// A group of untimed frames. #[cfg(feature = "async")] fn append_untimed(producer: &mut Producer) -> u64 { let mut group = producer.append_next().unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), bytes::Bytes::from_static(b"x")) .unwrap(); let group = group.finish().unwrap(); group.start } /// The wall clock judges an untimed track, so a new subscriber starts at the oldest /// cached group it doesn't find stale, the way a timed one does: a group is stale once /// its successor arrived a full budget ago. #[test] #[cfg(feature = "async")] fn an_untimed_subscriber_starts_at_the_oldest_fresh_group() { let mut producer = untimed_producer(); // Group N's successor arrives N + 1 seconds in. for _ in 0..5 { append_untimed(&mut producer); elapse(&producer, Duration::from_secs(1)); } // Four seconds after group 4 arrived, groups 0 to 2 were superseded 4, 3, and 2 // seconds ago: past a 1.5s budget. Group 3 was superseded 1s ago. let mut patient = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500)))); assert_eq!(drain(&mut patient), vec![3, 4]); // No budget takes the newest group alone, which is never stale. let mut live = producer.subscribe(None); assert_eq!(drain(&mut live), vec![4]); let mut replaying = producer.subscribe(replay()); assert_eq!(drain(&mut replaying), vec![0, 1, 2, 3, 4]); append_untimed(&mut producer); assert_eq!(drain(&mut live), vec![5], "every later group is delivered"); } /// An explicit start is a floor, not a request: the budget still skips what it finds /// stale above it, as on a timed track. #[test] #[cfg(feature = "async")] fn an_explicit_start_floors_an_untimed_track() { let mut producer = untimed_producer(); for _ in 0..5 { append_untimed(&mut producer); } let mut subscriber = producer.subscribe(replay().with_start(2)); assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); let mut live = producer.subscribe(Subscription::default().with_start(2)); assert_eq!(drain(&mut live), vec![4]); } /// An unfloored untimed subscription with an end starts at the latest group below it, /// as the lite publisher serves a SUBSCRIBE with an end and no start. #[test] #[cfg(feature = "async")] fn an_untimed_start_stays_under_the_end() { let mut producer = untimed_producer(); for _ in 0..5 { append_untimed(&mut producer); } let mut subscriber = producer.subscribe(Subscription::default().with_end(3)); subscriber.set_range(..3); assert_eq!(drain(&mut subscriber), vec![2]); } /// A refused single-frame write leaves no group behind: an empty open group would hold a /// subscriber waiting for a frame that never comes. #[test] fn a_mismatched_frame_appends_no_group() { let mut producer = untimed_producer(); assert!(matches!( producer.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); assert_eq!(producer.append_next().unwrap().start, 0); } /// Every group the ordered cursor can read right now, in key order. #[cfg(feature = "async")] fn drain_ordered(subscriber: &mut Ordered) -> Vec { let mut keys = Vec::new(); while let Some(Ok(Some(item))) = subscriber.next_group().now_or_never() { if let Item::Group(group) = item { keys.push(group.start); } } keys } /// A cached backlog is not free to deliver: a consumer that stalls and resumes would /// otherwise replay it at 1x and stay behind forever, since nothing it reads ever /// blocks. The budget applies as the cursor reads, exactly as on the arrival cursor. #[test] #[cfg(feature = "async")] fn next_group_sheds_a_stale_backlog() { let mut producer = track_producer("test", Info::timed()); for second in 0..4 { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(None).ordered(); assert_eq!(drain_ordered(&mut subscriber), vec![3]); // The arrival cursor, which the relay forwarders use, sheds the same backlog. let mut arrival = producer.subscribe(None); assert_eq!(drain(&mut arrival), vec![3]); } /// Only what is provably too old goes. A group is convicted by its *reach* (where its /// successor begins), so a budget spanning part of the backlog keeps every group that /// could still present something inside it, and the burst is delivered gap-free. #[test] #[cfg(feature = "async")] fn next_group_keeps_a_backlog_inside_the_budget() { let mut producer = track_producer("test", Info::timed()); for second in 0..4 { append_at(&mut producer, second * 1000); } // Group 0 reaches 1s, a full 2s behind the 3s edge, so it is gone. Group 1 // reaches 2s and could still present up to it: inside a 1.5s budget. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500)))) .ordered(); assert_eq!(drain_ordered(&mut subscriber), vec![1, 2, 3]); // A budget covering the whole history still bursts it in full. let mut replay = producer.subscribe(replay()).ordered(); assert_eq!(drain_ordered(&mut replay), vec![0, 1, 2, 3]); } /// The ordered cursor judges a group past a frameless successor by the next started one. #[test] #[cfg(feature = "async")] fn next_group_judges_past_a_frameless_successor() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); // seq 0 let _writer_1 = producer.append_next().unwrap(); // seq 1 stalls before its first frame append_at(&mut producer, 10_000); // seq 2 append_at(&mut producer, 11_000); // seq 3 // Groups 0 and 1 reach at most 10s, a second behind the edge: convicted. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))) .ordered(); assert_eq!(drain_ordered(&mut subscriber), vec![2, 3]); } #[test] #[cfg(feature = "async")] fn real_time_skips_older_keys_with_equal_ages() { let mut producer = track_producer("test", Info::timed()); append_frozen(&mut producer); append_frozen(&mut producer); let mut subscriber = producer.subscribe(None); assert_eq!(drain(&mut subscriber), vec![1]); } #[test] fn fetch_omission_keeps_unbounded_history() { let mut producer = track_producer("test", Info::timed()); for second in 0..4 { append_at(&mut producer, second * 1000); } // Omission adds no reader bound when the publisher declared none. let consumer = producer.consume(); let group = consumer.fetch_group(0, None).now_or_never().unwrap().unwrap(); assert_eq!(group.start, 0); } #[test] fn fetch_applies_zero_and_publisher_budgets() { let mut producer = track_producer("test", Info::timed().with_max_age(Duration::from_millis(1500))); for second in 0..4 { append_at(&mut producer, second * 1000); } let consumer = producer.consume(); let zero = group::Fetch::default().with_max_delay(Duration::ZERO); assert!(matches!( consumer.fetch_group(2, zero).now_or_never().unwrap(), Err(Error::Old) )); assert!(matches!( consumer.fetch_group(0, None).now_or_never().unwrap(), Err(Error::Old) )); // A reader cannot relax max_age, but can tighten it independently. let loose = group::Fetch::default().with_max_delay(Duration::from_secs(60)); assert_eq!(consumer.fetch_group(2, loose).now_or_never().unwrap().unwrap().start, 2); let tight = group::Fetch::default().with_max_delay(Duration::from_millis(500)); assert!(matches!( consumer.fetch_group(1, tight).now_or_never().unwrap(), Err(Error::Old) )); } #[test] fn cold_fetch_expires_without_waiting_for_the_upstream_group() { let mut producer = track_producer("test", Info::timed()); let _dynamic = producer.requests(); let mut successor = producer.create_group(group::Info { start: 1 }).unwrap(); successor .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); successor.finish().unwrap(); append_at(&mut producer, 2000); let options = group::Fetch::default().with_max_delay(Duration::ZERO); assert!(matches!( producer.consume().fetch_group(0, options).now_or_never().unwrap(), Err(Error::Old) )); } #[kio::test] #[cfg(feature = "async")] async fn fetched_group_keeps_its_budget_while_reading() { let mut producer = track_producer("test", Info::timed()); let mut old = producer.append_next().unwrap(); old.write_frame(old.key_time(), bytes::Bytes::from_static(b"first")) .unwrap(); let consumer = producer.consume(); let options = group::Fetch::default().with_max_delay(Duration::from_millis(500)); let mut fetched = consumer.fetch_group(0, options).await.unwrap(); assert!(fetched.read_frame().await.unwrap().is_some()); let pending = kio::clock::sim::spawn(async move { fetched.read_frame().await }); kio::clock::sim::yield_now().await; elapse(&producer, Duration::from_secs(1)); append_at(&mut producer, 1000); append_at(&mut producer, 2000); let result = pending.await.unwrap(); assert!( matches!(result, Err(Error::Old)), "a drained but unfinished group fails: {result:?}" ); } /// A start requested before the accept holds a reader of the start from the moment the /// track is visible, while its groups are read as they arrive, until the source /// resolves it. #[test] #[cfg(feature = "async")] fn a_requested_start_waits_for_the_source() { let waiter = kio::Waiter::noop(); let request = Request::new(Arc::new(broadcast::Info::default()), "test"); let consumer = request.consume(); let mut producer = request.request_start(2).accept(None); let mut subscriber = consumer.subscribe(None).now_or_never().unwrap().unwrap(); assert!( subscriber.poll_start(&waiter).is_pending(), "the floor is not the start" ); // Nothing below the floor is waited for, and a group above it reads at once. producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); assert_eq!(subscriber.assert_group().start, 3); assert!(subscriber.poll_start(&waiter).is_pending()); producer.start_at(3).unwrap(); assert_eq!(subscriber.poll_start(&waiter), Poll::Ready(Some(3))); } /// A copy accepted idle is never readable as live in between. #[test] fn an_idle_accept_is_not_live() { let request = Request::new(Arc::new(broadcast::Info::default()), "test"); let consumer = request.consume(); let mut producer = request.idle().accept(None); assert!(!consumer.is_live(), "idle from the accept"); producer.set_live(None, None).unwrap(); assert!(consumer.is_live()); } /// A fetched group installs metadata but never accepts the track's live feed. #[test] fn a_fetch_only_cache_is_not_live() { let waiter = kio::Waiter::noop(); let request = Request::new(Arc::new(broadcast::Info::default()), "test"); let dynamic = request.requests(); let consumer = request.consume(); let mut pending = consumer.fetch_group(2, None); let Poll::Ready(Ok(req)) = dynamic.poll_requested_fill_group(&waiter) else { panic!("fetch request should be ready"); }; let mut fetched = req.accept(None).unwrap().expect("a fresh group"); fetched .write_frame(crate::Timestamp::unscaled(fetched.start), b"2-0".as_ref()) .unwrap(); fetched.finish().unwrap(); let Poll::Ready(Ok(mut group)) = kio::Task::poll(&mut *pending, &waiter) else { panic!("fetched group should be cached"); }; assert_eq!(group.start, 2); let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("fetched frame should be cached"); }; assert_eq!(frame.payload.as_ref(), b"2-0"); assert!(!consumer.is_live(), "a fetch-only cache is not live"); let mut producer = request.accept(None); assert!(consumer.is_live(), "accepting the track starts its live feed"); producer.set_idle(); assert!(!consumer.is_live(), "an idle accepted copy is not live"); producer.set_live(None, None).unwrap(); assert!(consumer.is_live(), "the answered copy is live again"); // The fetch handler holds the same producer lifetime, so both leave. drop(dynamic); drop(producer); assert!(!consumer.is_live(), "a track whose producer left is not live"); } /// A one-shot fetch populates the shared cache but never the live arrival cursor, /// so it cannot make content available to a subscription look stale. #[kio::test] #[cfg(feature = "async")] async fn fetched_group_is_not_a_live_drift_edge() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); append_at(&mut producer, 0); let pending = consumer.fetch_group(100, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("fetch request is ready") .unwrap(); let mut fetched = req.accept(Info::timed()).unwrap().expect("a fresh group"); fetched .write_frame( Timestamp::from_millis(100_000).unwrap(), bytes::Bytes::from_static(b"fetched"), ) .unwrap(); fetched.finish().unwrap(); pending.await.unwrap(); let mut groups = producer.subscribe(None); assert_eq!(groups.assert_group().start, 0); groups.assert_no_group(); } /// A failed fill answer is no verdict on a gap the feed owes: an ordered reader neither takes /// the failed copy nor moves past it, and gets the feed's group once it lands. Mirrors JS /// `fill: a failed answer to a gap the feed owes holds the end and the ordered reader`. #[test] #[cfg(feature = "async")] fn a_failed_fill_answer_holds_an_ordered_reader() { for error in [Error::Cancel, Error::App(1)] { let producer = received_producer(None); let dynamic = producer.requests(); let consumer = producer.consume(); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut ordered), [0]); let pending = consumer.fetch_group(1, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("fill request is ready") .unwrap(); req.accept(None) .unwrap() .expect("a fresh group") .abort(error.clone()) .unwrap(); drop(pending); let waiter = kio::Waiter::noop(); assert!( ordered.poll_next_group(&waiter).is_pending(), "{error}: group 1 is still owed" ); put(&producer, 1, None); assert_eq!( drain_ordered(&mut ordered), [1, 2], "{error}: the feed's group lands in order" ); } } /// The live edge is resolved once per poll and applied to every group that poll /// walks off, so it has to be revalidated before it convicts anything: an eviction /// in between would otherwise discard a group on the strength of content that is no /// longer there to jump to. #[test] fn an_evicted_live_edge_convicts_nothing() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); let edge = append_at(&mut producer, 30_000); let state = producer.state.read(); let drift = Drift { budget: Duration::ZERO, bounded: false, edge: state.drift_edge(&Servable::ALL), coverage: None, }; assert!( state.is_stale(0, &drift.edge, drift.budget), "stale against a live edge" ); drop(state); // The edge dies between resolving it and judging the candidate. let slot = producer.modify().unwrap().remove_slot(edge).unwrap(); let _ = slot.group.abort(Error::Evicted); let state = producer.state.read(); assert!( !state.is_stale(0, &drift.edge, drift.budget), "a vanished edge is no reason to drop what is left" ); } #[test] #[cfg(feature = "async")] fn a_lower_key_is_never_the_live_edge() { let producer = track_producer("test", Info::timed()); // A high timestamp on a lower key is not a live edge: a group whose frames run // past where its successor starts sits there. Only groups above the candidate // anchor the measure, so it cannot convict the group that follows it. let mut straggler = producer.create_group(0u64.into()).unwrap(); for millis in [0, 60_000] { straggler .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } straggler.finish().unwrap(); let mut next = producer.create_group(1u64.into()).unwrap(); next.write_frame(next.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); next.finish().unwrap(); let mut subscriber = producer.subscribe(replay()); assert_eq!(drain(&mut subscriber), vec![0, 1]); } #[test] #[cfg(feature = "async")] fn a_range_behind_the_live_edge_keeps_its_newest_group() { let mut producer = track_producer("test", Info::timed()); for second in 0..=10 { append_at(&mut producer, second * 1000); } // Drift is measured under where the range ends, so with no budget its own newest // group is still fresh: the groups past it are not ones it reads. let mut subscriber = producer.subscribe(Subscription::default().with_range(0..2)); assert_eq!(drain(&mut subscriber), vec![1]); let mut subscriber = producer.subscribe(Subscription::default().with_range(0..2)).ordered(); // Group 0 is stale against it, which the read reports before the group. assert_eq!(items(&mut subscriber), ["gap 0..1 (old)", "group 1"]); // A budget that covers it delivers the range, and nothing above it. let mut subscriber = producer.subscribe(replay().with_end(2)); assert_eq!(drain(&mut subscriber), vec![0, 1]); } /// Drift is measured only against groups the ranges cover: a group in the gap between /// two spans is never read, so it neither sets the edge nor replaces the newest group /// below it. A group in the later span is read, so it does: here group 9, which contains /// its start, while group 8 below it is in the gap. #[test] #[cfg(feature = "async")] fn a_gap_between_ranges_is_not_a_drift_edge() { let ranges = || Subscription::default().with_ranges([Range::groups(0..2), Range::groups(10..12)]); let producer = track_producer("test", Info::timed()); for (key, millis) in [(0, 0), (1, 1000), (8, 8000), (9, 9000)] { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); group.finish().unwrap(); } // With no budget, the newest group the ranges cover is the edge. assert_eq!(drain(&mut producer.subscribe(ranges())), vec![9]); let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); ten.write_frame(Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); ten.finish().unwrap(); assert_eq!(drain(&mut producer.subscribe(ranges())), vec![10]); } /// A group handed out before the ranges moved above it is judged against the new run, /// not a cached group below its start: here group 1 would put group 0 18s behind the /// edge, past the budget, where the run's own group 10 puts it 9s behind. #[kio::test] #[cfg(feature = "async")] async fn a_held_group_is_judged_against_the_moved_run() { let at = |millis| Timestamp::from_millis(millis).unwrap(); let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(15))); let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(budget.clone()); let mut zero = producer.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); let mut held = subscriber.recv_group().await.unwrap().expect("group 0"); assert!(held.read_frame().await.unwrap().is_some()); // The run's own group 10 is held, so its time start names it. let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); ten.write_frame(at(10_000), bytes::Bytes::from_static(b"x")).unwrap(); ten.finish().unwrap(); subscriber.update(budget.with_range(10..20)).unwrap(); for (key, millis) in [(1, 1000), (19, 19_000)] { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group.write_frame(at(millis), bytes::Bytes::from_static(b"x")).unwrap(); group.finish().unwrap(); } let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); kio::clock::sim::yield_now().await; assert!(!pending.is_finished(), "group 0 is within the budget of its run"); } /// A drift snapshot taken while the window waits on its first group is taken again once /// that group settles it. Groups 0 and 1 land between the snapshot and the arrival: the /// settled `0..=0` judges group 0 against itself, not group 1, which it never reads. #[test] fn the_first_arrival_retakes_a_pending_drift_snapshot() { for ordered in [false, true] { let mut producer = track_producer("test", Info::timed()); let subscription = Subscription::default() .with_delay(Delay::default().with_max(Duration::ZERO)) .with_ranges([Range { start: Anchor::Time(0), end: Some(Anchor::Relative(0)), }]); let mut subscriber = producer.subscribe(subscription); let Inner::Plain(cursor) = &mut subscriber.inner else { unreachable!("a local subscriber reads its own cursor"); }; let waiter = kio::Waiter::noop(); cursor.sync(); assert!(cursor.filter.window.is_pending()); let Poll::Ready(Ok(drift)) = cursor.poll_drift(&cursor.servable, &waiter) else { unreachable!("a drift snapshot is always ready"); }; append_at(&mut producer, 0); append_at(&mut producer, 1000); let res = match ordered { true => cursor.poll_next_with(0, drift, &waiter), false => cursor.poll_recv_with(drift, &waiter), }; let Poll::Ready(Ok(Some(peeked))) = res else { panic!("group 0 is the newest group its range reads (ordered: {ordered})"); }; assert_eq!(peeked.group.start, 0); } } /// A [`Control`] replacing the ranges and the cursor moving its cap each change only /// their own part of what handed-out groups are judged against, whichever goes first, /// and with the cap moving between the ranges being written and republished. #[test] fn a_control_update_and_a_cap_never_publish_a_mismatched_pair() { let published = |subscriber: &Subscriber| { let Inner::Plain(cursor) = &subscriber.inner else { unreachable!("a local subscriber reads its own cursor"); }; let current = cursor.published.read(); (current.servable.clone(), current.cap) }; let ranges = Subscription::default().with_range(10..20); // Nothing in 10..20 sits below a cap at 5. let expected = (Servable::NONE, Some(5)); for order in ["ranges first", "cap first", "cap between"] { let producer = track_producer("test", Info::timed()); // Group 10 is held, so the time start names it. producer .append_group(Timestamp::from_millis(10).unwrap()) .unwrap() .finish() .unwrap(); let mut subscriber = producer.subscribe(Subscription::default()); let control = subscriber.control(); match order { "ranges first" => { control.update(ranges.clone()).unwrap(); subscriber.set_range(..5); } "cap first" => { subscriber.set_range(..5); control.update(ranges.clone()).unwrap(); } _ => { control.subscription.update(ranges.clone()).unwrap(); subscriber.set_range(..5); control.republish.as_ref().unwrap().publish(&control.subscription); } } assert!(published(&subscriber) == expected, "{order}"); let _ = subscriber.poll_recv_group(&kio::Waiter::noop()); assert!(published(&subscriber) == expected, "{order}, once the cursor syncs"); } } /// A search over a servable window of several runs, through the reader's index of the /// live groups, finds what a scan of every live group would, in either direction, as /// groups arrive, start, abort, leave the cache, or arrive again between searches that /// keep the same index, and as the window is edited or replaced. Each window is searched on two tracks, as a /// group's expiry that moved to a replacement copy searches its reader's. #[test] fn live_index_searches_match_a_scan_of_every_live_group() { // A small deterministic generator, so a failure replays. let seed = std::cell::Cell::new(0x2545_f491_4f6c_dd1d_u64); let next = |bound: u64| { let mut x = seed.get(); x ^= x << 13; x ^= x >> 7; x ^= x << 17; seed.set(x); x % bound }; const KEYS: u64 = 40; let at = |millis| Timestamp::from_millis(millis).unwrap(); for round in 0..300 { let producers = [ track_producer("test", Info::timed()), track_producer("copy", Info::timed()), ]; let runs = |count: u64| -> Vec<_> { (0..next(count + 1)) .map(|_| { let start = next(KEYS); start..=start + next(5) }) .collect() }; let mut cover = Cover::new(runs(8)); let mut layer = Vec::new(); let mut groups = [Vec::new(), Vec::new()]; let mut servable = Servable::NONE; let live = Arc::::default(); for step in 0..30 { let key = next(KEYS); let track = next(2) as usize; let (producer, groups) = (&producers[track], &mut groups[track]); match next(6) { 0 | 1 => { if let Ok(mut group) = producer.create_group(group::Info { start: key }) { if next(3) > 0 { group .write_frame(at(key * 10), bytes::Bytes::from_static(b"x")) .unwrap(); } groups.push(group); } } 2 => { if let Some(index) = groups.iter().position(|group| group.start == key) { let _ = groups.swap_remove(index).abort(Error::Cancel); } } 3 => { let _ = producer.modify().unwrap().uncache(key); } _ => {} } // Now and then the cap moves, keeping the runs and their index, or the runs are // edited. if step == 0 || next(4) == 0 { let cap = (next(4) == 0).then(|| next(KEYS + 2)); if next(2) == 0 { for run in std::mem::replace(&mut layer, runs(2)) { cover.add(run, -1); } for run in &layer { cover.add(run.clone(), 1); } } else if next(8) == 0 { // Ranges replaced whole: a cover of another lineage. layer.clear(); cover = Cover::new(runs(8)); } servable = Servable::covering(cover.clone(), cap, &live); } for (track, producer) in producers.iter().enumerate() { let state = producer.state.read(); let shown = |slot: &Slot| slot.is_shown().then_some(slot.group.start); for from in [0, next(KEYS), next(KEYS)] { for rev in [false, true] { let scan = state.live(from..).filter(|(key, _)| servable.contains(**key)); let expected = match rev { true => scan.map(|(_, slot)| slot).rev().find_map(shown), false => scan.map(|(_, slot)| slot).find_map(shown), }; let found = state.find_servable(&servable, from, rev, shown); let shape = match &servable { Servable::Run { start, end } => format!("run {start}..{end:?}"), Servable::Runs(several) => format!("{:?} below {:?}", several.groups, several.cap), }; let cached: Vec<_> = state.live(..).map(|(key, slot)| (*key, slot.is_shown())).collect(); assert_eq!( found, expected, "round {round} step {step} track {track} from {from}, rev {rev}: {shape}, cached {cached:?}" ); } } } } } } /// Once churn leaves the index holding more entries than the cache, a cache that then /// shrinks to its newest group still lets the index shrink, though every search answers /// at the newest entry and never meets the stale ones. #[test] fn the_live_index_shrinks_with_the_cache() { let producer = track_producer("test", Info::timed()); let groups: Vec<_> = (0..100) .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let runs = Cover::new([0..=999, 1001..=u64::MAX]); let servable = Servable::covering(runs, None, &Arc::default()); let Servable::Runs(several) = &servable else { panic!("several runs"); }; let entries = || -> usize { let indexes = several.live.0.lock().unwrap(); indexes.iter().map(|indexed| indexed.groups.len()).sum() }; let newest = |slot: &Slot| Some(slot.group.start); assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), Some(99) ); assert_eq!(entries(), 100); // Most of the cache leaves, so a prune keeps only the groups still cached. for key in 0..60 { producer.modify().unwrap().uncache(key).unwrap(); } assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), Some(99) ); assert_eq!(entries(), 40); // Then all but the newest leaves. for key in 60..99 { producer.modify().unwrap().uncache(key).unwrap(); } assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), Some(99) ); assert_eq!(entries(), 1, "the stale entries are pruned"); drop(groups); } /// Capping a servable window in place keeps exactly the groups the set of its runs /// keeps below the cap, so the single run stays off the heap without moving a bound. #[test] fn servable_below_matches_its_runs_below_the_cap() { // Inclusive (first, last) bounds, lowest first. let windows: [&[(u64, u64)]; 5] = [ &[], &[(0, u64::MAX)], &[(5, 9)], &[(5, u64::MAX)], &[(2, 3), (6, 8), (12, u64::MAX)], ]; let caps = [ None, Some(0), Some(2), Some(3), Some(5), Some(7), Some(9), Some(10), Some(13), Some(u64::MAX), ]; let points = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u64::MAX - 1, u64::MAX]; for bounds in windows { let runs = || bounds.iter().map(|&(first, last)| first..=last); // The runs counted once, and again with the first counted twice. let covers = [Cover::new(runs()), Cover::new(runs().chain(runs().take(1)))]; for groups in covers { let servable = Servable::covering(groups, None, &Arc::default()); if let [(first, last)] = bounds { assert!( servable == Servable::Run { start: *first, end: last.checked_add(1) }, "{bounds:?}" ); } for cap in caps { let expected = below(runs(), cap); let capped = servable.below(cap); for point in points { assert_eq!( capped.contains(point), expected.contains(&point), "{bounds:?} below {cap:?} at {point}" ); } // One run below the cap stays off the heap. if let Some(run) = expected.iter().next() && expected.len() == 1 { assert!( capped == Servable::Run { start: *run.start(), end: run.end().checked_add(1) }, "{bounds:?} below {cap:?}" ); } } } } } /// Ranges replaced through a [`Control`] republish what a held group is judged against /// at once, before the cursor next reads: group 0 is judged against the run's own /// group 10, not group 1, which the new ranges never read. #[kio::test] #[cfg(feature = "async")] async fn a_control_update_moves_the_run_a_held_group_is_judged_against() { let at = |millis| Timestamp::from_millis(millis).unwrap(); let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(15))); let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(budget.clone()); let mut zero = producer.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); let mut held = subscriber.recv_group().await.unwrap().expect("group 0"); assert!(held.read_frame().await.unwrap().is_some()); // The run's own group 10 is held, so its time start names it. let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); ten.write_frame(at(10_000), bytes::Bytes::from_static(b"x")).unwrap(); ten.finish().unwrap(); subscriber.control().update(budget.with_range(10..20)).unwrap(); for (key, millis) in [(1, 1000), (19, 19_000)] { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group.write_frame(at(millis), bytes::Bytes::from_static(b"x")).unwrap(); group.finish().unwrap(); } let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); kio::clock::sim::yield_now().await; assert!(!pending.is_finished(), "group 0 is within the budget of its run"); drop(subscriber); } #[test] #[cfg(feature = "async")] fn a_capped_subscriber_measures_drift_against_its_cap() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); append_at(&mut producer, 1000); // Capped at group 0: the route running on past the cap is data this subscriber // can never be served, so it isn't a live edge to be late against. let mut subscriber = producer.subscribe(Subscription::default()); subscriber.set_range(..1); assert_eq!(drain(&mut subscriber), vec![0]); // Raising the cap re-offers the parked group, now measured against the wider // edge it just admitted. subscriber.set_range(..); assert_eq!(drain(&mut subscriber), vec![1]); } #[test] #[cfg(feature = "async")] fn subscriber_control_updates_while_read_future_is_pending() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let control = subscriber.control(); let mut recv = Box::pin(subscriber.recv_group()); assert!(recv.as_mut().now_or_never().is_none()); control.update(Subscription::default().with_priority(7)).unwrap(); let aggregate = producer.subscription().expect("expected an active subscription"); assert_eq!(aggregate.priority, 7); } #[test] fn dropped_subscriber_leaves_no_ghost_in_aggregate() { // Regression (#2351): a departed subscriber must not keep contributing its // last subscription to the aggregate. When it did, a relay's linger loop // never observed the track going idle, and an identical viewer reconnecting // within the linger window was reset when the stale timer fired. let mut producer = track_producer("test", Info::timed()); let a = producer.subscribe(Subscription::default().with_priority(5)); // Prime the change cursor: the aggregate currently has one subscriber. let waiter = kio::Waiter::noop(); assert!( matches!(producer.poll_subscription_changed(&waiter), Poll::Ready(Ok(Some(_)))), "one live subscriber should aggregate to Some", ); // The only subscriber leaves. drop(a); // The aggregate must report the drop to None, not the ghost's last value. assert!( matches!(producer.poll_subscription_changed(&waiter), Poll::Ready(Ok(None))), "a dropped subscriber must not linger in the aggregate", ); // And the snapshot used by the linger loop must agree. assert!( producer.subscription().is_none(), "snapshot must exclude a dropped subscriber", ); } #[test] fn dropped_subscriber_wakes_the_aggregate() { // The value being right isn't enough: nothing re-polls the aggregate on its // own, so the drop has to wake the waiter. A subscriber contributing demand // takes `kio::Consumer::poll`'s Ready path, which registers no waiter, so // the departure needs the closed waiter armed explicitly. Without it a relay // never learns the last viewer left and holds the upstream subscription (and // the upstream's viewer count) open forever. use std::sync::atomic::{AtomicBool, Ordering}; let mut producer = track_producer("test", Info::timed()); let a = producer.subscribe(Subscription::default().with_priority(5)); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); // Prime the cursor, then confirm the next poll parks. assert!(matches!( producer.poll_subscription_changed(&waiter), Poll::Ready(Ok(Some(_))) )); assert!( producer.poll_subscription_changed(&waiter).is_pending(), "the aggregate is unchanged, so this poll must park", ); assert!(!woken.load(Ordering::SeqCst), "nothing happened yet"); drop(a); assert!( woken.load(Ordering::SeqCst), "the last subscriber leaving must wake the aggregate watcher", ); } #[test] fn widest_subscriber_update_wakes_the_aggregate() { // The value counterpart of the drop above. A subscriber that widens the // fold takes the Ready path too, so its next update registered no waiter. // A relay whose upstream cap came from a downstream reader then never // learned that the reader lifted it, and the groups parked upstream never // resumed: every session stayed up and nothing flowed. use std::sync::atomic::{AtomicBool, Ordering}; let mut producer = track_producer("test", Info::timed()); let _narrow = producer.subscribe(Subscription::default().with_end(4)); let mut wide = producer.subscribe(Subscription::default()); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(matches!( producer.poll_subscription_changed(&waiter), Poll::Ready(Ok(Some(_))) )); assert!(producer.poll_subscription_changed(&waiter).is_pending()); assert!(!woken.load(Ordering::SeqCst), "nothing happened yet"); wide.update(Subscription::default().with_end(6)).unwrap(); assert!( woken.load(Ordering::SeqCst), "the widest subscriber changing must wake the aggregate watcher", ); match producer.poll_subscription_changed(&waiter) { Poll::Ready(Ok(Some(sub))) => assert_eq!(sub.end(), Some(6)), other => panic!("expected the narrowed aggregate, got {other:?}"), } } /// An [`ArcWake`] that just records that it was woken. struct FlagWake(Arc); impl futures::task::ArcWake for FlagWake { fn wake_by_ref(arc_self: &Arc) { arc_self.0.store(true, std::sync::atomic::Ordering::SeqCst); } } #[test] fn out_of_order_max_key_at_front() { let producer = track_producer("test", Info::timed()); // Arrive out of order: seq 5 first, then 3, then 4. let _open = producer.create_group(group::Info { start: 5 }).unwrap(); let _open = producer.create_group(group::Info { start: 3 }).unwrap(); let _open = producer.create_group(group::Info { start: 4 }).unwrap(); // max_key = 5, which is at the front of the VecDeque. { let state = producer.state.read(); assert_eq!(state.max_key, Some(5)); } // Expire all three groups. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Append seq 6 (becomes new max_key). let _open = producer.append_next().unwrap(); // seq 6 // Seq 3, 4, 5 are all expired. Seq 5 was the old max_key but now 6 is. // All old groups are evicted. { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); assert_eq!(first_live_key(&state), 6); assert!(!state.lookup.contains_key(&3)); assert!(!state.lookup.contains_key(&4)); assert!(!state.lookup.contains_key(&5)); assert!(state.lookup.contains_key(&6)); } } #[test] fn max_key_at_front_blocks_trim() { let producer = track_producer("test", Info::timed()); // Arrive: seq 5, then seq 3. producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Seq 3 arrives late; max_key is still 5 (at front). producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); // Seq 5 is max_key (protected). Seq 3 is not expired (just created). // Nothing should be evicted. { let state = producer.state.read(); assert_eq!(live_groups(&state), 2); assert_eq!(first_arrival(&state), 0); } // Expire seq 3 as well. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Seq 2 arrives late, triggering eviction. producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); // Seq 5 is the live edge (protected) and still resolves at the front of // `arrival`, so nothing is trimmed and the queue still starts at position 0. // Seq 3 expired out of `lookup`, leaving a hole its arrival entry no longer // resolves; seq 2 is fresh and kept. { let state = producer.state.read(); assert_eq!(live_groups(&state), 2); assert_eq!(first_arrival(&state), 0); assert!(state.lookup.contains_key(&5)); assert!(!state.lookup.contains_key(&3)); assert!(state.lookup.contains_key(&2)); } // Consumer should still be able to read through the hole. let mut consumer = producer.subscribe(None); let group = consumer.assert_group(); // consume() starts at index 0; the first arrival entry that still resolves is seq 5. assert_eq!(group.start, 5); } #[test] #[cfg(feature = "async")] fn abort_drops_open_groups() { let producer = track_producer("test", Info::timed()); let _writer_1 = producer.append_next().unwrap(); let _writer_2 = producer.append_next().unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(live_groups(&producer.state.read()), 2); producer.clone().abort(Error::Cancel).unwrap(); // Nobody will finish them, so they leave the cache rather than park a reader. assert!( producer.state.read().lookup.is_empty(), "open groups are dropped on abort" ); let result = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(result, Err(Error::Cancel))); } #[test] #[cfg(feature = "async")] fn drop_unfinished_clears_cached_groups() { let producer = track_producer("test", Info::timed()); let writer = producer.clone(); let _open = writer.append_next().unwrap(); // A stale consumer keeps the channel (and thus the cache) alive. let mut consumer = producer.subscribe(None); assert_eq!(live_groups(&producer.state.read()), 1); // Drop every producer without finishing: the cache is released. drop(writer); drop(producer); let result = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(result, Err(Error::Dropped))); } /// Aborting a track aborts its open groups: a reader that already pulled one ends with /// the error, and the group's writer fails. #[test] #[cfg(feature = "async")] fn abort_ends_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); let mut writer = producer.append_next().unwrap(); writer .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); producer.clone().abort(Error::Cancel).unwrap(); let read = group.read_frame().now_or_never().expect("should not block"); assert!(matches!(read, Err(Error::Cancel)), "{read:?}"); let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Cancel)), "{write:?}"); } /// A reader still in an open group keeps the track used after its subscription leaves, /// so only its leaving lets an unused abort end the group's writer. #[test] #[cfg(feature = "async")] fn abort_unwanted_waits_for_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); let mut writer = producer.append_next().unwrap(); writer .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); drop(consumer); let producer = producer .abort_unwanted(Error::Cancel) .expect_err("the group's reader keeps the track"); drop(group); assert!(producer.abort_unwanted(Error::Cancel).is_ok()); let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Cancel)), "{write:?}"); } /// Dropping the last producer without a finish aborts its open groups with /// [`Error::Dropped`]. #[test] #[cfg(feature = "async")] fn drop_unfinished_ends_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); let mut writer = producer.append_next().unwrap(); writer .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); drop(producer); let read = group.read_frame().now_or_never().expect("should not block"); assert!(matches!(read, Err(Error::Dropped)), "{read:?}"); let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Dropped)), "{write:?}"); } /// Aborting a track also aborts a fetched head still being written, so its reader ends /// with the track's error rather than waiting on a fetch that can no longer land. #[kio::test] #[cfg(feature = "async")] async fn abort_ends_an_open_fetched_head() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut reading = consumer.peek_group(0).expect("the head serves group 0"); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); producer.abort(Error::App(7)).unwrap(); let read = reading.read_frame().now_or_never().expect("the abort settles the head"); assert!(matches!(read, Err(Error::App(7))), "{read:?}"); let write = head.write_frame(head.key_time(), bytes::Bytes::from_static(b"delta")); assert!(matches!(write, Err(Error::App(7))), "{write:?}"); } #[test] fn drop_after_abort_does_not_warn() { // abort() closes the channel after recording the abort. Drop must treat the // read-only guard returned by write() as clean or it emits a false WARN. let warns = count_drop_warnings("track::Producer dropped without finish", || { let producer = track_producer("test", Info::timed()); let keep = producer.clone(); let writer = producer.clone(); let group = writer.append_next().unwrap(); group.finish().unwrap(); let _consumer = producer.subscribe(None); writer.abort(Error::Cancel).unwrap(); drop(keep); }); assert_eq!(warns, 0, "abort-then-drop must not emit unfinished-producer WARN"); } #[test] fn drop_unfinished_warns() { let warns = count_drop_warnings("track::Producer dropped without finish", || { let producer = track_producer("test", Info::timed()); let writer = producer.clone(); let _open = writer.append_next().unwrap(); let _consumer = producer.subscribe(None); drop(writer); drop(producer); }); assert_eq!(warns, 1, "unfinished drop must emit one unfinished-producer WARN"); } #[test] #[cfg(feature = "async")] fn drop_finished_keeps_cached_groups() { let producer = track_producer("test", Info::timed()); producer.append_next().unwrap().finish().unwrap(); let producer = producer.finish().unwrap(); let mut consumer = producer.subscribe(None); drop(producer); // A cleanly finished track keeps its cache so the consumer can still drain. assert_eq!(consumer.assert_group().start, 0); let done = consumer.recv_group().now_or_never().expect("should not block").unwrap(); assert!(done.is_none(), "consumer should drain then see clean finish"); } /// A boundary declared ahead of the live edge, then the last producer dropping, means /// the missing groups will never come: the reader ends cleanly rather than with /// `Dropped`, as it would for a track that never declared its end. #[test] #[cfg(feature = "async")] fn drop_short_of_the_boundary_ends_cleanly() { let mut producer = track_producer("test", Info::timed()); producer.append_next().unwrap().finish().unwrap(); producer.finish_at(Position::group(3)).unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().start, 0); assert!( consumer.recv_group().now_or_never().is_none(), "groups 1 and 2 are owed" ); drop(producer); let done = consumer.recv_group().now_or_never().expect("should not block").unwrap(); assert!( done.is_none(), "the track ends at its boundary without the missing groups" ); } /// The key cursor ends the same way. A gap below the boundary is skipped, /// a later cached group is still delivered, and the read finishes cleanly. #[test] #[cfg(feature = "async")] fn ordered_ends_cleanly_when_sealed_short_of_the_boundary() { let mut producer = track_producer("test", Info::timed()); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); producer.finish_at(Position::group(4)).unwrap(); let mut ordered = producer.subscribe(replay()).ordered(); drop(producer); // A group never produced below a later one is given up, and the end is clean. assert_eq!(items(&mut ordered), ["group 0", "gap 1..2 (not found)", "group 2"]); } /// A finished track ends both readers with nothing left in range, whether the range /// ends at the cursor or the rest was given up. #[test] fn readers_end_once_a_finished_track_leaves_nothing_in_range() { for given_up in [false, true] { let producer = track_producer("test", Info::timed()); let end = if given_up { 4 } else { 3 }; let mut arrival = producer.subscribe(replay().with_range(0..end)); let mut ordered = producer.subscribe(replay().with_range(0..end)).ordered(); if given_up { producer.drop_groups(3..4, StreamError::DeliveryTimeout); } for key in [0, 1, 2, 4, 5] { write_group(&producer, key); } let _producer = producer.finish().unwrap(); for key in 0..3 { let Poll::Ready(Ok(Some(Item::Group(group)))) = ordered.poll_next_group(&kio::Waiter::noop()) else { panic!("group {key}, given_up={given_up}") }; assert_eq!(group.start, key); let Poll::Ready(Ok(Some(group))) = arrival.poll_recv_group(&kio::Waiter::noop()) else { panic!("arrival group {key}, given_up={given_up}") }; assert_eq!(group.start, key); } // The group given up is reported, then the read ends. if given_up { let Poll::Ready(Ok(Some(Item::Gap { range, error }))) = ordered.poll_next_group(&kio::Waiter::noop()) else { panic!("the given up group is reported") }; assert_eq!(range, 3..4); assert!( matches!(error, Error::Stream(StreamError::DeliveryTimeout)), "{error:?}" ); } assert!( matches!(ordered.poll_next_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), "given_up={given_up}" ); assert!( matches!(arrival.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), "arrival, given_up={given_up}" ); } } #[test] fn append_finish_cannot_be_rewritten() { let producer = track_producer("test", Info::timed()); // Finishing an empty track is valid (fin = 0, total groups = 0). Another clone can // neither finish again nor append. assert!(producer.clone().finish().is_ok()); assert!(producer.clone().finish().is_err()); assert!(producer.append_next().is_err()); } #[test] fn finish_after_groups() { let producer = track_producer("test", Info::timed()); let _open = producer.append_next().unwrap(); assert!(producer.clone().finish().is_ok()); assert!(producer.clone().finish().is_err()); assert!(producer.append_next().is_err()); } /// The handle `finish` returns still fills an older group below the end, refuses one at or /// past it, and ends its readers cleanly once the late group finishes. #[test] #[cfg(feature = "async")] fn finished_track_fills_a_late_group_below_its_end() { let producer = track_producer("test", Info::timed()); let mut ordered = producer.subscribe(replay()).ordered(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); let finished = producer.finish().unwrap(); assert!(matches!( finished.create_group(group::Info { start: 2 }), Err(Error::Closed) )); let mut late = finished.create_group(group::Info { start: 0 }).unwrap(); late.write_frame(late.key_time(), &b"late"[..]).unwrap(); let late = late.finish().unwrap(); assert_eq!(late.frame_count(), 1); let first = ordered .next_group() .now_or_never() .expect("group 0") .unwrap() .unwrap() .group() .unwrap(); assert_eq!(first.start, 0); let second = ordered .next_group() .now_or_never() .expect("group 1") .unwrap() .unwrap() .group() .unwrap(); assert_eq!(second.start, 1); let done = ordered.next_group().now_or_never().expect("end").unwrap(); assert!(done.is_none(), "the finished track ends after its late group"); assert!(finished.closed().now_or_never().expect("settled").is_ok()); } #[test] fn finish_at_rejects_a_boundary_at_or_below_the_live_edge() { let mut producer = track_producer("test", Info::timed()); let _open = producer.create_group(group::Info { start: 5 }).unwrap(); // The boundary is exclusive, so it must be strictly above the highest produced // group. 5 or below would orphan groups that already exist. assert!(producer.finish_at(Position::group(4)).is_err()); assert!(producer.finish_at(Position::group(5)).is_err()); assert!(producer.finish_at(Position::group(6)).is_ok()); assert_eq!(producer.finished(), Some(Position::group(6))); // Repeating the end is a no-op, moving it is rejected, and no group at or above the // boundary can be created. assert!(producer.finish_at(Position::group(6)).is_ok()); assert!(producer.finish_at(Position::group(7)).is_err()); assert!(producer.create_group(group::Info { start: 4 }).is_ok()); assert!(producer.create_group(group::Info { start: 6 }).is_err()); } #[test] fn a_mid_group_end_survives_eviction_and_filtered_readers() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let mut producer = track_producer_pooled("test", pool.clone()); let end = Position { group: 5, frame: 5 }; producer.finish_at(end).unwrap(); let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); for _ in 0..5 { group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } group.finish().unwrap(); let consumer = producer.consume(); drop(producer); // Date the last activity before advancing the cache clock past expiry. pool.sweep(); pool.step(Duration::from_secs(2)); pool.sweep(); assert!(consumer.peek_group(5).is_none(), "payload cache expired"); let mut sub = consumer.subscribe(replay()).now_or_never().unwrap().unwrap(); sub.set_range(..5); assert_eq!( sub.poll_finished(&kio::Waiter::noop()).map(|result| result.unwrap()), Poll::Ready(end) ); } #[test] fn a_wire_end_repeats_or_refines_but_never_moves() { let mut producer = track_producer("test", Info::timed()); // A FETCH_OK names only the boundary; END_OF_TRACK may then name the frame before it. producer.finish_at(Position::group(6)).unwrap(); producer.finish_at(Position { group: 5, frame: 5 }).unwrap(); producer.finish_at(Position { group: 5, frame: 5 }).unwrap(); assert_eq!(producer.finished(), Some(Position { group: 5, frame: 5 })); for moved in [Position { group: 5, frame: 4 }, Position::group(6), Position::group(5)] { assert!(matches!(producer.finish_at(moved), Err(Error::ProtocolViolation))); } } /// An end inside a group that already holds a frame at or past it would leave content /// the end says does not exist. #[test] fn a_mid_group_end_refuses_to_cut_existing_frames() { let mut producer = track_producer("test", Info::timed()); let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); for _ in 0..6 { group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } assert!(matches!( producer.finish_at(Position { group: 5, frame: 5 }), Err(Error::ProtocolViolation) )); assert_eq!(producer.finished(), None, "a refused end declares nothing"); producer.finish_at(Position { group: 5, frame: 6 }).unwrap(); } /// A mid-group end caps its group, whether the group exists yet or not. #[test] fn a_mid_group_end_caps_later_writes() { for early in [true, false] { let mut producer = track_producer("test", Info::timed()); let end = Position { group: 5, frame: 2 }; if early { producer.finish_at(end).unwrap(); } let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); if !early { producer.finish_at(end).unwrap(); } group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); assert!(matches!( group.write_frame(group.key_time(), b"x".as_slice()), Err(Error::Closed) )); let mut frames = frame::Buffer::<1>::new(); assert!( frames .push(frame::Frame::new(Timestamp::ZERO, bytes::Bytes::from_static(b"x"))) .is_ok() ); assert!(matches!(group.write_frames(&mut frames), Err(Error::Closed))); assert_eq!(group.frame_count(), 2); // A refused frame aborts its group, since a write past the end is a bug. assert!(matches!( group.create_frame(frame::Info { size: 1, timestamp: Timestamp::ZERO, }), Err(Error::Closed) )); assert!(producer.create_group(group::Info { start: 6 }).is_err()); } } #[test] fn end_reports_the_declared_end() { let mut producer = track_producer("test", Info::timed()); assert_eq!(producer.finished(), None); let _open = producer.create_group(group::Info { start: 5 }).unwrap(); assert_eq!(producer.finished(), None, "a group does not declare an end"); producer.finish_at(Position::group(9)).unwrap(); assert_eq!(producer.finished(), Some(Position::group(9))); // finish() would try to declare a second boundary, so callers check first. assert!(producer.finish().is_err()); } #[test] fn end_reports_the_live_edge_after_finish() { let producer = track_producer("test", Info::timed()); let _open = producer.create_group(group::Info { start: 5 }).unwrap(); producer.finish_shared().unwrap(); assert_eq!(producer.finished(), Some(Position::group(6))); } #[test] #[cfg(feature = "async")] fn finish_at_declares_a_future_boundary() { let mut producer = track_producer("test", Info::timed()); producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); // Learn the track ends at group 6 (exclusive 7) while the live edge is still 5. producer.finish_at(Position::group(7)).unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().start, 5); // The boundary is known immediately, but the track isn't done: group 6 is still // outstanding, so the consumer parks rather than seeing end-of-stream. let boundary = consumer .finished() .now_or_never() .expect("boundary is known immediately") .expect("would have errored"); assert_eq!(boundary, Position::group(7)); assert!( consumer.recv_group().now_or_never().is_none(), "should wait for the outstanding group" ); // The trailing group arrives (below the boundary), then the track completes. producer .create_group(group::Info { start: 6 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.assert_group().start, 6); let done = consumer .recv_group() .now_or_never() .expect("should not block") .expect("would have errored"); assert!(done.is_none(), "track completes once the boundary is reached"); } /// An end declared with an error keeps the history below it: every reader gets the groups /// first, then the error, and a fetch still reads them. A range short of the end ends /// cleanly, since it never reaches the error. #[kio::test] #[cfg(feature = "async")] async fn an_errored_end_keeps_its_history() { let mut producer = track_producer("test", Info::timed()); for key in 0..2 { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } producer.end_at(Position::group(2), Some(Error::App(7))).unwrap(); // A repeat must carry the same error. producer.end_at(Position::group(2), Some(Error::App(7))).unwrap(); assert!(matches!( producer.finish_at(Position::group(2)), Err(Error::ProtocolViolation) )); let mut arrival = producer.subscribe(replay().with_range(0..)); let mut ordered = producer.subscribe(replay().with_range(0..)).ordered(); for key in 0..2 { assert_eq!(arrival.assert_group().start, key); assert_eq!(ordered.next_group().await.unwrap().unwrap().group().unwrap().start, key); } assert!(matches!(arrival.recv_group().await, Err(Error::App(7)))); assert!(matches!(ordered.next_group().await, Err(Error::App(7)))); assert!(matches!(producer.closed().await, Err(Error::App(7)))); let mut capped = producer.subscribe(replay().with_range(0..)).with_range(..1); assert_eq!(capped.assert_group().start, 0); assert!(capped.recv_group().await.unwrap().is_none()); // So does a range whose relative end excludes the last group. let mut relative = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(0), end: Some(Anchor::Relative(1)), }])); assert_eq!(relative.assert_group().start, 0); assert!(relative.recv_group().await.unwrap().is_none()); // Unlike an abort, the history stays: a later reader and a fetch still get it. producer.clone().abort(Error::Timeout).unwrap(); let mut late = producer.subscribe(replay().with_range(0..)); assert_eq!(late.assert_group().start, 0); assert_eq!(late.assert_group().start, 1); assert!(matches!(late.recv_group().await, Err(Error::App(7)))); producer.consume().fetch_group(1, None).await.unwrap(); } /// `fail` ends the track in error at its live edge: a reader gets every group below it, then /// the error, and the track takes no second end. #[kio::test] #[cfg(feature = "async")] async fn a_failed_track_serves_its_groups_then_the_error() { let producer = track_producer("test", Info::default().with_max_age(None)); for _ in 0..3 { producer.append_next().unwrap().finish().unwrap(); } let mut reader = producer.subscribe(replay().with_range(0..)).ordered(); let _finished = producer.clone().fail(Error::App(7)).unwrap(); for key in 0..3 { assert_eq!(reader.next_group().await.unwrap().unwrap().group().unwrap().start, key); } assert!(matches!(reader.next_group().await, Err(Error::App(7)))); assert!(matches!(producer.append_next(), Err(Error::Closed))); assert!(matches!(producer.clone().finish(), Err(Error::Closed))); assert!(matches!(producer.clone().fail(Error::App(7)), Err(Error::Closed))); } /// A start is a timestamp: a timed track converts it into its timescale, an untimed track /// takes the key as an unscaled one, and a start that doesn't match its track is refused /// before any group exists. #[test] fn an_append_starts_at_a_timestamp_matching_its_track() { let timed = track_producer("timed", Info::timed()); assert_eq!( timed.append_group(Timestamp::from_secs(1).unwrap()).unwrap().start, 1_000 ); assert!(matches!( timed.append_group(Timestamp::unscaled(2_000)), Err(Error::TimestampMismatch) )); assert_eq!(timed.largest().map(|largest| largest.group), Some(1_000)); let untimed = track_producer("untimed", Info::default()); assert_eq!(untimed.append_group(Timestamp::unscaled(5)).unwrap().start, 5); assert!(matches!( untimed.append_group(Timestamp::from_millis(6).unwrap()), Err(Error::TimestampMismatch) )); assert_eq!(untimed.largest().map(|largest| largest.group), Some(5)); untimed .append_group(Timestamp::unscaled(u64::MAX - 1)) .unwrap() .finish() .unwrap(); assert!(matches!( untimed.append_group(Timestamp::unscaled(u64::MAX)), Err(Error::BoundsExceeded(_)) )); } /// A group ended by `finish` reads whole but says nothing about where it ends; `finish_at` /// declares the end a media reader checks against the next group. #[kio::test] #[cfg(feature = "async")] async fn a_finished_group_declares_no_end() { let producer = track_producer("test", Info::timed()); let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(0).unwrap(), &b"a"[..]).unwrap(); open.write_frame(Timestamp::from_millis(10).unwrap(), &b"b"[..]) .unwrap(); open.finish().unwrap(); let mut ended = producer.append_group(Timestamp::from_millis(20).unwrap()).unwrap(); ended .write_frame(Timestamp::from_millis(20).unwrap(), &b"c"[..]) .unwrap(); ended.finish_at(30).unwrap(); let mut reader = producer.subscribe(replay().with_range(0..)); let mut first = reader.assert_group(); assert_eq!(first.read_frame().await.unwrap().unwrap().payload, &b"a"[..]); assert_eq!(first.read_frame().await.unwrap().unwrap().payload, &b"b"[..]); assert!(first.read_frame().await.unwrap().is_none(), "every frame is read"); assert_eq!(first.end(), None); assert_eq!(reader.assert_group().end(), Some(30)); } #[kio::test] #[cfg(feature = "async")] async fn readers_end_at_the_boundary_with_missing_lower_groups() { let mut producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(None); let mut ordered = producer.subscribe(None).ordered(); producer.finish_at(Position::group(3)).unwrap(); let _open = producer.create_group(group::Info { start: 2 }).unwrap(); // Missing lower groups do not hold either reader for the wire's grace. // The last group's own stream remains open independently of the track end. assert_eq!(arrival.assert_group().start, 2); // They are reported as one gap, since nothing was produced below the group. let Some(Item::Gap { range, .. }) = ordered.next_group().await.unwrap() else { panic!("the missing groups are reported") }; assert_eq!(range, 0..2); assert_eq!(ordered.next_group().await.unwrap().unwrap().group().unwrap().start, 2); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); assert!(ordered.next_group().now_or_never().unwrap().unwrap().is_none()); // Reaching the boundary did not settle the open group: an abort still wins. producer.abort(Error::Timeout).unwrap(); assert!(matches!( arrival.recv_group().now_or_never().unwrap(), Err(Error::Timeout) )); let next = ordered.next_group().now_or_never().unwrap(); assert!( matches!(next, Err(Error::Timeout)), "{:?}", next.map(|i| i.map(|i| match i { Item::Gap { range, error } => format!("{range:?} {error}"), _ => "group".into(), })) ); } /// A received copy's end holds readers at a hole below it, since a lower group's stream /// may arrive after a higher one, until the hole fills or its grace passes. The last /// producer going ends them regardless. #[kio::test] #[cfg(feature = "async")] async fn a_pending_tail_holds_readers_at_a_hole() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); // The wire declares where the feed starts. producer.start_at(0).unwrap(); producer.finish_at(Position::group(3)).unwrap(); let _high = producer.create_group(group::Info { start: 2 }).unwrap(); assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); // The late lower groups are delivered, then the full tail ends the reader. let _low = producer.create_group(group::Info { start: 0 }).unwrap(); assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().is_none(), "group 1 is still owed"); let _mid = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // The grace passing ends readers at the boundary, hole and all. let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); let _high = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().is_none()); // The subscription's max delay is the grace. elapse(&producer, replay().budget()); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // So does the last producer going. let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); let _high = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(arrival.assert_group().start, 1); drop(producer); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// The grace alone ends a reader parked at a hole, with no write to wake it: below an /// arrival it runs from that arrival, and above the last one from the declared end. #[kio::test] #[cfg(feature = "async")] async fn a_parked_reader_wakes_when_the_tail_grace_passes() { // The subscription's max delay is the grace. let grace = replay().budget(); // Hole 0 sits below arrival 1; holes 2..4 sit above the last arrival. for (end, arrived_late) in [(2, false), (4, true)] { let mut producer = track_producer("test", None); producer.received(); elapse(&producer, Duration::ZERO); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, 1); if arrived_late { // The end comes later than the arrival, so its gap runs from the end. elapse(&producer, grace / 2); } producer.finish_at(Position::group(end)).unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!( arrival.poll_recv_group(&waiter).is_pending(), "end {end}: held at the hole" ); if arrived_late { // Hole 0 folds a grace after the arrival: the reader wakes to judge it a miss, // which nothing fills, and stays held for the holes above. elapse(&producer, grace / 2); assert!(woken.swap(false, Ordering::SeqCst), "end {end}: the fold went unjudged"); assert!(arrival.poll_recv_group(&waiter).is_pending()); elapse(&producer, grace / 2 - Duration::from_millis(1)); } else { elapse(&producer, grace - Duration::from_millis(1)); } assert!(!woken.load(Ordering::SeqCst), "end {end}: woken before the grace"); assert!(arrival.poll_recv_group(&waiter).is_pending()); elapse(&producer, Duration::from_millis(1)); assert!( woken.load(Ordering::SeqCst), "end {end}: the grace never woke the reader" ); assert!(matches!(arrival.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); } } /// An unbounded max delay never gives up on a hole, below an arrival or above the last /// one, until it fills. #[test] #[cfg(feature = "async")] fn an_unbounded_tail_holds_readers_at_a_hole() { // Hole 0 sits below arrival 1; hole 1 sits above arrival 0. for (arrived, missing) in [(1, 0), (0, 1)] { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay().with_delay(Delay::default().with_max(None))); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); producer .create_group(group::Info { start: arrived }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, arrived); elapse(&producer, Duration::from_secs(3600)); assert!( arrival.recv_group().now_or_never().is_none(), "hole {missing}: given up on" ); producer .create_group(group::Info { start: missing }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, missing); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } } /// An end declared ahead of a feed still delivering does not cut it off: each arrival /// restarts the grace for the groups above it. A feed that stalls past it ends readers. #[test] #[cfg(feature = "async")] fn a_delivering_feed_holds_an_end_declared_ahead() { let mut producer = track_producer("test", None); producer.received(); let grace = replay().budget(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(5)).unwrap(); for key in 0..3 { elapse(&producer, grace / 2); producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, key); } elapse(&producer, grace - Duration::from_millis(1)); assert!(arrival.recv_group().now_or_never().is_none(), "still within the grace"); elapse(&producer, Duration::from_millis(1)); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// A start declared after the end asks the feed for groups it may only now send, so the /// tail above the last arrival waits a fresh grace from it, not what is left of the end's. #[test] #[cfg(feature = "async")] fn a_late_start_restarts_the_tail_grace() { let mut producer = track_producer("test", None); producer.received(); let grace = replay().budget(); let mut arrival = producer.subscribe(replay()); producer.finish_at(Position::group(2)).unwrap(); elapse(&producer, grace - Duration::from_millis(1)); producer.start_at(0).unwrap(); elapse(&producer, Duration::from_millis(1)); assert!(arrival.recv_group().now_or_never().is_none(), "a fresh grace"); elapse(&producer, grace - Duration::from_millis(1)); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// Only a received copy waits for its tail: a local publisher's end below a hole ends /// readers once the edge reaches it, as the publisher skipped the group. #[test] #[cfg(feature = "async")] fn a_local_end_does_not_wait_for_a_tail() { let mut producer = track_producer("test", None); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// A group the publisher dropped is not in flight, so it holds no reader at the end. #[test] #[cfg(feature = "async")] fn a_dropped_group_fills_the_tail() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3)).unwrap(); producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); producer.drop_groups(0..2, StreamError::NotFound); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// While the start is pending, everything below the end is owed, but only what has /// not arrived or been dropped: an end with nothing missing below it ends readers at once. #[test] #[cfg(feature = "async")] fn a_pending_start_owes_nothing_already_accounted_for() { // An end before the first group owes nothing. let mut producer = track_producer("test", None); producer.received(); producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); producer.finish_at(Position::group(0)).unwrap(); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // Nor does one whose groups below it all arrived or were dropped. let mut producer = track_producer("test", None); producer.received(); producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, 0); producer.drop_groups(1..2, StreamError::NotFound); producer.finish_at(Position::group(2)).unwrap(); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // A hole is still owed until the grace passes. let mut producer = track_producer("test", None); producer.received(); producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, 1); producer.finish_at(Position::group(2)).unwrap(); assert!(arrival.recv_group().now_or_never().is_none(), "group 0 is owed"); elapse(&producer, replay().budget()); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// A fetched copy of a group the live feed still owes never reaches an arrival-order /// reader, so it does not fill the hole: the reader waits for the live group. #[kio::test] #[cfg(feature = "async")] async fn a_fetched_group_does_not_fill_a_pending_tail() { let mut producer = track_producer("test", None); producer.received(); let dynamic = producer.requests(); let consumer = producer.consume(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); let _high = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(arrival.assert_group().start, 1); let _pending = consumer.fetch_group(0, None); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); req.accept(None).unwrap().expect("a fresh group").finish().unwrap(); assert!( arrival.recv_group().now_or_never().is_none(), "the live group 0 is still owed" ); } /// An abort ends the wait for a pending tail. A group the tail still owed was cut off, /// so readers get the abort rather than a clean end without it. #[kio::test] #[cfg(feature = "async")] async fn an_abort_errors_a_pending_tail_short_of_a_group() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); let high = producer.create_group(group::Info { start: 1 }).unwrap(); high.finish().unwrap(); assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); producer.abort(Error::Dropped).unwrap(); assert!(matches!( arrival.recv_group().now_or_never().unwrap(), Err(Error::Dropped) )); } /// A reader that got every group ends cleanly, even when the track aborts before the /// reader polls past the last one. #[kio::test] #[cfg(feature = "async")] async fn an_abort_after_a_full_tail_ends_cleanly() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2)).unwrap(); for key in [1, 0] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } producer.clone().abort(Error::Dropped).unwrap(); assert_eq!(arrival.assert_group().start, 1); assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// A gap the grace already gave up on was lost before the abort, not cut off by it, so /// readers end cleanly. Within the grace, the same abort errors them. #[kio::test] #[cfg(feature = "async")] async fn an_abort_past_the_grace_ends_a_pending_tail_cleanly() { for (waited, ended) in [(ledger::GRACE / 2, false), (ledger::GRACE, true)] { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3)).unwrap(); for key in [0, 2] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, key); } assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); elapse(&producer, waited); producer.abort(Error::Dropped).unwrap(); let res = arrival.recv_group().now_or_never().unwrap(); assert_eq!(matches!(res, Ok(None)), ended, "waited {waited:?}"); } } /// An evicted group still arrived, so the pending tail counts it: readers end at the /// declared end rather than wait for the session to settle the tail. #[test] #[cfg(feature = "async")] fn an_evicted_group_still_fills_a_pending_tail() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3)).unwrap(); for key in [0, 1] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); assert_eq!(arrival.assert_group().start, key); } elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); assert!(!producer.state.read().lookup.contains_key(&0), "group 0 was evicted"); assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// The wire's end can land after the highest group, with lower groups still owed. A /// reader already past that group stays held at the hole: the end never shows without /// its pending tail. #[kio::test] #[cfg(feature = "async")] async fn an_end_after_the_highest_group_holds_the_reader() { let mut producer = track_producer("test", None); producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); let _high = producer.create_group(group::Info { start: 2 }).unwrap(); assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none()); producer.finish_at(Position::group(3)).unwrap(); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); assert!( matches!(producer.finish_at(Position::group(2)), Err(Error::ProtocolViolation)), "a second end is refused" ); assert_eq!(producer.finished(), Some(Position::group(3)), "the first end stands"); let mut _low = Vec::new(); for key in [0, 1] { _low.push(producer.create_group(group::Info { start: key }).unwrap()); assert_eq!(arrival.assert_group().start, key); } assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } /// An abort before the declared end settled wins over it: the boundary was reached, /// but a group below it was still open, so the track was cut off rather than ended. #[test] #[cfg(feature = "async")] fn abort_before_the_end_settles_wins() { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); let head = producer.create_group(group::Info { start: 0 }).unwrap(); head.finish().unwrap(); let _tail = producer.create_group(group::Info { start: 1 }).unwrap(); producer.finish_at(Position::group(2)).unwrap(); assert_eq!(consumer.assert_group().start, 0); producer.abort(Error::Timeout).unwrap(); let res = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Err(Error::Timeout))); } /// A local session closing is a dead route to the origin, not finished content, so a /// replacement route can take over where it left off. #[test] fn session_close_is_not_completion_and_keeps_the_resume_position() { let producer = track_producer("local", Info::timed()); let consumer = producer.consume(); producer.append_next().unwrap().finish().unwrap(); let mut group = producer.append_next().unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"a")) .unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"b")) .unwrap(); producer .abort(Error::Stream(StreamError::Session(crate::SessionError::Cancel))) .unwrap(); assert!(matches!( consumer.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(_)) )); assert_eq!(consumer.resume_position(), Some(Position { group: 1, frame: 2 })); } /// `closed` settles once the declared end is reached, not when it is declared, and an /// abort after that keeps the clean outcome. #[test] #[cfg(feature = "async")] fn closed_settles_on_a_reached_end() { let waiter = kio::Waiter::noop(); let mut producer = track_producer("test", Info::timed()); let demand = producer.demand(); producer.append_next().unwrap().finish().unwrap(); producer.finish_at(Position::group(2)).unwrap(); assert!(producer.poll_closed(&waiter).is_pending(), "group 1 is still to come"); assert!(demand.closed().now_or_never().is_none()); producer.append_next().unwrap().finish().unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!(matches!(demand.closed().now_or_never(), Some(Ok(())))); assert!( producer.poll_aborted(&waiter).is_pending(), "lower groups may still arrive" ); producer.clone().abort(Error::Cancel).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!(matches!(demand.closed().now_or_never(), Some(Ok(())))); } /// A group still open below a reached end keeps the outcome pending: it finishing settles /// the end and wakes the wait, while an abort before then cuts the track off. #[test] #[cfg(feature = "async")] fn closed_waits_for_open_groups_below_the_end() { use std::sync::atomic::{AtomicBool, Ordering}; let mut producer = track_producer("test", Info::timed()); let open = producer.append_next().unwrap(); producer.finish_at(Position::group(1)).unwrap(); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_closed(&waiter).is_pending(), "group 0 may still abort"); open.finish().unwrap(); assert!(woken.load(Ordering::SeqCst), "the group finishing wakes the wait"); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); let mut cut = track_producer("cut", Info::timed()); let _open = cut.append_next().unwrap(); cut.finish_at(Position::group(1)).unwrap(); assert!(cut.poll_closed(&kio::Waiter::noop()).is_pending()); cut.clone().abort(Error::Cancel).unwrap(); assert!(matches!( cut.poll_closed(&kio::Waiter::noop()), Poll::Ready(Err(Error::Cancel)) )); } /// A received end whose tail grace passed is final once `closed()` reports it: a later /// subscription with a longer grace, and an abort within it, leave the clean outcome. #[test] fn a_reported_end_survives_a_longer_grace() { let waiter = kio::Waiter::noop(); let mut producer = track_producer("test", None); producer.received(); let _short = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3)).unwrap(); for key in [0, 2] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } assert!(producer.poll_closed(&waiter).is_pending(), "held at the hole"); elapse(&producer, ledger::GRACE); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); let _long = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(ledger::GRACE * 10))); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); producer.clone().abort(Error::Cancel).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); } /// A settled end is final: a late group below it and a later abort leave the outcome /// `closed()` already saw. #[test] #[cfg(feature = "async")] fn a_settled_end_survives_a_late_group_and_an_abort() { let waiter = kio::Waiter::noop(); let mut producer = track_producer("test", Info::timed()); let demand = producer.demand(); producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); producer.finish_at(Position::group(6)).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); let _late = producer.create_group(group::Info { start: 4 }).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); producer.clone().abort(Error::Cancel).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!(matches!(demand.closed().now_or_never(), Some(Ok(())))); } /// An abort short of the end, or the last producer leaving without one, is the outcome. #[test] #[cfg(feature = "async")] fn closed_reports_an_unfinished_end() { let waiter = kio::Waiter::noop(); let aborted = track_producer("aborted", Info::timed()); aborted.clone().abort(Error::Timeout).unwrap(); assert!(matches!(aborted.poll_closed(&waiter), Poll::Ready(Err(Error::Timeout)))); let dropped = track_producer("dropped", Info::timed()); let demand = dropped.demand(); drop(dropped); assert!(matches!(demand.closed().now_or_never(), Some(Err(Error::Dropped)))); } /// An abort short of the end keeps the groups that finished for a reader that has not /// pulled them yet: it gets them, then the abort. The open group nobody will finish /// is gone, on both cursors. #[test] #[cfg(feature = "async")] fn abort_keeps_finished_groups_for_a_slow_reader() { let producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); for key in 0..2 { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let _open = producer.create_group(group::Info { start: 2 }).unwrap(); producer.abort(Error::Timeout).unwrap(); assert_eq!(arrival.assert_group().start, 0); assert_eq!(arrival.assert_group().start, 1); let res = arrival.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Err(Error::Timeout)), "expected the abort"); assert_eq!(drain_ordered(&mut ordered), [0, 1]); let res = ordered .next_group() .now_or_never() .expect("should not block") .map(|item| item.map(|item| item.group().expect("a group").start)); assert!(matches!(res, Err(Error::Timeout)), "expected the abort, got {res:?}"); } /// The last producer dropping without a finish keeps the finished groups the same way. #[test] #[cfg(feature = "async")] fn dropped_producer_keeps_finished_groups_for_a_slow_reader() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); drop(producer); assert_eq!(consumer.assert_group().start, 0); let res = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Err(Error::Dropped)), "expected the drop"); } /// A closed track's groups, its latest included, expire once idle, so a stale /// consumer cannot pin them. A live track's latest stays protected. #[test] fn closed_track_expires_its_latest_group() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let live = track_producer_pooled("live", pool.clone()); live.append_next().unwrap().finish().unwrap(); let aborted = track_producer_pooled("aborted", pool.clone()); aborted.append_next().unwrap().finish().unwrap(); let stale_aborted = aborted.consume(); aborted.abort(Error::Timeout).unwrap(); let finished = track_producer_pooled("finished", pool.clone()); finished.append_next().unwrap().finish().unwrap(); let finished = finished.finish().unwrap(); let stale_finished = finished.consume(); drop(finished); // The first pass dates the activity it has not seen yet; the next one expires it. for _ in 0..2 { pool.step(Duration::from_secs(2)); pool.sweep(); } assert!(live.consume().peek_group(0).is_some(), "a live track keeps its latest"); assert!( stale_aborted.peek_group(0).is_none(), "an aborted track's latest expired" ); assert!( stale_finished.peek_group(0).is_none(), "a sealed track's latest expired" ); } /// A cut on an ended track expires like the group it was: a stale consumer's reader /// gets it within the window, and nothing once the sweep retires it. #[test] #[cfg(feature = "async")] fn closed_track_retires_an_expired_cut() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let producer = track_producer_pooled("cut", pool.clone()); let mut group = producer.append_next().unwrap(); group.write_frame(group.key_time(), &b"a"[..]).unwrap(); group.abort(Error::Stream(StreamError::App(5))).unwrap(); let producer = producer.finish().unwrap(); let stale = producer.consume(); drop(producer); let mut reader = stale.subscribe(replay()).now_or_never().unwrap().unwrap(); assert_eq!( reader.assert_group().start, 0, "the cut is handed out within the window" ); // The first pass dates the activity it has not seen yet; the next one expires it. for _ in 0..2 { pool.step(Duration::from_secs(2)); pool.sweep(); } let mut late = stale.subscribe(replay()).now_or_never().unwrap().unwrap(); let res = late.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Ok(None)), "the expired cut is gone"); } /// An abort after every group below the declared end finished leaves the end standing. #[test] #[cfg(feature = "async")] fn abort_after_the_end_settles_ends_clean() { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); for key in 0..2 { let group = producer.create_group(group::Info { start: key }).unwrap(); group.finish().unwrap(); } producer.finish_at(Position::group(2)).unwrap(); assert_eq!(consumer.assert_group().start, 0); assert_eq!(consumer.assert_group().start, 1); producer.abort(Error::Timeout).unwrap(); let res = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Ok(None))); } /// The settled end stands on the ordered cursor too: a reader that starts after the /// abort gets every group below the end, then the clean end, not the abort. #[test] #[cfg(feature = "async")] fn abort_after_the_end_settles_ends_clean_for_an_ordered_reader() { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); for key in 0..2 { let group = producer.create_group(group::Info { start: key }).unwrap(); group.finish().unwrap(); } producer.finish_at(Position::group(2)).unwrap(); producer.abort(Error::Timeout).unwrap(); assert_eq!(drain_ordered(&mut consumer), [0, 1]); let end = consumer .next_group() .now_or_never() .expect("should not block") .map(|item| item.map(|item| item.group().expect("a group").start)); assert!(matches!(end, Ok(None)), "expected the clean end, got {end:?}"); } /// The settled end stands on the ordered cursor even where it has nothing to read: /// capped below the end, and with a gap below the cap. Parking is not an option on a /// closed track (the channel would turn it into the abort), so these end clean. #[test] #[cfg(feature = "async")] fn abort_after_the_end_settles_ends_clean_below_the_cap() { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None).ordered(); for key in 0..2 { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } producer.finish_at(Position::group(2)).unwrap(); producer.abort(Error::Timeout).unwrap(); consumer.set_range(..1); assert_eq!(drain_ordered(&mut consumer), [0]); let end = consumer .next_group() .now_or_never() .expect("should not block") .map(|item| item.map(|item| item.group().expect("a group").start)); assert!( matches!(end, Ok(None)), "capped at the end: expected the clean end, got {end:?}" ); let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None).ordered(); for key in [0, 2] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } producer.finish_at(Position::group(3)).unwrap(); producer.abort(Error::Timeout).unwrap(); consumer.set_range(..2); assert_eq!(drain_ordered(&mut consumer), [0]); let end = consumer .next_group() .now_or_never() .expect("should not block") .map(|item| item.map(|item| item.group().expect("a group").start)); assert!( matches!(end, Ok(None)), "gap below the cap: expected the clean end, got {end:?}" ); } #[test] #[cfg(feature = "async")] fn recv_group_finishes_without_waiting_for_gaps() { let producer = track_producer("test", Info::timed()); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); let producer = producer.finish().unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().start, 1); let done = consumer .recv_group() .now_or_never() .expect("should not block") .expect("would have errored"); assert!(done.is_none(), "track should finish without waiting for gaps"); } /// A received copy, as a session's subscription feeds it, on a pinned wall clock. #[cfg(feature = "async")] fn received_producer(info: impl Into>) -> Producer { let producer = track_producer("test", info); producer.received(); elapse(&producer, Duration::ZERO); producer } /// A reader on a received copy that fails waits out a re-offer still held for its group, /// then continues on it from its frame with nothing repeated or missed, even while another /// duplicate waits on that re-offer. A released hold leaves the failure, as does a re-offer /// starting past the reader's frame. #[kio::test] #[cfg(feature = "async")] async fn a_failed_copy_continues_on_its_held_reoffer() { for case in ["reoffer", "released", "past"] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{case}"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; let Ok(Received::Held(second)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("a second duplicate waits too"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!( reader.poll_read_frame(&waiter).is_pending(), "{case}: waits on the held re-offer" ); if case == "released" { drop((hold, second)); assert!(reader.read_frame().await.is_err(), "{case}"); continue; } let start = if case == "past" { 2 } else { 0 }; let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the re-offer takes the group"); }; drop(hold); assert!( reader.poll_read_frame(&waiter).is_pending(), "{case}: the second still holds" ); // The second duplicate now waits on the re-offer, which holds nobody reading the // failed copy. let Ok(Received::Held(third)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the second duplicate waits on the re-offer still arriving"); }; drop(second); if case == "past" { assert!(reader.read_frame().await.is_err(), "{case}: frame 1 is in no copy"); continue; } // Parked on frame 1, which the re-offer has yet to write. assert!(reader.poll_read_frame(&waiter).is_pending()); again.write_frame(at(300), &b"a"[..]).unwrap(); again.write_frame(at(301), &b"b"[..]).unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); again.finish().unwrap(); drop(third); assert!(reader.read_frame().await.unwrap().is_none()); } } /// A failed copy's end follows its frame reads: it waits out a held re-offer rather than /// report the failure, so a publisher watching it keeps the stream the re-offer continues. #[kio::test] #[cfg(feature = "async")] async fn a_failed_copy_finishes_on_its_held_reoffer() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "waits on the held re-offer"); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; drop(hold); again.write_frame(at(300), &b"a"[..]).unwrap(); again.write_frame(at(301), &b"b"[..]).unwrap(); assert!(reader.poll_finished(&waiter).is_pending(), "the re-offer is still open"); again.finish().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); } /// A re-offer that fails while still sending frames the reader took from an earlier copy /// hands over to the next held re-offer: only a frame the reader took from the failing copy /// itself is torn. One it was streaming from that copy is lost with it. #[kio::test] #[cfg(feature = "async")] async fn a_reoffer_failing_in_its_replay_hands_over_again() { for torn in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{torn}"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; let Ok(Received::Held(next)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("another re-offer waits on the first"); }; drop(hold); let waiter = kio::Waiter::noop(); let handle = again.handle(); let streaming = if torn { // The reader streams part of frame 1 from the re-offer before it fails. again.write_frame(at(300), &b"a"[..]).unwrap(); let mut frame = again .create_frame(frame::Info { size: 2, timestamp: at(301), }) .unwrap(); frame.write(&b"b"[..]).unwrap(); let mut part = reader.next_frame().await.unwrap().expect("frame 1 streams"); assert_eq!(&part.read_chunk().await.unwrap().unwrap()[..], b"b"); frame } else { // The re-offer fails partway through frame 0, which the reader already holds. let mut frame = again .create_frame(frame::Info { size: 2, timestamp: at(300), }) .unwrap(); frame.write(&b"a"[..]).unwrap(); assert!(reader.poll_read_frame(&waiter).is_pending()); frame }; handle.abort(Error::Cancel).unwrap(); drop(streaming); if torn { drop(next); assert!(reader.read_frame().await.is_err(), "frame 1 was handed out partly"); continue; } assert!( reader.poll_read_frame(&waiter).is_pending(), "waits on the next re-offer" ); let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the next re-offer takes the group"); }; drop(next); last.write_frame(at(300), &b"a"[..]).unwrap(); last.write_frame(at(301), &b"b"[..]).unwrap(); last.finish().unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert!(reader.read_frame().await.unwrap().is_none()); } } /// A reader whose end was judged on a re-offer past its prefetched frames keeps waiting /// when that re-offer fails while another is held for it, then reads its prefetched frame /// and continues on whichever copy holds the rest. #[kio::test] #[cfg(feature = "async")] async fn a_failed_copy_finishes_on_a_reoffer_of_a_failed_reoffer() { for again_fails in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = first.consume(); // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("the re-offer takes the group"); }; let Ok(Received::Held(next)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("another re-offer waits on the first"); }; drop(hold); let waiter = kio::Waiter::noop(); assert!( reader.poll_finished(&waiter).is_pending(), "{again_fails}: the re-offer is open" ); if !again_fails { drop(next); again.write_frame(at(302), &b"c"[..]).unwrap(); again.finish().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); assert!(reader.read_frame().await.unwrap().is_none()); continue; } again.abort(Error::Cancel).unwrap(); let res = reader.poll_finished(&waiter); assert!(res.is_pending(), "waits on the next re-offer: {res:?}"); let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the next re-offer takes the group"); }; drop(next); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { last.write_frame(at(millis), &payload[..]).unwrap(); } last.finish().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); assert!(reader.read_frame().await.unwrap().is_none()); } } /// A failed copy's end agrees with its frame reads when the reader still has frames it /// prefetched from it: a re-offer starting past the reader's frame but not past those /// continues it, and one starting past both does not. #[kio::test] #[cfg(feature = "async")] async fn a_failed_copy_finishes_past_its_prefetched_frames() { for start in [2, 3] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = first.consume(); // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "{start}: waits on the hold"); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the re-offer takes the group"); }; drop(hold); if start == 3 { assert!(reader.finished().await.is_err(), "frame 2 is in no copy"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert!(reader.read_frame().await.is_err(), "nor can a read find it"); continue; } again.write_frame(at(302), &b"c"[..]).unwrap(); let res = reader.poll_finished(&waiter); assert!(res.is_pending(), "the re-offer is still open: {res:?}"); // The end check keeps reading the re-offer, so a fetch would not abandon it. assert!( again.handle().poll_unwanted(&waiter).is_pending(), "the re-offer stays wanted" ); again.finish().unwrap(); // From where the reader stands, with frame 1 still to read. assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); assert!(reader.read_frame().await.unwrap().is_none()); } } /// A reader whose failed copy's end a re-offer answers past its prefetched frames reached /// the group's end, so it no longer keeps the track wanted. #[kio::test] #[cfg(feature = "async")] async fn a_failed_copy_finished_on_a_reoffer_releases_its_demand() { let producer = received_producer(Info::timed()); let mut subscriber = producer.subscribe(None); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = subscriber.assert_group(); drop(subscriber); // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("the re-offer takes the group"); }; drop(hold); again.write_frame(at(302), &b"c"[..]).unwrap(); again.finish().unwrap(); assert_eq!(producer.demand().wanted_start(), Some(0), "not read to its end yet"); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(producer.demand().wanted_start(), None, "the re-offer answered the end"); } /// A capped reader continues on a copy cut short at the back no earlier than its cap, which /// holds every frame it still reads. #[kio::test] #[cfg(feature = "async")] async fn a_capped_reader_continues_on_a_copy_cut_at_its_cap() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); // Received from frame 1, so a fetch of the group's head can sit beside it. let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 1) else { panic!("the first copy opens"); }; first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = first.consume().with_range(..3); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); let mut head = producer .state .write() .ok() .unwrap() .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) .unwrap() .expect("a fresh group"); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } head.finish().unwrap(); first.abort(Error::Cancel).unwrap(); // An uncapped reader needs frames past the head, which no copy holds. assert!(reader.clone().with_end(None).read_frame().await.is_err()); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); assert!(reader.read_frame().await.unwrap().is_none()); } /// A reader that raises its cap past a copy's cut before reaching it learns from its end /// what its frame reads will: the frames past the cut are missing, whether the ones before /// it are still in the copy or already prefetched. #[kio::test] #[cfg(feature = "async")] async fn a_raised_cap_before_a_cut_ends_in_its_error() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let mut head = producer .state .write() .ok() .unwrap() .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) .unwrap() .expect("a fresh group"); let mut reader = head.consume().with_range(..3); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } head.finish().unwrap(); // Takes frames 1 and 2 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let mut reader = reader.with_end(None); // A clone shares the cursor but not the prefetch, so frames 1 and 2 are the copy's. let mut unfetched = reader.clone(); let res = unfetched.finished().await; assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); let res = reader.finished().await; assert!(res.is_err(), "nor past the prefetch: {res:?}"); for payload in [b"b", b"c"] { assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], payload); } assert!(reader.read_frame().await.is_err(), "the reads agree"); } /// An end check waiting on a copy past the reader's prefetch follows the reader's cap: once /// the cap is lifted, a successor cut at the old cap no longer ends the group cleanly. #[kio::test] #[cfg(feature = "async")] async fn a_retained_end_check_follows_a_lifted_cap() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = first.consume().with_range(..3); // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let mut cut = producer .state .write() .ok() .unwrap() .insert_group_request(3, group::Bounds { start: 2, end: Some(3) }, None) .unwrap() .expect("a fresh group"); drop(hold); cut.write_frame(at(302), &b"c"[..]).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "the cut copy is still open"); let mut reader = reader.with_range(..); cut.finish().unwrap(); let res = reader.finished().await; assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); } /// A reader that skipped ahead of the frames its copy committed tore none of them, so it /// continues on a held re-offer from where it skipped to. #[kio::test] #[cfg(feature = "async")] async fn a_skipped_reader_continues_on_a_held_reoffer() { for chunked in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; let mut reader = first.consume(); if chunked { // Frame 0 streams to the reader while still being written, then commits. let mut frame = first .create_frame(frame::Info { size: 1, timestamp: at(300), }) .unwrap(); let mut part = reader.next_frame().await.unwrap().expect("frame 0 streams"); frame.write(&b"a"[..]).unwrap(); assert_eq!(&part.read_chunk().await.unwrap().unwrap()[..], b"a"); first = frame.finish().unwrap(); } else { first.write_frame(at(300), &b"a"[..]).unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); } reader.skip_to(3); let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!( reader.poll_read_frame(&waiter).is_pending(), "waits on the held re-offer" ); let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; drop(hold); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { again.write_frame(at(millis), &payload[..]).unwrap(); } again.finish().unwrap(); assert_eq!( &reader.read_frame().await.unwrap().unwrap().payload[..], b"d", "chunked: {chunked}" ); assert!(reader.read_frame().await.unwrap().is_none()); } } /// A capped reader that continued on a copy cut at its cap and then raises the cap reads /// no clean end at the cut: the frames past it are missing from that copy, so it continues /// on a copy that holds them, or fails without one. #[kio::test] #[cfg(feature = "async")] async fn a_raised_cap_past_an_adopted_cut_finds_the_rest() { for reoffer in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 1) else { panic!("the first copy opens"); }; first.write_frame(at(301), &b"b"[..]).unwrap(); let mut reader = first.consume().with_range(..3); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); let mut head = producer .state .write() .ok() .unwrap() .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) .unwrap() .expect("a fresh group"); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } head.finish().unwrap(); first.abort(Error::Cancel).unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); let mut reader = reader.with_end(None); let mut finished = reader.clone(); if !reoffer { let res = reader.read_frame().await; assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); let res = finished.finished().await; assert!(res.is_err(), "nor is the group's end: {res:?}"); continue; } let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { again.write_frame(at(millis), &payload[..]).unwrap(); } again.finish().unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"d"); assert!(reader.read_frame().await.unwrap().is_none()); // Its end, from where that reader stood, on the copy holding the rest. assert_eq!(finished.finished().await.unwrap(), 3); } } /// A reader waiting out a held re-offer is stalled, so its budget still gives up on it. #[kio::test] #[cfg(feature = "async")] async fn waiting_on_a_held_reoffer_expires() { for finished in [false, true] { let producer = received_producer(Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let at = |millis| Timestamp::from_millis(millis).unwrap(); let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(0), &b"a"[..]).unwrap(); let mut reader = subscriber.recv_group().await.unwrap().expect("group 3"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let Ok(Received::Held(_hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_read_frame(&waiter).is_pending(), "{finished}"); // Group 4 bounds group 3's reach at 1s, and group 5 puts the edge a second past it. let mut later = Vec::new(); for (start, millis) in [(4, 1000), (5, 2000)] { let Ok(Received::Group(mut next)) = producer.create_group_at(group::Info { start }, 0) else { panic!("group {start} opens"); }; next.write_frame(at(millis), &b"x"[..]).unwrap(); later.push(next); } let res = match finished { true => reader.finished().await.map(|_| ()), false => reader.read_frame().await.map(|_| ()), }; assert!(matches!(res, Err(Error::Old)), "{finished}: {res:?}"); } } /// Create and finish group `key`, with one frame at `millis` on a timed track. #[cfg(feature = "async")] fn put(producer: &Producer, key: u64, millis: Option) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); if let Some(millis) = millis { group .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) .unwrap(); } group.finish().unwrap(); } /// A received group reordered behind its successor is still delivered in order: the /// subscription owes it, so the reader waits for it within its budget. #[test] #[cfg(feature = "async")] fn next_group_waits_for_an_owed_group() { let producer = received_producer(None); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [0], "group 1 is still owed"); put(&producer, 1, None); assert_eq!(drain_ordered(&mut consumer), [1, 2]); } /// A local track owes nothing: a gap costs nothing, as before. #[test] #[cfg(feature = "async")] fn next_group_skips_a_local_gap() { let producer = track_producer("test", None); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [0, 2]); } /// An untimed track with groups 0 and 2..10 keeps delivering: the wall clock gives up /// on group 1 a budget after its successor arrived, waking the parked reader exactly then, /// and nothing after waits on it again. #[test] #[cfg(feature = "async")] fn next_group_passes_an_owed_group_at_the_budget() { let budget = Duration::from_millis(50); let producer = received_producer(None); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [0]); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(consumer.poll_next_group(&waiter).is_pending()); elapse(&producer, budget - Duration::from_millis(1)); assert!(!woken.load(Ordering::SeqCst), "woken before the budget"); assert!(consumer.poll_next_group(&waiter).is_pending()); elapse(&producer, Duration::from_millis(1)); assert!(woken.load(Ordering::SeqCst), "the budget never woke the reader"); assert_eq!(drain_ordered(&mut consumer), [2]); for key in 3..10 { put(&producer, key, None); assert_eq!(drain_ordered(&mut consumer), [key]); } } /// On a timed track, media time gives up on an owed group too: once the edge is a full /// budget past where its successor starts, and not a frame before. #[test] #[cfg(feature = "async")] fn next_group_passes_an_owed_group_past_the_media_deadline() { let producer = received_producer(Info::timed()); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(2)))) .ordered(); put(&producer, 5, Some(5000)); assert_eq!(drain_ordered(&mut consumer), [5]); let mut later = producer.create_group(group::Info { start: 7 }).unwrap(); later .write_frame(Timestamp::from_millis(7000).unwrap(), &b"start"[..]) .unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(consumer.poll_next_group(&waiter).is_pending(), "group 6 is still owed"); woken.store(false, Ordering::SeqCst); later .write_frame(Timestamp::from_millis(8999).unwrap(), &b"within"[..]) .unwrap(); assert!(consumer.poll_next_group(&waiter).is_pending()); woken.store(false, Ordering::SeqCst); later .write_frame(Timestamp::from_millis(9000).unwrap(), &b"deadline"[..]) .unwrap(); assert!(woken.load(Ordering::SeqCst), "the deadline never woke the reader"); assert_eq!(drain_ordered(&mut consumer), [7]); } /// A group landing between the owed one and the candidate wakes the reader: it can /// convict the owed group sooner, and is then read without waiting. #[test] #[cfg(feature = "async")] fn next_group_rejudges_an_owed_group_when_a_lower_successor_lands() { let producer = received_producer(Info::timed()); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))) .ordered(); put(&producer, 0, Some(0)); assert_eq!(drain_ordered(&mut consumer), [0]); let mut edge = producer.create_group(group::Info { start: 4 }).unwrap(); edge.write_frame(Timestamp::from_millis(4000).unwrap(), &b"edge"[..]) .unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(consumer.poll_next_group(&waiter).is_pending(), "group 1 is still owed"); // Group 1 reaches only to group 2's start, a full budget behind the edge. put(&producer, 2, Some(2000)); assert!(woken.load(Ordering::SeqCst), "the lower successor went unjudged"); assert_eq!(drain_ordered(&mut consumer), [2], "group 3 is still owed"); } /// The arrival ledger giving up on a gap wakes the reader, even when the owed group's /// successor now is a later arrival whose own budget runs longer. #[test] #[cfg(feature = "async")] fn next_group_passes_an_owed_group_when_its_gap_folds() { let budget = Duration::from_millis(50); let producer = received_producer(None); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))) .ordered(); put(&producer, 0, None); assert_eq!(drain_ordered(&mut consumer), [0]); let opener = producer.create_group(group::Info { start: 2 }).unwrap(); elapse(&producer, Duration::from_millis(40)); put(&producer, 3, None); opener.abort(Error::Cancel).unwrap(); producer.modify().unwrap().reclaim_aborted(2); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(consumer.poll_next_group(&waiter).is_pending(), "group 1 is still owed"); elapse(&producer, Duration::from_millis(9)); assert!(!woken.load(Ordering::SeqCst), "woken before the gap folds"); elapse(&producer, Duration::from_millis(1)); assert!(woken.load(Ordering::SeqCst), "the fold never woke the reader"); assert_eq!(drain_ordered(&mut consumer), [3]); } /// What the track knows will never land is passed at once: a drop, an aborted group /// whose slot was reclaimed, or a group from below where the feed starts. #[test] #[cfg(feature = "async")] fn next_group_passes_what_is_not_owed() { let subscription = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(30))); let producer = received_producer(None); let mut consumer = producer.subscribe(subscription.clone()).ordered(); put(&producer, 0, None); assert_eq!(drain_ordered(&mut consumer), [0]); producer.drop_groups(1..4, StreamError::Old); put(&producer, 4, None); assert_eq!(drain_ordered(&mut consumer), [4], "groups 1 to 3 were dropped"); let aborted = producer.create_group(group::Info { start: 5 }).unwrap(); aborted.abort(Error::Cancel).unwrap(); producer.modify().unwrap().reclaim_aborted(5); assert!(!producer.state.read().lookup.contains_key(&5), "the slot was reclaimed"); put(&producer, 6, None); assert_eq!(drain_ordered(&mut consumer), [6], "group 5 was aborted and reclaimed"); // Below the first arrival, a feed with no declared start owes nothing. let producer = received_producer(None); let mut consumer = producer.subscribe(subscription.clone()).ordered(); put(&producer, 1, None); assert_eq!(drain_ordered(&mut consumer), [1]); // A declared start owes everything from it. let mut producer = received_producer(None); let mut consumer = producer.subscribe(subscription).ordered(); producer.start_at(0).unwrap(); put(&producer, 1, None); assert!( drain_ordered(&mut consumer).is_empty(), "group 0 is owed from the start" ); put(&producer, 0, None); assert_eq!(drain_ordered(&mut consumer), [0, 1]); } /// A group the track gives up on comes back as a gap, in key order among the groups, with /// why; a group already returned is no gap, nor is a key the publisher skipped. #[test] #[cfg(feature = "async")] fn an_ordered_read_reports_what_it_gives_up() { let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 0, None); put(&producer, 3, None); producer.drop_groups(1..3, StreamError::DeliveryTimeout); producer.skip_groups(4..6); put(&producer, 6, None); assert_eq!( items(&mut ordered), [ "group 0", "gap 1..3 (delivery timeout)", "group 3", "group 6", "pending" ], "a skip is no gap" ); // A group handed out and then cut is read as the failure it is, never reported twice. let cut = producer.create_group(group::Info { start: 7 }).unwrap(); let Poll::Ready(Ok(Some(Item::Group(read)))) = ordered.poll_next_group(&kio::Waiter::noop()) else { panic!("group 7 is ready") }; assert_eq!(read.start, 7); cut.abort(Error::Cancel).unwrap(); assert_eq!(items(&mut ordered), ["pending"]); } /// A gap waits behind a group the read still owes below it, and goes out once nothing is /// owed, or the end comes: every gap is returned before the track ends. #[test] #[cfg(feature = "async")] fn a_gap_waits_behind_what_is_owed_below_it() { let producer = received_producer(None); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(None))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); producer.drop_groups(4..5, StreamError::DeliveryTimeout); assert_eq!( items(&mut ordered), ["group 0", "pending"], "group 1 is owed below the gap" ); put(&producer, 1, None); assert_eq!( items(&mut ordered), ["group 1", "group 2", "pending"], "group 3 is owed below the gap" ); put(&producer, 3, None); assert_eq!( items(&mut ordered), ["group 3", "gap 4..5 (delivery timeout)", "pending"] ); let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 0, None); producer.drop_groups(1..2, StreamError::DeliveryTimeout); let _producer = producer.finish().unwrap(); assert_eq!( items(&mut ordered), ["group 0", "gap 1..2 (delivery timeout)"], "the end is clean after every gap" ); } /// A gap returned is told once: a later drop of the same keys, even a final one that would /// replace a stale report on the wire, is no second gap. #[test] #[cfg(feature = "async")] fn a_gap_is_returned_once() { let producer = received_producer(None); let budget = Duration::from_millis(50); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(items(&mut ordered), ["group 0", "pending"]); // The budget gives up on the group still owed, as stale. elapse(&producer, budget); assert_eq!(items(&mut ordered), ["gap 1..2 (old)", "group 2", "pending"]); producer.drop_groups(1..2, StreamError::DeliveryTimeout); put(&producer, 3, None); assert_eq!(items(&mut ordered), ["group 3", "pending"]); } /// A skip replaces a drop still waiting for the groups below it, so the keys it names are /// no gap, and the part of a drop it leaves still is. #[test] #[cfg(feature = "async")] fn a_skip_replaces_a_buffered_gap() { let producer = received_producer(None); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(None))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); producer.drop_groups(4..8, StreamError::DeliveryTimeout); assert_eq!(items(&mut ordered), ["group 0", "pending"]); producer.skip_groups(5..7); producer.drop_groups(1..2, StreamError::DeliveryTimeout); assert_eq!( items(&mut ordered), ["gap 1..2 (delivery timeout)", "group 2", "pending"] ); put(&producer, 3, None); assert_eq!( items(&mut ordered), [ "group 3", "gap 4..5 (delivery timeout)", "gap 7..8 (delivery timeout)", "pending" ] ); } /// A group landing after the read passed its key is given up as old, since the read never /// goes back; the group it was passed for is no gap. #[test] #[cfg(feature = "async")] fn a_cached_group_the_seek_passes_as_stale_is_a_gap() { let producer = received_producer(None); put(&producer, 0, None); put(&producer, 1, None); elapse(&producer, Duration::from_millis(100)); put(&producer, 2, None); let budget = Duration::from_millis(50); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))) .ordered(); assert_eq!(items(&mut ordered), ["gap 0..1 (old)", "group 1", "group 2", "pending"]); } /// A final drop replaces the part of a stale gap it covers while that gap still waits, so /// the buffered gaps never overlap: the stale run keeps what lies either side of it. #[test] #[cfg(feature = "async")] fn a_buffered_gap_gives_way_to_what_replaces_it() { let producer = received_producer(None); let mut ordered = producer.subscribe(replay()).ordered(); ordered.gaps.insert(4, (8, StreamError::Old.into())); ordered.forget_gaps(5..7); ordered.gaps.insert(5, (7, StreamError::DeliveryTimeout.into())); let held: Vec<_> = ordered.gaps.iter().map(|(start, (end, _))| (*start, *end)).collect(); assert_eq!(held, [(4, 5), (5, 7), (7, 8)]); ordered.forget_gaps(0..100); assert!(ordered.gaps.is_empty()); } /// What a range excludes from a gap that is partly covered stays a gap, so a range that /// widens again over it reports it. #[test] #[cfg(feature = "async")] fn a_gap_a_range_clips_keeps_what_the_range_excluded() { let producer = received_producer(None); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(None))) .ordered(); put(&producer, 2, None); producer.drop_groups(0..2, StreamError::DeliveryTimeout); producer.drop_groups(3..6, StreamError::DeliveryTimeout); let first = ordered.next_group().now_or_never().expect("ready").unwrap().unwrap(); assert!(matches!(first, Item::Gap { range, .. } if range == (0..2))); ordered.set_range(4..5); assert_eq!(items(&mut ordered), ["gap 4..5 (delivery timeout)", "pending"]); ordered.set_range(..); assert_eq!( items(&mut ordered), ["gap 3..4 (delivery timeout)", "gap 5..6 (delivery timeout)", "pending"] ); } #[test] #[cfg(feature = "async")] fn a_late_arrival_is_a_gap() { let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 0, None); put(&producer, 5, None); assert_eq!(items(&mut ordered), ["group 0", "group 5", "pending"]); put(&producer, 3, None); assert_eq!(items(&mut ordered), ["gap 3..4 (old)", "pending"]); put(&producer, 6, None); assert_eq!( items(&mut ordered), ["group 6", "pending"], "reported once, nothing else" ); } /// A gap is clipped to what the reader asked for. #[test] #[cfg(feature = "async")] fn a_gap_is_clipped_to_the_reader_range() { let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered().with_range(2..4); producer.drop_groups(0..6, StreamError::DeliveryTimeout); put(&producer, 5, None); assert_eq!(items(&mut ordered), ["gap 2..4 (delivery timeout)", "pending"]); } /// A drop found while a higher gap waits goes out first, in key order, rather than staying /// hidden behind it. #[test] #[cfg(feature = "async")] fn a_lower_drop_found_late_goes_before_a_buffered_gap() { let producer = received_producer(None); let mut ordered = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(None))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); producer.drop_groups(4..5, StreamError::DeliveryTimeout); assert_eq!(items(&mut ordered), ["group 0", "pending"]); producer.drop_groups(1..2, StreamError::DeliveryTimeout); assert_eq!( items(&mut ordered), ["gap 1..2 (delivery timeout)", "group 2", "pending"] ); put(&producer, 3, None); assert_eq!( items(&mut ordered), ["group 3", "gap 4..5 (delivery timeout)", "pending"] ); } /// A range narrowed after a gap went out ahead of a group read keeps that group back, /// handing it out again once the range widens, and clips the gaps still waiting. #[test] #[cfg(feature = "async")] fn a_range_moved_after_a_gap_applies_to_what_is_buffered() { let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 2, None); producer.drop_groups(0..2, StreamError::DeliveryTimeout); producer.drop_groups(3..6, StreamError::DeliveryTimeout); let first = ordered.next_group().now_or_never().expect("ready").unwrap().unwrap(); assert!(matches!(first, Item::Gap { range, .. } if range == (0..2))); ordered.set_range(..2); assert_eq!( items(&mut ordered), ["pending"], "capped below the held group, which waits for the range" ); ordered.set_range(..4); assert_eq!( items(&mut ordered), ["group 2", "gap 3..4 (delivery timeout)", "pending"], "the held group returns, and the gap above is clipped to the range" ); let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 2, None); put(&producer, 3, None); producer.drop_groups(0..2, StreamError::DeliveryTimeout); let first = ordered.next_group().now_or_never().expect("ready").unwrap().unwrap(); assert!(matches!(first, Item::Gap { range, .. } if range == (0..2))); ordered.set_range(3..); assert_eq!( items(&mut ordered), ["group 3", "pending"], "a floor raised over the held group passes it" ); } /// A gap a moved range only partly covers returns that part, and the rest stays for the /// range to widen over. #[test] #[cfg(feature = "async")] fn a_moved_range_returns_a_buffered_gap_by_the_part_it_covers() { let producer = track_producer("test", None); let mut ordered = producer.subscribe(replay()).ordered(); put(&producer, 2, None); producer.drop_groups(0..2, StreamError::DeliveryTimeout); producer.drop_groups(3..6, StreamError::DeliveryTimeout); let first = ordered.next_group().now_or_never().expect("ready").unwrap().unwrap(); assert!(matches!(first, Item::Gap { range, .. } if range == (0..2))); ordered.set_range(..4); assert_eq!( items(&mut ordered), ["group 2", "gap 3..4 (delivery timeout)", "pending"], "capped inside the gap" ); ordered.set_range(..); assert_eq!( items(&mut ordered), ["gap 4..6 (delivery timeout)", "pending"], "widening returns the rest" ); } /// The wait follows the reader's budget: a zero budget waits for nothing, a budget /// narrowed mid-wait ends it, and an unbounded one waits until the hole resolves. #[test] #[cfg(feature = "async")] fn next_group_waits_on_an_owed_group_within_the_budget() { let producer = received_producer(None); let mut consumer = producer.subscribe(Subscription::default()).ordered(); put(&producer, 0, None); assert_eq!(drain_ordered(&mut consumer), [0]); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [2], "a zero budget waits for nothing"); let producer = received_producer(None); let mut consumer = producer.subscribe(replay()).ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [0]); consumer.control().update(Subscription::default()).unwrap(); assert_eq!(drain_ordered(&mut consumer), [2], "the narrowed budget gave up"); let producer = received_producer(None); let mut consumer = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(None))) .ordered(); put(&producer, 0, None); put(&producer, 2, None); assert_eq!(drain_ordered(&mut consumer), [0]); elapse(&producer, Duration::from_secs(3600)); assert!(drain_ordered(&mut consumer).is_empty(), "no deadline, however long"); put(&producer, 1, None); assert_eq!(drain_ordered(&mut consumer), [1, 2]); } #[test] #[cfg(feature = "async")] fn next_group_skips_late_arrivals() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None).ordered(); // Seq 5 arrives first. producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); let group = consumer .next_group() .now_or_never() .expect("should not block") .expect("would have errored") .expect("track should not be closed") .group() .expect("a group"); assert_eq!(group.start, 5); // Seq 3 arrives late, skipped because 3 <= 5. producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); // Seq 4 arrives late and is also skipped. producer .create_group(group::Info { start: 4 }) .unwrap() .finish() .unwrap(); // Seq 7 arrives and is returned, after the groups it passed are reported. producer .create_group(group::Info { start: 7 }) .unwrap() .finish() .unwrap(); assert_eq!(items(&mut consumer), ["gap 3..5 (old)", "group 7", "pending"]); } #[test] #[cfg(feature = "async")] fn next_group_returns_arrivals_in_order() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); // Seq 3 arrives first, then seq 5. Both should be returned in arrival order. producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); let group = consumer .next_group() .now_or_never() .expect("should not block") .expect("would have errored") .expect("track should not be closed") .group() .expect("a group"); assert_eq!(group.start, 3); let group = consumer .next_group() .now_or_never() .expect("should not block") .expect("would have errored") .expect("track should not be closed") .group() .expect("a group"); assert_eq!(group.start, 5); } /// The key a peek reserves, without taking it. fn peek(subscriber: &mut Subscriber, ordered: bool) -> Poll> { subscriber .poll_peek_group(ordered, &kio::Waiter::noop()) .map(|res| res.unwrap().map(|reserved| reserved.key())) } #[test] fn skipping_another_copys_groups_preserves_arrival_bounds_and_the_last_key() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); for key in [u64::MAX - 2, u64::MAX - 1] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let _finished = producer.finish().unwrap(); assert_eq!(peek(&mut subscriber, true), Poll::Ready(Some(u64::MAX - 2))); subscriber.skip_ordered(u64::MAX - 1); assert_eq!(peek(&mut subscriber, true), Poll::Ready(Some(u64::MAX - 1))); subscriber.skip_ordered(u64::MAX); subscriber.skip_ordered(0); assert_eq!(peek(&mut subscriber, true), Poll::Ready(None)); assert_eq!(subscriber.record().delivered.runs(), 0, "skips are not deliveries"); assert_eq!(subscriber.take_stale().groups, 0, "duplicates are not stale skips"); let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() }; assert_eq!(cursor.min_key, 0); assert_eq!(cursor.end_key, None); assert_eq!(cursor.index, 0); assert_eq!(peek(&mut subscriber, false), Poll::Ready(Some(u64::MAX - 2))); } #[test] fn an_ordered_skip_counts_only_convictions_not_delivered_elsewhere() { let mut producer = track_producer("test", Info::timed()); for second in 0..4 { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(None); let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(true, &kio::Waiter::noop()) else { panic!("group 3 is servable"); }; assert_eq!(reserved.key(), 3); subscriber.skip_ordered(2); assert_eq!(subscriber.take(reserved).unwrap().start, 3); assert_eq!(subscriber.take_stale().groups, 1, "only group 2 was skipped for age"); assert_eq!(subscriber.take_stale().groups, 0, "counted once"); } /// A peek moves no cursor, so the bounds in force at the next read decide it. #[test] fn changed_bounds_after_a_peek_decide_the_next_read() { for ordered in [false, true] { for raise_floor in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); for key in [1, 2] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let waiter = kio::Waiter::noop(); assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(1))); if raise_floor { subscriber.set_range(2..); let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("group 2 is inside the new floor"); }; assert_eq!(group.start, 2); } else { subscriber.set_range(..0); assert!(subscriber.poll_group(ordered, &waiter).is_pending()); subscriber.set_range(..); let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("group 1 was parked, not consumed"); }; assert_eq!(group.start, 1); } } } } #[test] fn taking_a_peeked_group_does_not_substitute_after_eviction() { for ordered in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); for key in [1, 3] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(ordered, &kio::Waiter::noop()) else { panic!("group 1 is servable"); }; evict(&producer, 1); assert_eq!( subscriber.take(reserved).unwrap().start, 1, "ordered={ordered}: eviction must not substitute group 3" ); } } /// A group a widening re-offers stays owed while it is only peeked. #[test] fn peeking_keeps_a_re_offered_group_owed() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); subscriber.set_range(..0); for key in [1, 2] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let waiter = kio::Waiter::noop(); assert!(subscriber.poll_group(false, &waiter).is_pending()); subscriber.set_range(..); for _ in 0..2 { assert_eq!(peek(&mut subscriber, false), Poll::Ready(Some(1))); } let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(false, &waiter) { read.push(group.start); } assert_eq!(read, [1, 2]); } #[test] fn peeking_keeps_each_orders_cursor_and_rechecks_bounds() { for ordered in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); for key in [2, 1] { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } let waiter = kio::Waiter::noop(); let first = if ordered { 1 } else { 2 }; for _ in 0..2 { assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(first))); } let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("the peeked group is still available"); }; assert_eq!(group.start, first); let second = if ordered { 2 } else { 1 }; subscriber.set_range(..0); assert!(subscriber.poll_peek_group(ordered, &waiter).is_pending()); subscriber.set_range(..); assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(second))); } } #[test] #[cfg(feature = "async")] fn ordered_and_arrival_cursors_are_independent() { let producer = track_producer("test", Info::timed()); let mut ordered = producer.subscribe(replay()).ordered(); let mut arrival = producer.subscribe(replay()); // Out-of-order arrivals: seq 5 first, then seq 3. producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); // The ordered handle returns the smallest key first, regardless of // arrival order. let group = ordered .next_group() .now_or_never() .expect("should not block") .expect("would have errored") .expect("track should not be closed") .group() .expect("a group"); assert_eq!(group.start, 3); // The plain handle walks arrivals, so it still starts at seq 5. assert_eq!(arrival.assert_group().start, 5); } #[test] #[cfg(feature = "async")] fn end_at_caps_next_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); for s in 0..6 { producer .create_group(group::Info { start: s }) .unwrap() .finish() .unwrap(); } consumer.set_range(..3); // Groups 0, 1, 2 are within the cap. assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 0 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 1 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 2 ); // Group 3 is beyond the cap: next_group parks even though cached groups exist. assert!( consumer.next_group().now_or_never().is_none(), "capped consumer must block instead of returning out-of-range groups" ); } #[test] #[cfg(feature = "async")] fn end_at_release_drains_cached_groups() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); for s in 0..6 { producer .create_group(group::Info { start: s }) .unwrap() .finish() .unwrap(); } consumer.set_range(..2); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 0 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 1 ); assert!(consumer.next_group().now_or_never().is_none(), "capped at 2"); // Raise the cap; previously-blocked cached groups become available again. consumer.set_range(..5); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 2 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 3 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 4 ); assert!(consumer.next_group().now_or_never().is_none(), "capped at 5"); // Remove the cap; everything remaining flows. consumer.set_range(..); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 5 ); assert!(consumer.next_group().now_or_never().is_none(), "no more groups"); } #[test] #[cfg(feature = "async")] fn end_at_lower_than_cursor_parks_consumer() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); for s in 0..3 { producer .create_group(group::Info { start: s }) .unwrap() .finish() .unwrap(); } // Drain everything with no cap. assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 0 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 1 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 2 ); // Lower the cap below the cursor. New groups beyond the cap are blocked. consumer.set_range(..2); producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 4 }) .unwrap() .finish() .unwrap(); assert!( consumer.next_group().now_or_never().is_none(), "cap is below cursor; nothing returnable until cap rises" ); // Restoring the cap to no-limit (or any value >= cursor) releases them. consumer.set_range(..); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 3 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 4 ); } #[test] #[cfg(feature = "async")] fn end_at_toggling_around_late_arrivals() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); consumer.set_range(..6); // Out-of-order arrivals all within the cap. producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 5 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); // One beyond the cap; should be held even though it arrived in the middle. producer .create_group(group::Info { start: 8 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 4 }) .unwrap() .finish() .unwrap(); // next_group walks in key order through everything <= cap. assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 2 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 3 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 4 ); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 5 ); // Now blocked: 8 is still beyond the cap. assert!(consumer.next_group().now_or_never().is_none()); // Raise the cap; cached seq 8 is finally served. consumer.set_range(..11); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 8 ); } /// `recv_group` (arrival order) honors the `end_at` cap like `next_group`: a group /// beyond it is passed, holding nothing, and a raised cap finds it in the cache again. /// Once the track finishes, a capped reader ends with what it could read. #[test] #[cfg(feature = "async")] fn end_at_caps_recv_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); for s in 0..3 { producer .create_group(group::Info { start: s }) .unwrap() .finish() .unwrap(); } consumer.set_range(..2); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 2"); consumer.set_range(..); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); consumer.set_range(..3); let _open = producer.create_group(group::Info { start: 3 }).unwrap(); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 3"); let _producer = producer.finish().unwrap(); assert!( matches!(consumer.recv_group().now_or_never(), Some(Ok(None))), "a finished track ends a capped reader" ); } /// A group beyond the cap must not block in-range groups that arrive behind /// it: a relay can ingest a burst micro-reordered (newest first). #[test] #[cfg(feature = "async")] fn recv_group_serves_arrivals_behind_the_cap() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); consumer.set_range(..2); // Reordered burst: the beyond-cap group arrives first. producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 2"); consumer.set_range(..3); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); } #[kio::test] #[cfg(feature = "async")] async fn group_ranges_preserve_the_floor_when_the_cap_changes() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); consumer.set_range(2..=2); producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 2); consumer.set_range(..4); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 3 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 3); consumer.set_range(0..=4); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 4 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 4); } /// A raised `start_at` drops parked groups it overtook instead of re-offering /// them once the cap rises. #[test] #[cfg(feature = "async")] fn a_raised_floor_drops_parked_recv_groups() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); consumer.set_range(..1); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert!( consumer.recv_group().now_or_never().is_none(), "group 1 parked at the cap" ); consumer.set_range(2..); producer .create_group(group::Info { start: 2 }) .unwrap() .finish() .unwrap(); assert_eq!( consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2, "the overtaken parked group is dropped, not re-offered" ); } /// A parked group the producer aborts (eviction/expiry) is dropped: it is /// neither delivered once the cap rises nor allowed to hold the stream open /// after the track finishes. This is what bounds parking by the cache policy. #[test] #[cfg(feature = "async")] fn evicted_parked_recv_groups_are_dropped() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); consumer.set_range(..1); let straggler = producer.create_group(group::Info { start: 1 }).unwrap(); assert!( consumer.recv_group().now_or_never().is_none(), "group 1 parked at the cap" ); // The cache evicts the parked group (abort-as-tombstone), then the track ends. straggler.abort(Error::Old).unwrap(); let _producer = producer.finish().unwrap(); consumer.set_range(..); assert!( matches!(consumer.recv_group().now_or_never(), Some(Ok(None))), "a dead parked group must not be delivered or hold the stream open" ); } /// An exclusive cap at 0 is the empty range: no group is delivered, even group 0. #[test] #[cfg(feature = "async")] fn end_at_zero_is_the_empty_range() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); consumer.set_range(..0); assert!( consumer.recv_group().now_or_never().is_none(), "empty cap delivers nothing" ); consumer.set_range(..1); assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); assert!(consumer.recv_group().now_or_never().is_none(), "group 1 stays parked"); } /// A requested empty range plus a local empty cap still parks while another /// subscriber keeps aggregate demand unbounded. #[test] #[cfg(feature = "async")] fn empty_local_cap_holds_while_another_subscriber_requests_everything() { let producer = track_producer("test", Info::timed()); let mut everything = producer.subscribe(replay()); let mut empty = producer.subscribe(replay().with_order(Order::Ascending)); empty.set_range((Bound::Unbounded, Bound::Excluded(0))); for s in 0..3 { producer .create_group(group::Info { start: s }) .unwrap() .finish() .unwrap(); } assert_eq!( everything.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0 ); assert!( empty.recv_group().now_or_never().is_none(), "local empty cap must not ride the unbounded aggregate" ); empty.set_range(..2); assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(empty.recv_group().now_or_never().is_none(), "still capped at 2"); empty.set_range(..); assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); } /// A frame-limited exclusive end includes the last group and stops before that frame. #[test] #[cfg(feature = "async")] fn end_at_frame_limited_last_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); let end = Position::new(1, 2); consumer.set_range((Bound::Unbounded, Bound::Included(end.group))); for s in 0..3u64 { let mut group = producer.create_group(group::Info { start: s }).unwrap(); for i in 0..3u8 { group.write_frame(group.key_time(), vec![i]).unwrap(); } group.finish().unwrap(); } assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, 0 ); let mut last = consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap(); assert_eq!(last.start, 1); last = last.with_range(..end.frame); assert_eq!( last.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0], 0 ); assert_eq!( last.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0], 1 ); assert!( last.read_frame().now_or_never().unwrap().unwrap().is_none(), "frame cap is exclusive" ); assert!( consumer.next_group().now_or_never().is_none(), "group 2 is past the exclusive group cap" ); } /// Group `u64::MAX` is reserved, so every group fits an exclusive range: neither a create /// nor an append reaches it, and an inclusive bound there withholds nothing. #[test] #[cfg(feature = "async")] fn the_reserved_key_is_refused() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); consumer.set_range(..=u64::MAX); assert!(matches!( producer.create_group(group::Info { start: u64::MAX }), Err(Error::BoundsExceeded(_)) )); producer .create_group(group::Info { start: u64::MAX - 1 }) .unwrap() .finish() .unwrap(); assert!(matches!(producer.append_next(), Err(Error::BoundsExceeded(_)))); assert_eq!( consumer .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap() .start, u64::MAX - 1 ); // The track's end lands on the reserved key. let _finished = producer.finish().unwrap(); assert!(consumer.next_group().now_or_never().unwrap().unwrap().is_none()); } #[test] fn rescale_matches_the_track_timeline() { let timed = Info::timed(); let ms = Timestamp::from_micros(2_400).unwrap(); let rescaled = timed.rescale(ms).unwrap(); assert_eq!(rescaled, Timestamp::from_millis(2).unwrap()); // Halfway rounds down too, as `@moq/net`'s `Track.rescale` does. let half = timed.rescale(Timestamp::from_micros(1_500).unwrap()).unwrap(); assert_eq!(half, Timestamp::from_millis(1).unwrap()); assert!(matches!( timed.rescale(Timestamp::unscaled(2)), Err(Error::TimestampMismatch) )); // (2^61-1) seconds overflows conversion to the milli-scale track. let huge = Timestamp::from_secs((1 << 61) - 1).unwrap(); assert!(matches!(timed.rescale(huge), Err(Error::TimestampMismatch))); let untimed = Info::default(); assert_eq!(untimed.rescale(Timestamp::unscaled(2)).unwrap(), Timestamp::unscaled(2)); assert!(matches!(untimed.rescale(ms), Err(Error::TimestampMismatch))); } #[test] fn write_frame_rejects_an_oversized_frame_before_appending_its_group() { let mut producer = track_producer("test", Info::timed()); let frame = bytes::Bytes::from(vec![0; group::MAX_CACHE_BYTES as usize + 1]); assert!(matches!( producer.write_frame(Timestamp::ZERO, frame), Err(Error::FrameTooLarge) )); assert_eq!( producer.latest_key(), None, "the rejected frame did not publish a group" ); } #[kio::test] #[cfg(feature = "async")] async fn fetch_cache_hit() { let producer = track_producer("test", Info::timed()); // Produce a cached group. let mut group = producer.append_next().unwrap(); // seq 0 group .write_frame(group.key_time(), bytes::Bytes::from_static(b"hello")) .unwrap(); group.finish().unwrap(); // A cached group resolves immediately and never queues a request. `peek_group` // also returns it synchronously. let dynamic = producer.requests(); let consumer = producer.consume(); assert!(consumer.peek_group(0).is_some()); let mut g = consumer.fetch_group(0, None).await.unwrap(); assert_eq!(g.start, 0); assert_eq!(&g.read_frame().await.unwrap().unwrap().payload[..], b"hello"); // Nothing was queued for the dynamic handler to serve. assert!(dynamic.poll_requested_fill_group(&kio::Waiter::noop()).is_pending()); } #[kio::test] #[cfg(feature = "async")] async fn fetch_miss_signals_dynamic() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); // A cache miss isn't in `peek_group`, but a dynamic handler exists, so // `fetch_group` stays pending and queues a request. `*pending` derefs the // wrapper to the inner `Fetching` (a `kio::Task`). assert!(consumer.peek_group(5).is_none()); let mut pending = consumer.fetch_group(5, group::Fetch::default().with_priority(7)); assert!(kio::Task::poll(&mut *pending, &kio::Waiter::noop()).is_pending()); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); assert_eq!(req.key(), 5); assert_eq!(req.priority(), 7); // Serve it by accepting the request; the fetch then resolves. let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); group.finish().unwrap(); let mut g = pending.await.unwrap(); assert_eq!(g.start, 5); assert_eq!(&g.read_frame().await.unwrap().unwrap().payload[..], b"hi"); } #[kio::test] #[cfg(feature = "async")] async fn fetch_miss_rejects() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let pending = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); req.reject(Error::Cancel); assert!(matches!(pending.await, Err(Error::Cancel))); let fetch = producer.state.read().fills.clone(); assert!(fetch.read().is_empty()); } #[kio::test] #[cfg(feature = "async")] async fn fetch_miss_drop_rejects() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let pending = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); drop(req); assert!(matches!(pending.await, Err(Error::Dropped))); } /// A request stays wanted while any joined fetch waits, and once the last leaves it /// is withdrawn: a later fetch starts a fresh request instead of joining it. #[kio::test] #[cfg(feature = "async")] async fn fetch_request_unused_once_every_fetch_leaves() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let second = consumer.fetch_group(5, None); drop(first); assert!(req.poll_unwanted(&kio::Waiter::noop()).is_pending()); drop(second); assert!(req.poll_unwanted(&kio::Waiter::noop()).is_ready()); let retry = consumer.fetch_group(5, None); let fresh = dynamic .requested_fill_group() .now_or_never() .expect("the retry queues a fresh request") .unwrap(); drop(req); assert!(fresh.poll_unwanted(&kio::Waiter::noop()).is_pending()); fresh .accept(Info::timed()) .unwrap() .expect("a fresh group") .finish() .unwrap(); assert_eq!(retry.await.unwrap().start, 5); } /// A fetch abandoned before any handler took it is withdrawn from the queue too, so /// no handler serves it. #[kio::test] #[cfg(feature = "async")] async fn fetch_abandoned_while_queued_never_reaches_the_handler() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); drop(consumer.fetch_group(5, None)); assert!(dynamic.requested_fill_group().now_or_never().is_none()); let fetch = producer.state.read().fills.clone(); assert!(fetch.read().is_empty()); } /// The withdrawal happens as the last caller leaves, not when the handler notices: a /// fetch arriving before the handler drops the abandoned request is not failed with it. #[kio::test] #[cfg(feature = "async")] async fn fetch_after_the_last_caller_left_is_not_dropped() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); drop(first); let mut retry = consumer.fetch_group(5, None); drop(req); assert!(kio::Task::poll(&mut *retry, &kio::Waiter::noop()).is_pending()); let fresh = dynamic .requested_fill_group() .now_or_never() .expect("the retry queues a fresh request") .unwrap(); fresh .accept(Info::timed()) .unwrap() .expect("a fresh group") .finish() .unwrap(); assert_eq!(retry.await.unwrap().start, 5); } /// A handler that accepts after every caller left still caches the group, so a fetch /// that queued a fresh request meanwhile resolves from it and that request is moot. #[kio::test] #[cfg(feature = "async")] async fn fetch_accept_after_withdrawal_caches_the_group() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); drop(first); let retry = consumer.fetch_group(5, None); let fresh = dynamic .requested_fill_group() .now_or_never() .expect("the retry queues a fresh request") .unwrap(); let mut group = req .accept(Info::timed()) .expect("a withdrawn request still accepts") .expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); group.finish().unwrap(); let mut fetched = retry.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"hi"); assert!(!fresh.is_wanted(), "the retry left the fresh request"); assert!(matches!(fresh.accept(Info::timed()), Ok(None))); } /// Dropping an auto trait from a published type is a semver break, so the group /// consumer a cached fetch holds must not cost `Fetching` its unwind safety. #[test] fn fetching_is_unwind_safe() { fn assert_unwind_safe() {} assert_unwind_safe::(); assert_unwind_safe::(); } /// A fetch that hits the cache holds the group until polled, so a handler that aborts /// the group once nobody wants it (an abandoned upstream fetch) leaves it alone. #[kio::test] #[cfg(feature = "async")] async fn fetch_hit_keeps_the_group_wanted() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); drop(first.await.unwrap()); let hit = consumer.fetch_group(5, None); assert!( !group.handle().abort_unwanted(Error::Cancel), "the pending hit wants the group" ); let mut fetched = hit.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"head"); } /// A bounded FIN is local to its request, never proof that the original group ended. #[kio::test] #[cfg(feature = "async")] async fn bounded_transport_copies_never_answer_whole_fetches() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let request = |start, end| FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end, }; let bounded = consumer.fetch_range(0, request(2, Some(4))); let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (0, Some(4))); let mut prefix = fill.accept(Info::timed()).unwrap().expect("a fresh group"); for _ in 0..4 { prefix.write_frame(prefix.key_time(), b"prefix".as_slice()).unwrap(); } prefix.finish().unwrap(); let mut bounded = bounded.await.unwrap().with_range(2..).with_end(Some(4)); for expected in 2..4 { assert_eq!(bounded.index(), expected); assert!(bounded.read_frame().await.unwrap().is_some()); } assert!(bounded.read_frame().await.unwrap().is_none()); assert!( consumer.cached_groups(0..1).is_empty(), "a bounded prefix is no whole cached group" ); // A narrower bound reuses the same stored prefix. assert!( consumer .fetch_range(0, request(1, Some(3))) .now_or_never() .unwrap() .is_ok() ); let mut whole = consumer.fetch_group(0, None); assert!(kio::Task::poll(&mut *whole, &kio::Waiter::noop()).is_pending()); let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (0, None)); let mut complete = fill.accept(Info::timed()).unwrap().expect("a fresh group"); for _ in 0..6 { complete.write_frame(complete.key_time(), b"prefix".as_slice()).unwrap(); } complete.finish().unwrap(); let mut whole = whole.await.unwrap(); for _ in 0..6 { assert!(whole.read_frame().await.unwrap().is_some()); } assert!(whole.read_frame().await.unwrap().is_none()); assert_eq!(producer.state.read().lookup[&0].copies().count(), 1); } /// Tail starts replace one backfill slot rather than retaining a copy per peer range. #[kio::test] #[cfg(feature = "async")] async fn bounded_tail_refills_coalesce_behind_a_live_copy() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); for start in (1..100).rev() { let fetched = consumer.fetch_range( 0, FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end: Some(start + 1), }, ); let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (start, Some(start + 1))); let mut tail = fill.accept(Info::timed()).unwrap().expect("a fresh group"); tail.write_frame(tail.key_time(), b"tail".as_slice()).unwrap(); tail.finish().unwrap(); assert_eq!(fetched.await.unwrap().index(), start); assert_eq!(producer.state.read().lookup[&0].copies().count(), 2); } let mut sub = producer.subscribe(None); let group = sub.recv_group().await.unwrap().unwrap(); assert_eq!( group.with_range(0..).index(), 200, "a bounded head does not replace the live feed for whole-group readers" ); live.abort(Error::Cancel).unwrap(); // The backfill remains cache-owned after the live copy failed. assert!(!producer.state.read().lookup[&0].is_aborted()); } /// A bounded fill that a later, disjoint one displaces from the cache still answers the /// fetch that asked for it. #[kio::test] #[cfg(feature = "async")] async fn a_displaced_bounded_fill_still_answers_its_fetch() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); let range = |start, end| FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end: Some(end), }; let fill = async |frames: std::ops::Range| { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (frames.start, Some(frames.end))); let mut group = fill.accept(Info::timed()).unwrap().expect("a fresh group"); for _ in frames { group.write_frame(group.key_time(), b"tail".as_slice()).unwrap(); } group.finish().unwrap(); }; let later = consumer.fetch_range(0, range(4, 10)); fill(4..10).await; let earlier = consumer.fetch_range(0, range(0, 3)); fill(0..3).await; assert_eq!(earlier.await.unwrap().index(), 0); assert_eq!(later.await.unwrap().index(), 4); } /// A copy cut short at the back never displaces a fetched one that reaches the group's /// end, so whole-group readers keep it, and it still answers its own fetch. #[kio::test] #[cfg(feature = "async")] async fn a_bounded_fill_never_displaces_a_whole_tail() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); let fill = async |start: u64, end: Option| { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (start, end)); let mut group = fill.accept(Info::timed()).unwrap().expect("a fresh group"); group.write_frame(group.key_time(), b"tail".as_slice()).unwrap(); group }; let tail = consumer.fetch_group( 0, group::Fetch { frame_start: 5, ..Default::default() }, ); let _tail = fill(5, None).await; let head = consumer.fetch_range( 0, FetchOptions { options: group::Fetch::default(), end: Some(1), }, ); fill(0, Some(1)).await.finish().unwrap(); assert_eq!(head.await.unwrap().index(), 0); assert_eq!(tail.await.unwrap().index(), 5); let mut sub = producer.subscribe(None); assert_eq!(sub.assert_group().with_range(0..).index(), 5); } /// A track abort ends every copy still being written, including one the cache never held /// or has since displaced: its reader gets the abort and its writer fails. #[kio::test] #[cfg(feature = "async")] async fn a_track_abort_ends_uncached_copies() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let range = |start, end| FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end, }; let fill = async |frames: FetchOptions| { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!( (fill.frame_start(), fill.frame_end()), (frames.options.frame_start, frames.end) ); let mut group = fill.accept(Info::timed()).unwrap().expect("a fresh group"); group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); group }; let fetch = async |key, frames: FetchOptions| { let pending = consumer.fetch_range(key, frames.clone()); let held = fill(frames).await; (held, pending.await.unwrap()) }; let mut lives = Vec::new(); for key in 0..2 { let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(200).unwrap(); live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); lives.push(live); } // A bounded copy beside a cached whole tail never enters the cache. let _tail = fetch(0, range(5, None)).await; let kept = fetch(0, range(0, Some(3))).await; // A later copy displaces a live group's bounded head, and a fetched-only bounded copy. let head = fetch(1, range(4, Some(10))).await; let _wider = fetch(1, range(0, Some(3))).await; let fetched = fetch(2, range(0, Some(3))).await; let _wider = fetch(2, range(0, None)).await; let mut detached = [kept, head, fetched]; for (_, fetched) in &mut detached { fetched.read_frame().await.unwrap().unwrap(); } producer.abort(Error::Cancel).unwrap(); for (mut held, mut fetched) in detached { assert!(matches!(fetched.read_frame().now_or_never(), Some(Err(Error::Cancel)))); assert!(held.write_frame(held.key_time(), b"late".as_slice()).is_err()); } } /// A track abort keeps a finished bounded copy for the reader still draining it, as it /// does a finished whole group. #[kio::test] #[cfg(feature = "async")] async fn a_track_abort_keeps_a_finished_bounded_copy() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let fetch = consumer.fetch_range( 0, FetchOptions { options: group::Fetch::default(), end: Some(2), }, ); let mut group = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); for _ in 0..2 { group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); } group.finish().unwrap(); let mut fetched = fetch.await.unwrap(); producer.abort(Error::Cancel).unwrap(); assert!(fetched.read_frame().await.unwrap().is_some()); assert!(fetched.read_frame().await.unwrap().is_some()); assert!(fetched.read_frame().await.unwrap().is_none()); } /// A finished bounded copy below the declared end is nothing in flight, so an abort after /// the end is reached still ends the track cleanly. #[kio::test] #[cfg(feature = "async")] async fn a_finished_bounded_copy_lets_the_end_settle() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let fetch = consumer.fetch_range( 0, FetchOptions { options: group::Fetch::default(), end: Some(1), }, ); let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!(fill.frame_end(), Some(1)); let mut group = fill.accept(Info::timed()).unwrap().expect("a fresh group"); group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); // The end is reached while the copy is still being written, so only the abort // can find it settled. producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); producer.finish_at(Position::group(2)).unwrap(); group.finish().unwrap(); fetch.await.unwrap(); producer.abort(Error::Timeout).unwrap(); assert!(matches!( consumer.poll_complete(&kio::Waiter::noop()), Poll::Ready(Ok(())) )); } /// A copy cut short at the back never answers a whole-group reader. A live group the /// publisher cut is still handed out as that cut, and one it cancelled stays a hole, /// whatever bounded copy of it remains. #[kio::test] #[cfg(feature = "async")] async fn a_back_cut_copy_never_answers_a_whole_group_reader() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let bounded = async |key: u64| { let fetch = consumer.fetch_range( key, FetchOptions { options: group::Fetch::default(), end: Some(1), }, ); let mut group = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); group.finish().unwrap(); fetch.await.unwrap(); }; let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); for (key, err) in [(0, Error::App(5)), (1, Error::Cancel)] { let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(200).unwrap(); live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); bounded(key).await; live.abort(err).unwrap(); } // Fetched only, so it reaches the ordered reader alone. bounded(2).await; let cut = arrival.assert_group(); assert_eq!(cut.start, 0); assert!(matches!(cut.with_range(200..).read_frame().await, Err(Error::App(5)))); arrival.assert_no_group(); let cut = ordered .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap(); assert_eq!(cut.start, 0); assert!(matches!(cut.with_range(200..).read_frame().await, Err(Error::App(5)))); assert!( ordered.next_group().now_or_never().is_none(), "neither the cancelled group nor a copy ending short of its group is read" ); } /// Queued ranges widen before dispatch; a taken cap does not promise an unseen suffix. #[kio::test] #[cfg(feature = "async")] async fn bounded_fetches_widen_before_dispatch_and_split_after() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let request = |end| FetchOptions { options: group::Fetch::default(), end: Some(end), }; let _a = consumer.fetch_range(0, request(3)); let _b = consumer.fetch_range(0, request(5)); let first = dynamic.requested_fill_group().await.unwrap(); assert_eq!(first.frame_end(), Some(5)); let _c = consumer.fetch_range(0, request(8)); let second = dynamic.requested_fill_group().await.unwrap(); assert_eq!((first.frame_end(), second.frame_end()), (Some(5), Some(8))); } /// An accepted group starts where its request did before anyone can see it, so a /// wider fetch arriving before the handler writes misses instead of being handed a /// reader that would skip the head it asked for. #[kio::test] #[cfg(feature = "async")] async fn accepted_group_starts_at_the_request() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let _tail = consumer.fetch_group( 5, group::Fetch { frame_start: 3, ..Default::default() }, ); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let _group = req.accept(Info::timed()).unwrap().expect("a fresh group"); let mut whole = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("the wider fetch misses and queues its own request") .unwrap(); assert_eq!(req.frame_start(), 0); assert!(kio::Task::poll(&mut *whole, &kio::Waiter::noop()).is_pending()); } /// Once a group nobody wanted is aborted, a later fetch misses rather than reading /// the abort, and queues a fresh request. #[kio::test] #[cfg(feature = "async")] async fn fetch_misses_an_abandoned_group() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let group = req.accept(Info::timed()).unwrap().expect("a fresh group"); drop(first.await.unwrap()); assert!(group.handle().abort_unwanted(Error::Cancel)); let retry = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("the retry queues a fresh request") .unwrap(); req.accept(Info::timed()) .unwrap() .expect("a fresh group") .finish() .unwrap(); assert!(retry.await.unwrap().read_frame().await.unwrap().is_none()); } #[kio::test] #[cfg(feature = "async")] async fn fetch_reject_does_not_poison_retry() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let pending = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); req.reject(Error::Cancel); assert!(matches!(pending.await, Err(Error::Cancel))); let retry = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"retry")) .unwrap(); group.finish().unwrap(); let mut group = retry.await.unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"retry"); } /// A group cached from a frame-bounded subscription starts partway in. Serving it to /// someone who asked for the whole group would silently hand back a tail, so it is a /// miss and the fetch goes upstream instead. #[kio::test] #[cfg(feature = "async")] async fn fetch_ignores_a_group_that_starts_too_late() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); // The live subscription resumed mid-group, so only the tail is cached. let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); group.start_at(3).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"tail")) .unwrap(); group.finish().unwrap(); // A fetch for the tail is covered and resolves from the cache. let fetch = consumer.fetch_group( 0, group::Fetch { frame_start: 3, ..Default::default() }, ); let cached = fetch.now_or_never().expect("covered by the cache").unwrap(); assert_eq!(cached.index(), 3); // A fetch for the whole group is not, so it queues for a handler rather than // resolving to the tail. let mut fetch = std::pin::pin!(consumer.fetch_group(0, None)); assert!( futures::poll!(fetch.as_mut()).is_pending(), "must not answer from the tail" ); let request = dynamic.requested_fill_group().await.unwrap(); assert_eq!((request.key(), request.frame_start()), (0, 0)); // Serving it replaces the too-narrow entry rather than colliding with it. let mut whole = request.accept(Info::timed()).unwrap().expect("a fresh group"); whole .write_frame(whole.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); whole.finish().unwrap(); let mut served = fetch.await.unwrap(); assert_eq!(served.index(), 0); assert_eq!( served.read_frame().await.unwrap().unwrap().payload, bytes::Bytes::from_static(b"head") ); } /// A fetch for the head of a live group the feed started partway into keeps that group /// in arrival order: subscribers get the fetched copy while it lasts, and the live /// group again once the fetch is gone. #[kio::test] #[cfg(feature = "async")] async fn fetch_fills_the_head_of_a_live_group() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } assert_eq!(fetch.await.unwrap().index(), 0); // Arrival order hands out the fetched copy, from its first frame. let mut sub = producer.subscribe(None); let mut group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); assert_eq!( group.read_frame().await.unwrap().unwrap().payload, bytes::Bytes::from_static(b"snapshot") ); // An abandoned fetch leaves the live group in its slot, still written by the feed. head.abort(Error::Cancel).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta2")) .unwrap(); let mut sub = producer.subscribe(None); let group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); // Positioning at the start clamps up to the live group's first frame. let mut group = group.with_range(0..); assert_eq!(group.index(), 1); assert_eq!( group.read_frame().await.unwrap().unwrap().payload, bytes::Bytes::from_static(b"delta") ); } /// A bounded read of a live group the feed started partway into asks its head as a fill, /// and reads the filled copy from its first frame rather than lagging. #[kio::test] #[cfg(feature = "async")] async fn a_bounded_read_fills_the_head_of_a_live_group() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); live.finish().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..1)); let reading = kio::clock::sim::spawn(async move { let mut group = sub.recv_group().await.unwrap().expect("the group"); let mut frames = Vec::new(); while let Some(frame) = group.read_frame().await.unwrap() { frames.push(frame.payload); } (frames, sub.recv_group().await.unwrap().is_none()) }); let request = kio::clock::timeout(std::time::Duration::from_secs(1), dynamic.requested_fill_group()) .await .expect("the head is asked") .unwrap(); assert_eq!(request.frame_start(), 0); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } head.finish().unwrap(); let (frames, ended) = reading.await.unwrap(); assert_eq!(frames, [&b"snapshot"[..], b"delta"]); assert!(ended, "the read ends with its range"); } /// A head fill is judged against the reader's window, as the cursor judges a cached /// group: a newer group outside the window leaves it fresh, even with no max delay. #[kio::test] #[cfg(feature = "async")] async fn a_head_fill_is_judged_against_the_reader_window() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); live.finish().unwrap(); let mut newer = producer.create_group(group::Info { start: 1 }).unwrap(); newer .write_frame(Timestamp::from_secs(1).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); newer.finish().unwrap(); let mut sub = producer.subscribe(Subscription::default().with_range(0..1)); let reading = kio::clock::sim::spawn(async move { let group = sub.recv_group().await.unwrap().map(|group| group.start); (group, sub.recv_group().await.unwrap().is_none()) }); let request = kio::clock::timeout(std::time::Duration::from_secs(1), dynamic.requested_fill_group()) .await .expect("the head is asked") .unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } head.finish().unwrap(); assert_eq!(reading.await.unwrap(), (Some(0), true)); } /// A head still asked when a newer group in the reader's window supersedes it is /// written off, so the bounded read ends without waiting on the fill, whether or not the /// cache still holds the headless copy. #[kio::test] #[cfg(feature = "async")] async fn a_head_still_asked_goes_stale_within_the_window() { for evict in [false, true] { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); live.finish().unwrap(); let mut sub = producer.subscribe(Subscription::default().with_range(0..2)); assert!(sub.recv_group().now_or_never().is_none(), "the head is still asked"); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(request.key(), 0); let mut newer = producer.create_group(group::Info { start: 1 }).unwrap(); newer .write_frame(Timestamp::from_millis(1).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); newer.finish().unwrap(); if evict { producer.modify().unwrap().evict_group(0, Error::Evicted); } let group = kio::clock::timeout(std::time::Duration::from_secs(1), sub.recv_group()) .await .expect("group 1 timed out") .unwrap() .expect("group 1"); assert_eq!(group.start, 1, "evict={evict}"); let end = kio::clock::timeout(std::time::Duration::from_secs(1), sub.recv_group()) .await .unwrap_or_else(|_| panic!("evict={evict}: the read never ended")) .unwrap(); assert!(end.is_none(), "evict={evict}: the read ends with group 0 written off"); drop(request); } } /// Answered heads go out in the subscription's order, as owed and filled groups do. #[kio::test] #[cfg(feature = "async")] async fn answered_heads_go_out_in_the_subscription_order() { for (order, expected) in [(Order::Ascending, [0, 1]), (Order::Descending, [1, 0])] { let producer = track_producer("test", None); let dynamic = producer.requests(); for key in 0..2 { let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(1).unwrap(); live.write_frame( crate::Timestamp::unscaled(live.start), bytes::Bytes::from_static(b"delta"), ) .unwrap(); live.finish().unwrap(); } let mut sub = producer.subscribe(replay().with_order(order).with_range(0..2)); assert!( sub.recv_group().now_or_never().is_none(), "{order:?}: the heads are asked" ); let mut requests = Vec::new(); while let Some(request) = dynamic.requested_fill_group().now_or_never() { requests.push(request.unwrap()); } assert_eq!(requests.len(), 2, "{order:?}: both heads are asked"); for request in requests { let mut head = request.accept(Info::default()).unwrap().expect("a fresh group"); head.write_frame( crate::Timestamp::unscaled(head.start), bytes::Bytes::from_static(b"snapshot"), ) .unwrap(); head.finish().unwrap(); } let mut got = Vec::new(); for _ in 0..2 { let group = sub.recv_group().now_or_never().unwrap().unwrap().expect("a head"); got.push(group.start); } assert_eq!(got, expected, "{order:?}"); assert!( matches!(sub.recv_group().now_or_never(), Some(Ok(None))), "{order:?}: the read ends" ); } } /// A head answered after the reader's ranges moved off its group is not handed out. #[kio::test] #[cfg(feature = "async")] async fn a_head_answered_outside_the_ranges_is_not_handed_out() { let producer = track_producer("test", None); let dynamic = producer.requests(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame( crate::Timestamp::unscaled(live.start), bytes::Bytes::from_static(b"delta"), ) .unwrap(); live.finish().unwrap(); framed(&producer, 1, 1).finish().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..2)); let group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(group.start, 1); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(request.key(), 0); sub.update(replay().with_range(1..2)).unwrap(); let mut head = request.accept(Info::default()).unwrap().expect("a fresh group"); head.write_frame( crate::Timestamp::unscaled(head.start), bytes::Bytes::from_static(b"snapshot"), ) .unwrap(); head.finish().unwrap(); assert!( matches!(sub.recv_group().now_or_never(), Some(Ok(None))), "the narrowed read ends without group 0" ); } /// An ordered bounded read whose head no fill serves gives that group up and reads on, /// rather than handing it out to lag. #[kio::test] #[cfg(feature = "async")] async fn a_bounded_read_gives_up_a_group_whose_head_is_refused() { let producer = track_producer("test", None); let dynamic = producer.requests(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame( crate::Timestamp::unscaled(live.start), bytes::Bytes::from_static(b"delta"), ) .unwrap(); live.finish().unwrap(); framed(&producer, 1, 1).finish().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..2)).ordered(); let reading = kio::clock::sim::spawn(async move { let mut read = Vec::new(); while let Some(item) = sub.next_group().await.unwrap() { read.push(match item { Item::Group(group) => format!("group {}", group.start), Item::Gap { range, .. } => format!("gap {range:?}"), }); } read }); let request = kio::clock::timeout(std::time::Duration::from_secs(1), dynamic.requested_fill_group()) .await .expect("the head is asked") .unwrap(); request.reject(Error::NotFound); assert_eq!(reading.await.unwrap(), ["gap 0..1", "group 1"]); } /// A refused mid-group end caps none of its group's copies, even one that fit under it. #[kio::test] #[cfg(feature = "async")] async fn a_refused_end_leaves_every_copy_uncapped() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); // The live copy holds frames 0 and 1, the fetched head frames 0 to 2. let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), b"delta".as_slice()).unwrap(); let fetch = consumer.fetch_group(0, None); let mut head = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); for index in 0..3 { let payload = if index == 0 { b"x".as_slice() } else { b"delta".as_slice() }; head.write_frame(head.key_time(), payload).unwrap(); } fetch.await.unwrap(); assert!(matches!( producer.finish_at(Position { group: 0, frame: 2 }), Err(Error::ProtocolViolation) )); assert_eq!(producer.finished(), None); live.write_frame(live.key_time(), b"delta".as_slice()).unwrap(); } /// A live group with a fetched head: group 0, fed from frame 1, plus a two-frame head. #[cfg(feature = "async")] async fn live_group_with_head(producer: &Producer) -> (group::Producer, group::Finished) { let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } let head = head.finish().unwrap(); assert_eq!(fetch.await.unwrap().index(), 0); (live, head) } /// A live group with a fetched head of group 0 still writing, and the track's end /// declared past it. The live copy ended as `live` says: aborted with it, else finished. #[cfg(feature = "async")] async fn live_group_under_a_writing_head(live: Option) -> (Producer, group::Producer) { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); group.start_at(1).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let mut head = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } match live { Some(err) => group.abort(err).unwrap(), None => { group.finish().unwrap(); } } producer.finish_at(Position::group(1)).unwrap(); (producer, head) } /// The fetched head serving a group to its end settles the declared end once it finishes, /// whether the live copy beside it was cut, evicted, or finished, and a later abort keeps /// that clean end, whether `closed()` saw it first or only the abort did. #[kio::test] #[cfg(feature = "async")] async fn a_fetched_head_settles_the_end_once_it_finishes() { use std::sync::atomic::{AtomicBool, Ordering}; for live in [Some(Error::Cancel), Some(Error::Evicted), None] { for polled in [false, true] { let (producer, head) = live_group_under_a_writing_head(live.clone()).await; let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); if polled { assert!( producer.poll_closed(&waiter).is_pending(), "{live:?}: the head still writes" ); } head.finish().unwrap(); if polled { assert!( woken.load(Ordering::SeqCst), "{live:?}: the head finishing wakes the wait" ); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(()))), "{live:?}"); } producer.clone().abort(Error::Timeout).unwrap(); let noop = kio::Waiter::noop(); assert!( matches!(producer.poll_closed(&noop), Poll::Ready(Ok(()))), "{live:?} polled={polled}" ); assert!( matches!(producer.consume().poll_complete(&noop), Poll::Ready(Ok(()))), "{live:?} polled={polled}" ); } } } /// A fetched head aborting hands its group back to the live copy: a finished one settles /// the end, while a cut one leaves nothing that delivered the group, so an abort still /// cuts the track off. #[kio::test] #[cfg(feature = "async")] async fn an_aborted_head_hands_the_end_back_to_the_live_copy() { use std::sync::atomic::{AtomicBool, Ordering}; for (live, settles) in [(None, true), (Some(Error::App(5)), false)] { let (producer, head) = live_group_under_a_writing_head(live.clone()).await; let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_closed(&waiter).is_pending(), "{live:?}"); head.abort(Error::Cancel).unwrap(); assert!( woken.load(Ordering::SeqCst), "{live:?}: the head aborting wakes the wait" ); assert_eq!(producer.poll_closed(&waiter).is_ready(), settles, "{live:?}"); producer.clone().abort(Error::Timeout).unwrap(); let outcome = producer.poll_closed(&kio::Waiter::noop()); match settles { true => assert!(matches!(outcome, Poll::Ready(Ok(()))), "{live:?}: {outcome:?}"), false => assert!( matches!(outcome, Poll::Ready(Err(Error::Timeout))), "{live:?}: {outcome:?}" ), } } } /// A finished fetched head serves its group to the end, so the declared end settles while /// the live copy beside it is still written, and that copy aborting later keeps it clean. #[kio::test] #[cfg(feature = "async")] async fn a_finished_head_settles_the_end_over_an_open_live_copy() { let mut producer = track_producer("test", Info::timed()); let consumer = producer.consume(); let noop = kio::Waiter::noop(); let (live, _head) = live_group_with_head(&producer).await; producer.finish_at(Position::group(1)).unwrap(); assert!(matches!(producer.poll_closed(&noop), Poll::Ready(Ok(())))); live.abort(Error::Cancel).unwrap(); producer.clone().abort(Error::Timeout).unwrap(); assert!(matches!(producer.poll_closed(&noop), Poll::Ready(Ok(())))); assert!(matches!(consumer.poll_complete(&noop), Poll::Ready(Ok(())))); } /// A fetch below the end that was cancelled, or failed, before the live feed delivered its /// group holds nothing the feed promised, so the end settles at once rather than once the /// expiry scan reclaims its slot, and a later abort keeps that clean end. #[kio::test] #[cfg(feature = "async")] async fn an_aborted_fetch_below_the_end_does_not_hold_it() { use std::sync::atomic::{AtomicBool, Ordering}; for (err, parked) in [(Error::Cancel, false), (Error::Cancel, true), (Error::App(5), true)] { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let noop = kio::Waiter::noop(); // The feed delivered group 1 and never group 0, which a fetch asks of the source. producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); let fetch = consumer.fetch_group(0, None); let mut fetched = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); fetched .write_frame(fetched.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); if parked { producer.finish_at(Position::group(2)).unwrap(); assert!( producer.poll_closed(&waiter).is_pending(), "{err:?}: the fetch still writes" ); } fetched.abort(err.clone()).unwrap(); drop(fetch); if parked { assert!( woken.load(Ordering::SeqCst), "{err:?}: the fetch failing wakes the wait" ); } else { producer.finish_at(Position::group(2)).unwrap(); } assert!( matches!(producer.poll_closed(&noop), Poll::Ready(Ok(()))), "{err:?} parked={parked}: the end settles" ); producer.clone().abort(Error::Timeout).unwrap(); assert!(matches!(producer.poll_closed(&noop), Poll::Ready(Ok(()))), "{err:?}"); assert!(matches!(consumer.poll_complete(&noop), Poll::Ready(Ok(()))), "{err:?}"); } } /// A fetched head still writing holds the largest stream open after the live copy /// aborted and the track ended, so a later head frame is reported rather than lost. #[kio::test] #[cfg(feature = "async")] async fn a_fetched_head_holds_the_largest_open() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut reader = consumer.clone(); let noop = kio::Waiter::noop(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); assert_eq!( reader.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 1 })) ); let _fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap().expect("a fresh group"); for payload in [&b"snapshot"[..], b"delta"] { head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } live.abort(Error::Cancel).unwrap(); let _producer = producer.finish().unwrap(); assert!(reader.poll_largest(&noop).is_pending(), "the head still writes"); head.write_frame(head.key_time(), bytes::Bytes::from_static(b"more")) .unwrap(); assert_eq!( reader.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 2 })) ); head.finish().unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); } #[kio::test] #[cfg(feature = "async")] async fn a_live_abort_does_not_remove_a_parked_fetched_head() { let producer = track_producer("test", Info::timed()); let (live, _head) = live_group_with_head(&producer).await; let mut subscriber = producer.subscribe(replay()); subscriber.set_range(..0); assert!(subscriber.poll_recv_group(&kio::Waiter::noop()).is_pending()); live.abort(Error::Cancel).unwrap(); assert!(subscriber.poll_recv_group(&kio::Waiter::noop()).is_pending()); subscriber.set_range(..); let mut group = subscriber.recv_group().await.unwrap().unwrap(); assert_eq!( group.read_frame().await.unwrap().unwrap().payload, bytes::Bytes::from_static(b"snapshot") ); } /// The live copy ending early (its subscription went away) leaves a fetched head /// that still holds the whole group servable, instead of reclaiming the slot. #[kio::test] #[cfg(feature = "async")] async fn aborted_live_group_keeps_its_fetched_head() { let producer = track_producer("test", Info::timed()); let (live, _head) = live_group_with_head(&producer).await; live.abort(Error::Cancel).unwrap(); // The next group demotes 0 into the eviction order and runs an expiry scan. let _open = producer.create_group(group::Info { start: 1 }).unwrap(); let mut group = producer.consume().peek_group(0).expect("the head still serves group 0"); assert_eq!( group.read_frame().await.unwrap().unwrap().payload, bytes::Bytes::from_static(b"snapshot") ); } /// A group the publisher cut while a fetched copy still serves its end is delivered, /// and never also reported dropped. #[kio::test] #[cfg(feature = "async")] async fn a_cut_a_fetched_copy_covers_is_not_reported() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); let (live, _head) = live_group_with_head(&producer).await; live.abort(Error::Stream(StreamError::App(5))).unwrap(); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_drop(&waiter).is_pending(), "the copy covers the cut"); assert_eq!(drain_groups(&mut subscriber), [0]); } /// Reads land on the fetched head, so they keep the slot from expiring even though /// the live copy beside it sits idle. #[kio::test] #[cfg(feature = "async")] async fn reading_the_head_keeps_the_slot_from_expiring() { let producer = track_producer("test", Info::timed()); let (live, _head) = live_group_with_head(&producer).await; live.finish().unwrap(); let _open = producer.create_group(group::Info { start: 1 }).unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // A fetch hit on the head, past the window the live copy was last written in. producer.consume().fetch_group(0, None).await.unwrap(); let _open = producer.create_group(group::Info { start: 2 }).unwrap(); assert!(producer.consume().peek_group(0).is_some(), "a read head keeps its slot"); } /// Fill group 0 as a live copy plus a fetched head written after it, optionally /// abort the live copy, then write past capacity into a protected latest group, so /// slot 0 is the only evictable content. Returns whether pressure evicted it. #[cfg(feature = "async")] async fn pressure_evicts_a_headed_slot(abort_live: bool) -> bool { let (producer, _pool) = pooled_producer(10_000); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let live = live.finish().unwrap(); let fetch = consumer.fetch_group(0, None); let mut head = dynamic .requested_fill_group() .await .unwrap() .accept(Info::timed()) .unwrap() .expect("a fresh group"); head.write_frame(head.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); head.finish().unwrap(); drop(fetch.await.unwrap()); if abort_live { live.abort(Error::Cancel).unwrap(); } let mut next = producer.create_group(group::Info { start: 1 }).unwrap(); for _ in 0..30 { next.write_frame(next.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); } consumer.peek_group(0).is_none() } /// A head left alone in its slot still joins the pool's access average when the /// slot is demoted, so memory pressure can evict it. #[kio::test] #[cfg(feature = "async")] async fn a_head_only_slot_yields_to_memory_pressure() { assert!(pressure_evicts_a_headed_slot(true).await); } /// A slot weighs its copies the way the pool's average samples them, so two copies /// with different access stamps can't keep it above the average on their own. #[kio::test] #[cfg(feature = "async")] async fn a_two_copy_slot_yields_to_memory_pressure() { assert!(pressure_evicts_a_headed_slot(false).await); } /// Joining a queued fetch widens its range. A caller that arrives once the range is /// already on the wire asks for its own frames rather than being handed the narrower group. #[kio::test] #[cfg(feature = "async")] async fn fetch_widens_or_asks_afresh() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let from = |frame_start| group::Fetch { frame_start, ..Default::default() }; let mut narrow = std::pin::pin!(consumer.fetch_group(0, from(5))); assert!(futures::poll!(narrow.as_mut()).is_pending()); // Still queued: widening is honored, because nothing has read the range yet. let mut wider = std::pin::pin!(consumer.fetch_group(0, from(2))); assert!(futures::poll!(wider.as_mut()).is_pending()); let request = dynamic.requested_fill_group().await.unwrap(); assert_eq!(request.frame_start(), 2, "widened while queued"); // Handed off now: the handler holds its own copy of the range, so a later wider // caller cannot move what is already being served. It asks for its own. let mut widest = std::pin::pin!(consumer.fetch_group(0, from(0))); assert!(futures::poll!(widest.as_mut()).is_pending()); assert_eq!(request.frame_start(), 2, "the in-flight range is already on the wire"); let whole = dynamic.requested_fill_group().await.unwrap(); assert_eq!( (whole.key(), whole.frame_start()), (0, 0), "the earlier frames are asked" ); // A handler may keep tracking the group's readers past the accept, as the lite // subscriber does, which holds the result channel open. let _readers = request.readers(); let mut group = request.accept(Info::timed()).unwrap().expect("a fresh group"); // A handler numbers the frames from where it was asked to start, as `serve_fetch` // does; that offset is what makes the cached group too narrow for `widest`. group.start_at(2).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"from2")) .unwrap(); // The two callers it covers resolve; the one it doesn't waits on its own fill // rather than being handed a group that starts above what it asked for. assert_eq!(narrow.await.unwrap().index(), 5); assert_eq!(wider.await.unwrap().index(), 2); assert!(futures::poll!(widest.as_mut()).is_pending()); group.finish().unwrap(); let mut group = whole.accept(None).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"from0")) .unwrap(); assert_eq!(widest.await.unwrap().index(), 0); group.finish().unwrap(); } #[kio::test] #[cfg(feature = "async")] async fn fetch_coalesces_concurrent() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); // Two fetches for the same uncached group produce ONE handler request, // carrying the higher of the two priorities. let mut first = consumer.fetch_group( 5, group::Fetch::default() .with_priority(1) .with_max_delay(Duration::from_secs(1)), ); let second = consumer.fetch_group( 5, group::Fetch::default() .with_priority(7) .with_max_delay(Duration::from_secs(10)), ); assert!(kio::Task::poll(&mut *first, &kio::Waiter::noop()).is_pending()); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); assert_eq!(req.key(), 5); assert_eq!(req.priority(), 7); assert_eq!(req.max_delay(), Some(Duration::from_secs(10))); assert!( dynamic.poll_requested_fill_group(&kio::Waiter::noop()).is_pending(), "the second fetch queued a duplicate request" ); // A fetch arriving while the request is already in flight joins it too. let third = consumer.fetch_group(5, None); // One accept resolves all of them. let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); group.finish().unwrap(); assert_eq!(first.await.unwrap().start, 5); assert_eq!(second.await.unwrap().start, 5); assert_eq!(third.await.unwrap().start, 5); } /// A looser caller that joined after a tighter budget went on the wire is not failed by /// that budget's `Old`: it asks again with its own. #[kio::test] #[cfg(feature = "async")] async fn a_late_looser_fetch_asks_again_after_old() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let tight = consumer.fetch_group(5, group::Fetch::default().with_max_delay(Duration::from_secs(1))); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(req.max_delay(), Some(Duration::from_secs(1))); let mut loose = consumer.fetch_group(5, None); assert!(kio::Task::poll(&mut *loose, &kio::Waiter::noop()).is_pending()); req.reject(Error::Old); assert!(matches!(tight.await, Err(Error::Old))); assert!(kio::Task::poll(&mut *loose, &kio::Waiter::noop()).is_pending()); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(req.max_delay(), None); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); group.finish().unwrap(); assert_eq!(loose.await.unwrap().start, 5); } #[kio::test] #[cfg(feature = "async")] async fn fetch_coalesced_reject_fails_all() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(5, None); let second = consumer.fetch_group(5, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); req.reject(Error::Cancel); assert!(matches!(first.await, Err(Error::Cancel))); assert!(matches!(second.await, Err(Error::Cancel))); // The rejected attempt is gone: a retry starts a fresh one. let mut retry = consumer.fetch_group(5, None); assert!(kio::Task::poll(&mut *retry, &kio::Waiter::noop()).is_pending()); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); assert_eq!(req.key(), 5); } #[kio::test] #[cfg(feature = "async")] async fn fetch_queued_fails_when_handlers_leave() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); // Queued but never popped: the last handler leaving fails it fast. let mut pending = consumer.fetch_group(5, None); assert!(kio::Task::poll(&mut *pending, &kio::Waiter::noop()).is_pending()); drop(dynamic); assert!(matches!(pending.await, Err(Error::NotFound))); // And the attempt didn't leak. let fetch = producer.state.read().fills.clone(); assert!(fetch.read().is_empty()); } #[kio::test] #[cfg(feature = "async")] async fn fetch_miss_no_dynamic_not_found() { // A track with no `Requests` can't serve old content, so a cache miss // resolves to NotFound instead of blocking forever. let producer = track_producer("test", Info::timed()); let _open = producer.append_next().unwrap(); // seq 0, but we miss on seq 5 let consumer = producer.consume(); assert!(matches!(consumer.fetch_group(5, None).await, Err(Error::NotFound))); } #[kio::test] #[cfg(feature = "async")] async fn fetch_past_final_not_found() { let producer = track_producer("test", Info::timed()); let _open = producer.append_next().unwrap(); // seq 0 producer.finish_shared().unwrap(); // final_key = 1 // A group at or past the final key can never exist, even with a handler, // so it resolves to NotFound. let dynamic = producer.requests(); let consumer = producer.consume(); assert!(matches!(consumer.fetch_group(5, None).await, Err(Error::NotFound))); // And it doesn't signal the dynamic handler. assert!(dynamic.poll_requested_fill_group(&kio::Waiter::noop()).is_pending()); } /// Mint a track whose groups charge into a bounded [`cache::Pool`]. fn pooled_producer(capacity: u64) -> (Producer, cache::Pool) { let config = cache::Config::default() .with_capacity(capacity) .with_expiry(cache::DEFAULT_EXPIRY); let pool = cache::Pool::new(config); let broadcast = broadcast::Info { pool: pool.clone(), ..Default::default() }; let producer = Producer::new(Arc::new(broadcast), "test", Info::timed()); (producer, pool) } fn finished_group(producer: &mut Producer, size: usize) -> u64 { let mut group = producer.append_next().unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; size])) .unwrap(); let group = group.finish().unwrap(); group.start } /// While the pool is over capacity, every append accrues debt and pays it by /// evicting this track's own oldest groups, so the newest content survives. #[test] fn debt_evicts_oldest_group() { // Fits one 10k group; each additional group pushes the pool over budget. let (mut producer, pool) = pooled_producer(10_000); finished_group(&mut producer, 10_000); // seq 0 finished_group(&mut producer, 10_000); // seq 1: over budget, debt starts accruing finished_group(&mut producer, 10_000); // seq 2: pays by evicting seq 0 let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none(), "oldest group is evicted"); assert!(consumer.peek_group(2).is_some(), "latest group survives"); // Steady state carries the protected live edge plus the just-demoted group // (debt is charged before the demotion, so eviction lags one append). assert!( pool.used() <= 2 * (10_000 + cache::ENTRY_OVERHEAD), "usage hovers near capacity: {}", pool.used() ); // A fresh subscriber skips the evicted groups entirely. let mut subscriber = producer.subscribe(replay()); assert!(subscriber.assert_group().start > 0, "evicted group is not delivered"); } /// A plain reader inflating a finished compressed track pays the debt its inflated copies /// add, though no writer is left to: the pool returns to its capacity. #[test] #[cfg(feature = "async")] fn inflating_settles_the_pool_debt() { let capacity = 1 << 20; let pool = cache::Pool::new(cache::Config::default().with_capacity(capacity)); let broadcast = Arc::new(broadcast::Info { pool: pool.clone(), ..Default::default() }); let producer = Producer::new(broadcast, "test", Info::timed().with_compression(Compression::Deflate)); let frame = bytes::Bytes::from(vec![0u8; 256 * 1024]); let groups = 16; for _ in 0..groups { let mut group = producer.append_next().unwrap(); group.write_frame(group.key_time(), frame.clone()).unwrap(); group.finish().unwrap(); } assert!(pool.used() < capacity, "the slices fit: {}", pool.used()); let consumer = producer.consume(); for key in 0..groups { let Some(mut group) = consumer.peek_group(key) else { continue; }; match group.read_frame().now_or_never().expect("cached") { Ok(Some(read)) => assert_eq!(read.payload.len(), frame.len()), Ok(None) => panic!("group {key} ended without its frame"), Err(err) => assert!(matches!(err, Error::Evicted), "group {key}: {err}"), } } // The live edge and the group being read stay, whatever the budget. let slack = 2 * (frame.len() as u64 + cache::ENTRY_OVERHEAD); assert!( pool.used() <= capacity + slack, "inflated copies left the pool over capacity: {}", pool.used() ); } /// The latest group is never in the eviction order, so it survives any budget. #[kio::test] #[cfg(feature = "async")] async fn latest_group_never_evicted() { // Far too small for even one group: the latest survives anyway. let (mut producer, pool) = pooled_producer(100); finished_group(&mut producer, 1000); // seq 0 assert!(pool.used() > 100, "the latest may exceed the budget"); // Later writes evict the demoted seq 0; each new latest is untouchable in turn. finished_group(&mut producer, 1000); // seq 1: demotes seq 0 finished_group(&mut producer, 1000); // seq 2: pays by evicting seq 0 let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none()); let mut group = consumer.peek_group(2).expect("latest survives"); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.len(), 1000); } /// A FETCH cache hit refreshes the group's access time: anything accessed more /// recently than the pool-wide average is protected, so the eviction walk skips /// it and evicts a never-read group instead, even one that arrived later. #[kio::test] #[cfg(feature = "async")] async fn fetch_refresh_survives_eviction() { let (mut producer, _pool) = pooled_producer(11_000); let consumer = producer.consume(); finished_group(&mut producer, 3_000); // seq 0 elapse(&producer, Duration::from_secs(1)); finished_group(&mut producer, 3_000); // seq 1 elapse(&producer, Duration::from_secs(1)); finished_group(&mut producer, 3_000); // seq 2 elapse(&producer, Duration::from_millis(500)); // FETCH seq 0: the cache hit lifts its access time above the average. let mut fetched = consumer.fetch_group(0, None).await.unwrap(); assert_eq!(fetched.read_frame().await.unwrap().unwrap().payload.len(), 3_000); elapse(&producer, Duration::from_millis(500)); // Pressure: seq 0 is first in eviction order but freshly accessed, so it // rotates to the back and the never-read seq 1 dies instead. finished_group(&mut producer, 3_000); // seq 3 elapse(&producer, Duration::from_secs(1)); finished_group(&mut producer, 3_000); // seq 4 assert!(consumer.peek_group(0).is_some(), "refreshed group survives"); assert!(consumer.peek_group(1).is_none(), "unread group is evicted instead"); } /// A consumer holding an evicted group surfaces the eviction, not a hang or a /// truncated clean end. #[kio::test] #[cfg(feature = "async")] async fn eviction_aborts_readers() { let (mut producer, _pool) = pooled_producer(10_000); let mut subscriber = producer.subscribe(None); finished_group(&mut producer, 10_000); // seq 0 let mut group0 = subscriber.assert_group(); finished_group(&mut producer, 10_000); // seq 1: demotes seq 0 finished_group(&mut producer, 10_000); // seq 2: pays by evicting seq 0 let read = group0.read_frame().await; assert!(matches!(read, Err(Error::Evicted)), "expected Evicted, got {read:?}"); } /// Waiting for enough debt to evict an old group must not shed the live group's /// frames instead: its reader has not had a chance to take them yet. #[kio::test] #[cfg(feature = "async")] async fn carried_debt_keeps_the_live_group() { let unit = 100 * cache::ENTRY_OVERHEAD; let (mut producer, _) = pooled_producer(22 * unit); finished_group(&mut producer, 20 * unit as usize); let mut group = producer.append_next().unwrap(); let mut reader = group.consume(); for _ in 0..3 { group.write_frame(group.key_time(), vec![0u8; unit as usize]).unwrap(); } group.finish().unwrap(); for _ in 0..3 { assert_eq!(reader.read_frame().await.unwrap().unwrap().payload.len(), unit as usize); } assert!(reader.read_frame().await.unwrap().is_none()); } /// A write smaller than the next victim carries debt instead of evicting: a /// large group dies only once enough debt accumulates, never to pay off a /// far smaller write. #[test] fn small_writes_carry_debt() { // Payloads dwarf the fixed per-group charge, so the budget arithmetic below is // about bytes written rather than bookkeeping. let unit = 100 * cache::ENTRY_OVERHEAD; let (mut producer, pool) = pooled_producer(22 * unit); let consumer = producer.consume(); finished_group(&mut producer, 20 * unit as usize); // seq 0, the large victim-to-be // The first few small writes owe far less than seq 0's size: the debt // carries over instead of evicting it. for _ in 0..3 { finished_group(&mut producer, unit as usize); } assert!(consumer.peek_group(0).is_some(), "debt smaller than the victim carries"); // Enough small writes accumulate the debt to finally evict it. for _ in 0..20 { finished_group(&mut producer, unit as usize); } assert!( consumer.peek_group(0).is_none(), "accumulated debt evicts the large group" ); // Steady state hovers within about one group of capacity: a victim smaller // than the outstanding debt is never evicted, so the excess stays bounded. assert!(pool.used() <= 24 * unit, "usage hovers near capacity: {}", pool.used()); } /// A capacity shrink reclaims a failed group's held head before any cached group, even /// in a pool with no idle window and once the failed copy's track is gone. #[test] fn a_shrink_reclaims_held_heads_first() { let pool = cache::Pool::new(cache::Config::default().with_capacity(1 << 40)); let broadcast = Arc::new(broadcast::Info { pool: pool.clone(), ..Default::default() }); let failed = Producer::new(broadcast.clone(), "failed", Info::timed()); let mut group = failed.append_next().unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1_000])) .unwrap(); let handoff = group.consume().handoff(); group.abort(crate::Error::Cancel).unwrap(); drop(failed); assert!(handoff.head().is_some(), "held for a successor route"); let mut producer = Producer::new(broadcast, "live", Info::timed()); finished_group(&mut producer, 1_000); // Room for the cached group, not the held head beside it. pool.resize(pool.used() - 1); finished_group(&mut producer, 1_000); assert!( handoff.head().is_none(), "the shrink left a stalled resumer's head held" ); assert!(producer.consume().peek_group(0).is_some(), "a cached group went first"); } /// One write pays at most twice what it produced, so a capacity shrink (or one /// track's burst) drains gradually instead of one writer dumping its whole /// backlog in a single call. #[test] fn payment_capped_per_write() { let (mut producer, pool) = pooled_producer(1 << 40); for _ in 0..10 { finished_group(&mut producer, 1_000); } // The governor slashes the target; nothing is reclaimed synchronously. pool.resize(100); let before = pool.used(); // One 1k write may evict at most ~2k of backlog, not all ten groups. finished_group(&mut producer, 1_000); let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none(), "the oldest groups are evicted"); assert!(consumer.peek_group(1).is_none()); assert!(consumer.peek_group(2).is_some(), "the backlog drains gradually"); assert!(pool.used() > before - 4_000, "one write must not dump the backlog"); } /// Accepting a track after pre-accept backfill must keep the same write /// counter: the counter is owned by the track state, so replacing the info /// can't strand the bytes already-created groups keep charging. #[kio::test] #[cfg(feature = "async")] async fn accept_preserves_write_accounting() { let config = cache::Config::default() .with_capacity(12_000) .with_expiry(cache::DEFAULT_EXPIRY); let pool = cache::Pool::new(config); let broadcast = broadcast::Info { pool: pool.clone(), ..Default::default() }; let request = Request::new(Arc::new(broadcast), "test"); let dynamic = request.requests(); let consumer = request.consume(); // Serve a backfill before the track is accepted, then grow it. let pending = consumer.fetch_group(0, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut backfill = req.accept(Info::timed()).unwrap().expect("a fresh group"); pending.await.unwrap(); backfill .write_frame(backfill.key_time(), bytes::Bytes::from(vec![0u8; 30_000])) .unwrap(); // Accept with a fresh Info: the pre-accept group's writes must still be // drained by this track's future charges. let producer = request.accept(Info::timed()); producer.append_next().unwrap().finish().unwrap(); producer.append_next().unwrap().finish().unwrap(); assert!( producer.consume().peek_group(0).is_none(), "pre-accept backfill growth is reclaimed after accept" ); assert!(pool.used() <= 13_000, "usage converges: {}", pool.used()); } /// Re-serving a key many times must not accumulate eviction hints: stale /// hints die on stamp mismatch and compaction reclaims them. #[test] fn recreated_key_bounds_eviction_hints() { let (producer, _pool) = pooled_producer(1 << 40); producer.create_group(5u64.into()).unwrap().finish().unwrap(); for _ in 0..200 { let group = producer.create_group(1u64.into()).unwrap(); group.abort(Error::Cancel).unwrap(); } let state = producer.state.read(); assert!( state.evict.len() <= 2 * state.lookup.len() + EVICT_SLACK, "stale hints are compacted: {} entries for {} slots", state.evict.len(), state.lookup.len() ); } /// A frame write within the same coarse tick still outranks merely-inserted /// content, so the freshly-written group survives and the empty one pays. #[test] fn same_tick_write_outranks_inserted() { // Payloads dwarf the fixed per-group charge, so the budget arithmetic below is // about bytes written rather than bookkeeping. let unit = 100 * cache::ENTRY_OVERHEAD; // No time advances: every stamp lands in the same tick. let (mut producer, _pool) = pooled_producer(10 * unit); producer.append_next().unwrap().finish().unwrap(); // seq 0: empty finished_group(&mut producer, 3 * unit as usize); // seq 1: written finished_group(&mut producer, 3 * unit as usize); // seq 2 finished_group(&mut producer, 3 * unit as usize); // seq 3 finished_group(&mut producer, 3 * unit as usize); // seq 4: over budget, pays let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none(), "insert-only content pays first"); assert!(consumer.peek_group(1).is_some(), "same-tick written content survives"); } /// A track that only appends frames to an open group, never inserting another /// group, still settles its eviction debt once enough bytes accumulate. #[test] fn frame_only_writer_pays() { let (producer, pool) = pooled_producer(2_000); let mut demoted = producer.append_next().unwrap(); // seq 0 producer.append_next().unwrap().finish().unwrap(); // seq 1 demotes seq 0 // One large frame crosses the charge threshold: the write itself pays, // with no further group insert on this track. demoted .write_frame(demoted.key_time(), bytes::Bytes::from(vec![0u8; 300_000])) .unwrap(); assert!( pool.used() <= 5_000, "the frame write settled the debt: {}", pool.used() ); assert!(matches!(demoted.finish(), Err(Error::Evicted))); } /// One `Info` describing several tracks must not join their eviction accounting: /// each track opens its own account against the pool. #[test] fn each_track_owns_its_account() { let broadcast = Arc::new(broadcast::Info::default()); let info = Info::timed(); let a = Producer::new(broadcast.clone(), "a", info.clone()); let b = Producer::new(broadcast, "b", info); let a = a.state.read().cache.clone(); let b = b.state.read().cache.clone(); assert!(!Arc::ptr_eq(&a, &b), "each track owns its account"); } /// A `Requests` still serving fetches keeps the track alive, so the publisher /// letting go isn't an abrupt teardown: the handler can still serve the cache. #[test] fn a_dynamic_defers_teardown() { let (mut producer, pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); finished_group(&mut producer, 100); drop(producer); assert!(pool.used() > 0, "the handler still serves the cache"); drop(dynamic); assert_eq!(pool.used(), 0, "the last handle tears it down"); } /// A finished track releases everything once every handle is gone. /// /// Its groups hold the cache account, and the account links back here, so that link /// has to be weak: anything stronger makes the state (and every cached frame in it) /// immortal, even with no producer or consumer left. #[test] fn finished_track_frees_its_cache() { let (mut producer, pool) = pooled_producer(1 << 40); finished_group(&mut producer, 100); let state = producer.state.downgrade(); let producer = producer.finish().unwrap(); drop(producer); assert!(state.upgrade().is_none(), "the track state is freed"); assert_eq!(pool.used(), 0, "so are its cached bytes"); } /// A group settling its eviction debt upgrades the account's weak handle, which /// counts as a producer on the track state. Teardown must not mistake that for a /// surviving publisher, or an abrupt drop silently behaves like a clean finish. #[test] fn teardown_ignores_a_settling_group() { let (producer, pool) = pooled_producer(1 << 40); // Open, so only the abrupt teardown releases it. let mut group = producer.append_next().unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); drop(group); // Stand in for a concurrent `cache::Track::settle`, mid-upgrade. let settling = producer.state.downgrade().upgrade().expect("open"); drop(producer); // What remains is the drop the teardown recorded for the open group and its expiry, held // by the state. assert_eq!( pool.used(), RUN_OVERHEAD + BORN_OVERHEAD, "the abrupt teardown still released the cache" ); drop(settling); assert_eq!(pool.used(), 0, "the state's drop set goes with it"); } /// A subscriber holding one cached group must not pin the whole track: a group /// carries the track's properties by value, not a handle back to its state. #[test] fn cached_group_outlives_its_track() { let (mut producer, pool) = pooled_producer(1 << 40); let key = finished_group(&mut producer, 100); let group = producer.consume().peek_group(key).expect("cached"); let state = producer.state.downgrade(); let producer = producer.finish().unwrap(); drop(producer); assert!(state.upgrade().is_none(), "the track state is freed"); assert!(pool.used() > 0, "the retained group keeps its own bytes"); drop(group); assert_eq!(pool.used(), 0, "which it releases when dropped"); } /// A backfill served before the track was accepted settles its own debt: the /// account exists from the moment the state does, so acceptance replacing the /// `Info` can't leave already-created groups writing for free. #[kio::test] #[cfg(feature = "async")] async fn pre_accept_backfill_settles_late_writes() { let config = cache::Config::default() .with_capacity(2_000) .with_expiry(cache::DEFAULT_EXPIRY); let pool = cache::Pool::new(config); let broadcast = broadcast::Info { pool: pool.clone(), ..Default::default() }; let request = Request::new(Arc::new(broadcast), "test"); let dynamic = request.requests(); let consumer = request.consume(); // Serve backfill seq 0 before the track is accepted. let pending = consumer.fetch_group(0, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut backfill = req.accept(Info::timed()).unwrap().expect("a fresh group"); pending.await.unwrap(); // Accept, then demote the backfill with a live group. let producer = request.accept(Info::timed()); producer.append_next().unwrap().finish().unwrap(); // No further insert: the late write into the demoted backfill is the only // thing that can pay the debt it just took on. backfill .write_frame(backfill.key_time(), bytes::Bytes::from(vec![0u8; 300_000])) .unwrap(); assert!( pool.used() <= 5_000, "the frame write settled the debt: {}", pool.used() ); } /// A late frame write restarts the LRU clock (the window measures time since /// last written or fetched), so an actively-growing group is not expired as /// idle mid-write. #[test] fn write_restarts_retention_clock() { let (producer, _pool) = pooled_producer(1 << 40); let mut straggler = producer.append_next().unwrap(); // seq 0 producer.append_next().unwrap().finish().unwrap(); // seq 1 demotes seq 0 // Idle past the window, then the straggler receives a late frame. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); straggler .write_frame(straggler.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); producer.append_next().unwrap().finish().unwrap(); // seq 2 runs expiry let consumer = producer.consume(); assert!(consumer.peek_group(0).is_some(), "the write restarted the clock"); // Once the writes stop, the group ages out normally. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer.append_next().unwrap().finish().unwrap(); // seq 3 runs expiry assert!(consumer.peek_group(0).is_none(), "idle content still expires"); } /// Continuously refreshed entries at the front of the eviction order must not /// starve expiry of entries behind them: the scan cursor rotates. #[kio::test] #[cfg(feature = "async")] async fn refreshed_front_does_not_starve_expiry() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); producer.create_group(10u64.into()).unwrap().finish().unwrap(); for key in 1..=5u64 { let pending = consumer.fetch_group(key, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); group.finish().unwrap(); pending.await.unwrap(); } // Age everything out, then refresh the first four backfills so they sit // fresh at the front of the eviction order, hiding the expired fifth. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); for key in 1..=4u64 { consumer.fetch_group(key, None).await.unwrap(); } // The rotating cursor reaches the fifth entry within a few writes. for _ in 0..3 { producer.append_next().unwrap().finish().unwrap(); } assert!(consumer.peek_group(5).is_none(), "expired backfill is reclaimed"); assert!(consumer.peek_group(1).is_some(), "refreshed backfill survives"); } /// A publisher re-creating an aborted key is delivered exactly once, at /// its actual arrival position: the historical arrival entry is dead. #[test] fn recreated_key_delivered_once() { let (producer, _pool) = pooled_producer(1 << 40); producer.create_group(0u64.into()).unwrap().finish().unwrap(); let aborted = producer.create_group(1u64.into()).unwrap(); aborted.abort(Error::Cancel).unwrap(); producer.create_group(2u64.into()).unwrap().finish().unwrap(); producer.create_group(1u64.into()).unwrap().finish().unwrap(); let mut subscriber = producer.subscribe(replay()); assert_eq!(subscriber.assert_group().start, 0); assert_eq!(subscriber.assert_group().start, 2); assert_eq!( subscriber.assert_group().start, 1, "replacement arrives at its own position" ); subscriber.assert_no_group(); } /// An aborted group releases its access sample along with its bytes, from any /// handle: ghost samples must not linger in the pool mean where they'd hold it /// in the past and over-protect every live group. #[test] fn aborted_group_leaves_no_ghost_sample() { let (producer, pool) = pooled_producer(1 << 40); let group0 = producer.append_next().unwrap(); let _open = producer.append_next().unwrap(); // demotes seq 0 into the mean assert!(pool.average().is_some(), "demoted group is sampled"); group0.abort(Error::Cancel).unwrap(); assert_eq!(pool.average(), None, "the abort must remove the sample"); } /// Empty groups still carry fixed overhead; they must repay the budget when /// evicted rather than being unevictable freeloaders. #[test] fn empty_groups_repay_overhead() { let (producer, pool) = pooled_producer(1_000); for _ in 0..100 { let group = producer.append_next().unwrap(); group.finish().unwrap(); } // A budget below one entry keeps the live group and the one it just demoted. assert!( pool.used() <= 2 * cache::ENTRY_OVERHEAD, "empty-group overhead must stay near the budget: {}", pool.used() ); } /// Late growth on an already-demoted group is billed: the gross-write counter /// feeds debt on the next append, so a straggler can't grow unbounded. #[test] fn growth_on_demoted_group_is_billed() { let (producer, pool) = pooled_producer(2_000); let mut straggler = producer.append_next().unwrap(); // seq 0 producer.append_next().unwrap().finish().unwrap(); // seq 1 demotes seq 0 // The demoted group balloons: no eviction yet (nothing ran), but billed. straggler .write_frame(straggler.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); // The next append observes the growth and evicts the straggler. producer.append_next().unwrap().finish().unwrap(); // seq 2 let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none(), "the ballooned group is evicted"); assert!( pool.used() <= 2 * cache::ENTRY_OVERHEAD, "growth is reclaimed, leaving the two empty groups: {}", pool.used() ); } /// A stale arrival entry whose key was later re-served by fetched backfill must /// not put the replacement in the arrival walk, which a subscriber reads as backfill. #[kio::test] #[cfg(feature = "async")] async fn a_refilled_key_stays_out_of_the_arrival_walk() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); producer.create_group(0u64.into()).unwrap().finish().unwrap(); let aborted = producer.create_group(1u64.into()).unwrap(); aborted.abort(Error::Cancel).unwrap(); producer.create_group(2u64.into()).unwrap().finish().unwrap(); // Re-serve seq 1 as backfill; its slot replaces the aborted one, and the // old arrival entry for seq 1 now resolves to it. let pending = consumer.fetch_group(1, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); group.finish().unwrap(); pending.await.unwrap(); // The backfill serves by key, but never in arrival order. assert!(consumer.peek_group(1).is_some()); let mut walked = Vec::new(); let index = Cell::new(0); let window = { let state = producer.state.read(); Window::new(&replay().ranges, None, None, &*state).unwrap() }; while let Poll::Ready(Ok(Some((group, next)))) = producer.state.read().poll_recv_group(&index, 0, &window) { walked.push(group.start); index.set(next + 1); } assert_eq!(walked, [0, 2]); // A reader takes it as the fill answer for the group the feed gave up, as it would // have asked a fill for it, and the stale arrival entry adds nothing. let mut subscriber = producer.subscribe(replay()); let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().start).collect(); read.sort(); assert_eq!(read, [0, 1, 2]); subscriber.assert_no_group(); } /// An expired backfill can't hide behind a refreshed one: the eviction-order /// expiry scans a bounded prefix instead of stopping at the first fresh entry. #[kio::test] #[cfg(feature = "async")] async fn expired_backfill_behind_refreshed_reclaimed() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); producer.create_group(5u64.into()).unwrap().finish().unwrap(); for key in [2u64, 3u64] { let pending = consumer.fetch_group(key, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); group.finish().unwrap(); pending.await.unwrap(); } // Keep seq 2 fresh while seq 3 (behind it in eviction order) expires. elapse(&producer, cache::DEFAULT_EXPIRY / 2 + Duration::from_secs(1)); consumer.fetch_group(2, None).await.unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY / 2 + Duration::from_secs(1)); producer.create_group(6u64.into()).unwrap().finish().unwrap(); let consumer = producer.consume(); assert!(consumer.peek_group(2).is_some(), "refreshed backfill survives"); assert!(consumer.peek_group(3).is_none(), "expired backfill is reclaimed"); } /// A FETCH hit within the same coarse clock tick still protects the group: the /// refresh stamps one tick ahead, so it reads strictly newer than the mean. #[kio::test] #[cfg(feature = "async")] async fn same_tick_fetch_protects() { // No time advances at all: every timestamp lands in the same tick. let (mut producer, _pool) = pooled_producer(11_000); let consumer = producer.consume(); finished_group(&mut producer, 3_000); // seq 0 finished_group(&mut producer, 3_000); // seq 1 finished_group(&mut producer, 3_000); // seq 2 consumer.fetch_group(0, None).await.unwrap(); finished_group(&mut producer, 3_000); // seq 3 finished_group(&mut producer, 3_000); // seq 4 assert!(consumer.peek_group(0).is_some(), "same-tick refresh protects"); assert!(consumer.peek_group(1).is_none(), "the unread group dies instead"); } /// A refetched group that reclaims max_key is the live edge again: it must /// not re-enter the eviction order, or memory pressure could evict the newest /// content. #[kio::test] #[cfg(feature = "async")] async fn refetched_latest_stays_protected() { let (producer, _pool) = pooled_producer(10_000); let dynamic = producer.requests(); let consumer = producer.consume(); let straggler = producer.append_next().unwrap(); // seq 0 // The publisher aborts its own latest group; the key stays at the live edge. let latest = producer.append_next().unwrap(); // seq 1 latest.abort(Error::Cancel).unwrap(); // Re-fetch it: the replacement takes over max_key. let pending = consumer.fetch_group(1, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1000])) .unwrap(); group.finish().unwrap(); pending.await.unwrap(); // The refetched latest is protected by omission: it has no entry in the // eviction order, so no amount of debt can select it. { let state = producer.state.read(); assert!(state.lookup.contains_key(&1), "refetched group is cached"); assert!( state.evict.iter().all(|(key, _)| *key != 1), "the live edge must not be an eviction candidate" ); } drop(straggler); } /// An evicted group is a cache miss, so a fetch re-fetches it and the accepted /// replacement serves the key again (not `Error::Duplicate`). #[kio::test] #[cfg(feature = "async")] async fn eviction_allows_refetch() { let (mut producer, _pool) = pooled_producer(10_000); let dynamic = producer.requests(); finished_group(&mut producer, 10_000); // seq 0 finished_group(&mut producer, 10_000); // seq 1: demotes seq 0 finished_group(&mut producer, 10_000); // seq 2: pays by evicting seq 0 let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none()); let pending = consumer.fetch_group(0, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); assert_eq!(req.key(), 0); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"refetched")) .unwrap(); group.finish().unwrap(); let mut group = pending.await.unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"refetched"); } /// An aborted group is dead whatever frame it starts at: the key is claimable /// again, and a cache lookup misses rather than handing back a slot that can no /// longer serve it. /// /// The abort and the group's first frame decide this together, so they are read /// under one guard. Read separately, the abort can land between them and the slot /// answers as a live duplicate on the strength of an offset it only still has /// because it died. That interleaving is what the single guard rules out; this /// pins the committed semantics it has to preserve. #[test] fn an_aborted_group_releases_its_key() { let producer = track_producer("test", Info::timed()); let consumer = producer.consume(); let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); // While it lives the slot answers, so the key is taken. assert!(matches!( producer.create_group(group::Info { start: 3 }), Err(Error::Duplicate) )); assert!(consumer.peek_group(3).is_some()); group.abort(Error::Cancel).unwrap(); assert!(consumer.peek_group(3).is_none(), "an aborted slot is a cache miss"); producer .create_group(group::Info { start: 3 }) .expect("an aborted slot releases its key") .finish() .unwrap(); } /// Lowering the floor re-offers what the arrival walk could have read, a live group /// before its first frame, and owes a fetched one as cached backfill. Only the walked /// groups count as unread for a copy handing over. #[kio::test] #[cfg(feature = "async")] async fn a_lowered_floor_reoffers_live_groups_and_owes_fetched_ones() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); producer.create_group(5u64.into()).unwrap().finish().unwrap(); let mut subscriber = producer.subscribe(replay().with_range(5..)); assert_eq!(subscriber.assert_group().start, 5); let pending = consumer.fetch_group(2, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut fetched = req.accept(Info::timed()).unwrap().expect("a fresh group"); fetched .write_frame(fetched.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); fetched.finish().unwrap(); pending.await.unwrap(); let _frameless = producer.create_group(group::Info { start: 3 }).unwrap(); subscriber.update(replay().with_range(0..)).unwrap(); assert_eq!(subscriber.assert_group().start, 3, "offered before its first frame"); assert_eq!(subscriber.assert_group().start, 2, "cached backfill is owed"); subscriber.assert_no_group(); // A cursor floor lowered before the next read counts only walked groups as unread. let mut lowered = producer.subscribe(replay()); lowered.set_range(5..); assert_eq!(lowered.assert_group().start, 5); lowered.set_range(0..); let delivered = super::super::delivered::Delivered::default(); let unread = Unread { start: 0, end: Some(3), delivered: &delivered, }; assert!(!lowered.has_unread_group(&unread), "the fetched group 2 is not unread"); } /// A group another reader's fill cached is owed to a subscriber whose window holds it, /// as its own fill's answer would be, though the arrival walk never reaches it: a warm /// cache serves what a cold one fills. #[kio::test] #[cfg(feature = "async")] async fn cached_backfill_is_owed_to_a_subscriber() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); // The publisher starts at seq 5; earlier groups exist only upstream. producer.create_group(5u64.into()).unwrap().finish().unwrap(); producer.create_group(6u64.into()).unwrap().finish().unwrap(); // Fetch the gap: it lands in the cache and resolves the fetch... let pending = consumer.fetch_group(2, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); group.finish().unwrap(); let mut fetched = pending.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"backfill"); assert!(consumer.peek_group(2).is_some(), "backfill is cached for later fetches"); // ...and a subscriber reads it beside the live groups, asking no fill of its own. let mut subscriber = producer.subscribe(replay()); let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().start).collect(); read.sort_unstable(); assert_eq!(read, [2, 5, 6]); subscriber.assert_no_group(); // A bounded read of it alone ends once it took it. let mut bounded = producer.subscribe(replay().with_range(2..3)); assert_eq!(read_now(&mut bounded, false), Some(Some(2))); assert_eq!(read_now(&mut bounded, false), Some(None)); assert!( dynamic.requested_fill_group().now_or_never().is_none(), "the cache answered both reads" ); } /// The feed's copy of a group a fetch cached first takes the slot, so a subscriber reads /// it, while the fetch's reader keeps its own copy, abandoned or not. #[kio::test] #[cfg(feature = "async")] async fn the_feed_takes_a_group_a_fetch_cached_first() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); let mut subscriber = producer.subscribe(replay()); let pending = consumer.fetch_group(0, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut fetching = req.accept(Info::timed()).unwrap().expect("a fresh group"); fetching .write_frame(fetching.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut fetched = pending.await.unwrap(); subscriber.assert_no_group(); let Ok(Received::Group(mut live)) = producer.create_group_at(0u64.into(), 0) else { panic!("the feed takes the slot"); }; live.write_frame(live.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut group = subscriber.assert_group(); assert_eq!(group.start, 0); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"snapshot"); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"snapshot"); // The fetch giving up never cuts the group short for the subscriber. drop(fetched); fetching.abort(Error::Cancel).unwrap(); live.finish().unwrap(); assert!(group.read_frame().await.unwrap().is_none()); assert!(matches!( producer.create_group_at(0u64.into(), 0), Err(Error::Duplicate) )); } /// Fetched backfill isn't in arrival order, so it ages out through the eviction /// order instead of lingering until the track closes. #[kio::test] #[cfg(feature = "async")] async fn expired_backfill_reclaimed() { let (producer, pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); producer.create_group(5u64.into()).unwrap().finish().unwrap(); // Serve a backfill fetch for an old key. let pending = consumer.fetch_group(2, None); let req = dynamic .requested_fill_group() .now_or_never() .expect("should not block") .unwrap(); let mut group = req.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1000])) .unwrap(); group.finish().unwrap(); pending.await.unwrap(); let used = pool.used(); // Age past the pool's LRU window; the next write reclaims the backfill. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer.create_group(6u64.into()).unwrap().finish().unwrap(); assert!(consumer.peek_group(2).is_none(), "expired backfill is reclaimed"); assert!(pool.used() < used, "its bytes are released"); } /// Append a finished group of a 3 byte first frame and a `delta` byte delta. fn refresh_group(producer: &Producer, delta: usize) { let mut group = producer.append_next().unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"key"[..]) .unwrap(); if delta > 0 { group .write_frame(crate::Timestamp::unscaled(group.start), vec![0u8; delta]) .unwrap(); } group.finish().unwrap(); } /// Poll `demand` for a change without waiting. fn wanted_changed(demand: &mut Demand) -> Option { match demand.poll_wanted_changed(&kio::Waiter::noop()) { Poll::Ready(res) => Some(res.unwrap()), Poll::Pending => None, } } /// Whether `demand` asks for a new group now. fn refreshing(demand: &Demand) -> bool { match demand.poll_refresh(&kio::Waiter::noop()) { Poll::Ready(res) => { res.unwrap(); true } Poll::Pending => false, } } /// The first consumer of a track with no group asks for one, and nobody wants it once /// the last one leaves. #[test] fn wanted_follows_the_first_consumer() { let producer = track_producer("test", Info::default().with_refresh(Refresh::default().with_ratio(1))); let mut demand = producer.demand(); assert!(!demand.wanted()); assert!(!refreshing(&demand), "nobody asks"); assert_eq!(wanted_changed(&mut demand), Some(false), "the first call reports"); assert_eq!(wanted_changed(&mut demand), None, "then only changes"); let subscriber = producer.subscribe(None); assert!(demand.wanted()); assert!(refreshing(&demand), "the first group"); assert_eq!(wanted_changed(&mut demand), Some(true)); refresh_group(&producer, 0); assert!(!refreshing(&demand), "the group answered it"); drop(subscriber); assert!(!demand.wanted()); assert_eq!(wanted_changed(&mut demand), Some(false)); } /// A join reads the newest group, and once its bytes after the first frame exceed the /// ratio times the first frame it also wants the group after the newest. The cached group /// is served either way: refresh is a request, not a hold. #[test] fn a_join_over_the_ratio_wants_a_new_group() { for (delta, wanted) in [(0, 0), (3, 0), (4, 1)] { let producer = track_producer("test", Info::default().with_refresh(Refresh::default().with_ratio(1))); refresh_group(&producer, delta); let mut subscriber = producer.subscribe(None); assert_eq!(producer.demand().wanted_start(), Some(wanted), "{delta} byte delta"); assert_eq!(subscriber.assert_group().start, 0, "{delta} byte delta"); } } /// A handed-out group keeps its track wanted once its subscription leaves, until a /// reader reads its end: a finished group owes nothing more, even while still held. #[test] fn a_group_read_keeps_its_track_wanted_until_its_end() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let mut writing = producer.append_next().unwrap(); writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let mut group = subscriber.assert_group(); let other = group.clone(); drop(subscriber); assert_eq!(producer.demand().wanted_start(), Some(0), "the group is read"); writing.finish().unwrap(); assert_eq!(producer.demand().wanted_start(), Some(0), "not read to its end yet"); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); assert!(!producer.demand().wanted(), "a clone reading the end ends it for all"); drop(other); } /// A reader stopping at its range cap has not reached the group's end: the group is /// still open, so it keeps its track wanted for the others and for a raised cap. #[test] fn a_range_cap_is_not_the_groups_end() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); let mut writing = producer.append_next().unwrap(); writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let group = subscriber.assert_group(); let mut capped = group.clone().with_range(..1); drop(subscriber); let waiter = kio::Waiter::noop(); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); assert!(matches!(capped.poll_finished(&waiter), Poll::Ready(Ok(1)))); assert_eq!( producer.demand().wanted_start(), Some(0), "the cap is not the group's end" ); drop(group); let producer = producer .abort_unwanted(Error::Cancel) .expect_err("the capped reader can raise its cap"); writing.write_frame(writing.key_time(), &b"delta"[..]).unwrap(); let mut capped = capped.with_range(..); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); writing.finish().unwrap(); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); assert_eq!(producer.demand().wanted_start(), None, "read to the group's end"); } /// A fetched group keeps its track wanted after the fetch resolved, while it is read. #[test] #[cfg(feature = "async")] fn a_fetched_group_keeps_its_track_wanted() { let producer = track_producer("test", Info::timed()); let mut writing = producer.append_next().unwrap(); writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let group = producer .consume() .fetch_group(0, None) .now_or_never() .expect("cached") .unwrap(); assert_eq!(producer.demand().wanted_start(), Some(0)); drop(group); assert_eq!(producer.demand().wanted_start(), None); } /// Joins while a request is pending ask for the same group, so the publisher hears it /// once. The group reaching it leaves nothing pending, and only a join over the ratio of /// that group asks again. #[test] fn joins_coalesce_until_the_group_lands() { let producer = track_producer("test", Info::default().with_refresh(Refresh::default().with_ratio(1))); let demand = producer.demand(); refresh_group(&producer, 4); let _a = producer.subscribe(None); assert_eq!(demand.asked(), Some(1)); let _b = producer.subscribe(None); assert_eq!(demand.asked(), Some(1), "coalesced"); refresh_group(&producer, 0); assert_eq!(demand.asked(), None, "group 1 answers it"); let _c = producer.subscribe(None); assert_eq!(demand.asked(), None, "group 1 is under the ratio"); refresh_group(&producer, 4); let _d = producer.subscribe(None); assert_eq!(demand.asked(), Some(3)); assert!(refreshing(&demand)); } /// A track without a ratio never asks for a group on a join. #[test] fn joins_without_refresh_ask_nothing() { let producer = track_producer("test", Info::default()); refresh_group(&producer, 1000); let _subscriber = producer.subscribe(None); assert_eq!(producer.demand().wanted_start(), Some(0)); } /// Only a join at the live edge counts: a subscription resuming from a fixed group /// already has the newest group's start. #[test] fn a_fixed_start_is_no_join() { let producer = track_producer("test", Info::default().with_refresh(Refresh::default().with_ratio(1))); refresh_group(&producer, 4); let _subscriber = producer.subscribe(Subscription::default().with_range(0..)); assert_eq!(producer.demand().wanted_start(), Some(0)); } /// The demand reports the track's closure as its abort reason. #[test] fn wanted_changed_errors_once_closed() { let producer = track_producer("test", Info::default()); let mut demand = producer.demand(); producer.abort(Error::Cancel).unwrap(); assert!(matches!( demand.poll_wanted_changed(&kio::Waiter::noop()), Poll::Ready(Err(Error::Cancel)) )); assert_eq!(demand.wanted_start(), None); } /// A refresh on a track with only a ratio is judged like a join: under the ratio it asks /// nothing, over it the group past the newest. #[test] fn a_refresh_without_an_interval_is_judged_by_the_ratio() { let producer = track_producer("test", Info::default().with_refresh(Refresh::default().with_ratio(1))); let subscriber = producer.subscribe(None); refresh_group(&producer, 3); assert!(sent(&mut subscriber.refresh()), "nothing to send"); assert_eq!(producer.demand().asked(), None); refresh_group(&producer, 4); let _refresh = subscriber.refresh(); assert_eq!(producer.demand().asked(), Some(2)); } /// A track that does not refresh never asks, however a reader asks. #[test] fn a_refresh_without_opting_in_asks_nothing() { let producer = track_producer("test", Info::default()); let subscriber = producer.subscribe(None); refresh_group(&producer, 0); assert!(sent(&mut subscriber.refresh()), "nothing to send"); elapse(&producer, Duration::from_secs(3600)); assert!(!refreshing(&producer.demand())); } /// The interval caps how often a one-frame track makes groups: a request asks at once /// when the newest group is old enough, and otherwise waits until it is, then asks once /// however many readers asked meanwhile. A group landing first answers it. #[test] fn an_interval_defers_a_refresh_until_the_newest_group_is_old_enough() { let interval = Duration::from_secs(5); let info = Info::default().with_refresh(Refresh::default().with_interval(interval)); let producer = track_producer("test", info); let demand = producer.demand(); let subscriber = producer.subscribe(None); elapse(&producer, Duration::ZERO); assert!(refreshing(&demand), "the first group needs no interval"); refresh_group(&producer, 0); assert!(!refreshing(&demand)); elapse(&producer, Duration::from_secs(2)); let _refresh = subscriber.refresh(); let late = producer.subscribe(None); assert!(!refreshing(&demand), "deferred, not dropped"); elapse(&producer, Duration::from_secs(2)); assert!(!refreshing(&demand), "still within the interval"); elapse(&producer, Duration::from_secs(1)); assert!(refreshing(&demand), "due once the newest group is the interval old"); assert_eq!(demand.asked(), Some(1), "one group for every request"); refresh_group(&producer, 0); assert!(!refreshing(&demand), "the new group answers every request"); // A request made long after the newest group asks at once. elapse(&producer, Duration::from_secs(60)); let _late = late.refresh(); assert!(refreshing(&demand)); refresh_group(&producer, 0); // A group landing while a request waits answers it. let _refresh = subscriber.refresh(); refresh_group(&producer, 0); elapse(&producer, interval); assert!(!refreshing(&demand), "the landed group answered it"); } /// A deferred refresh wakes the publisher parked on it once it comes due. #[test] fn a_deferred_refresh_wakes_the_publisher() { let interval = Duration::from_secs(5); let info = Info::default().with_refresh(Refresh::default().with_interval(interval)); let producer = track_producer("test", info); let demand = producer.demand(); let subscriber = producer.subscribe(None); elapse(&producer, Duration::ZERO); refresh_group(&producer, 0); let _refresh = subscriber.refresh(); struct Count(std::sync::atomic::AtomicUsize); impl std::task::Wake for Count { fn wake(self: Arc) { self.wake_by_ref(); } fn wake_by_ref(self: &Arc) { self.0.fetch_add(1, Ordering::SeqCst); } } let count = Arc::new(Count(std::sync::atomic::AtomicUsize::new(0))); let waiter = kio::Waiter::new(std::task::Waker::from(count.clone())); assert!(demand.poll_refresh(&waiter).is_pending()); elapse(&producer, Duration::from_secs(4)); assert_eq!(count.0.load(Ordering::SeqCst), 0, "not due yet"); elapse(&producer, Duration::from_secs(1)); assert!(count.0.load(Ordering::SeqCst) > 0, "woken when due"); assert!(demand.poll_refresh(&waiter).is_ready()); } /// With both, a join must outweigh the ratio, while a refresh needs only the interval. #[test] fn a_refresh_skips_the_ratio_when_an_interval_is_declared() { let refresh = Refresh::default().with_ratio(1).with_interval(Duration::from_secs(1)); let producer = track_producer("test", Info::default().with_refresh(refresh)); let demand = producer.demand(); let subscriber = producer.subscribe(None); elapse(&producer, Duration::ZERO); refresh_group(&producer, 0); elapse(&producer, Duration::from_secs(1)); let _join = producer.subscribe(None); assert!(!refreshing(&demand), "a join under the ratio asks nothing"); let _refresh = subscriber.refresh(); assert!(refreshing(&demand), "a refresh asks"); } /// Whether `refresh` resolved, without waiting. fn sent(refresh: &mut Refreshing) -> bool { match refresh.poll(&kio::Waiter::noop()) { Poll::Ready(res) => { res.unwrap(); true } Poll::Pending => false, } } /// A one-frame track whose refresh interval is 5s, with a subscriber, and a group made at /// time zero. fn interval_track() -> (Producer, Subscriber) { let info = Info::default().with_refresh(Refresh::default().with_interval(Duration::from_secs(5))); let producer = track_producer("test", info); let subscriber = producer.subscribe(None); elapse(&producer, Duration::ZERO); refresh_group(&producer, 0); (producer, subscriber) } /// A refresh resolves once it is sent: after the interval, when the publisher reads it. /// Every caller in one interval window shares the one request and resolves with it. #[test] fn concurrent_refreshes_resolve_once_the_request_is_sent() { let (producer, subscriber) = interval_track(); let demand = producer.demand(); let mut first = subscriber.refresh(); elapse(&producer, Duration::from_secs(2)); let mut second = subscriber.refresh(); let other = producer.subscribe(None); let mut third = other.ordered().control().refresh(); assert!(!sent(&mut first) && !sent(&mut second) && !sent(&mut third), "deferred"); elapse(&producer, Duration::from_secs(3)); assert!(!sent(&mut first), "due, but nothing has read it"); assert!(refreshing(&demand), "the publisher reads it"); assert!( sent(&mut first) && sent(&mut second) && sent(&mut third), "one send resolves all" ); assert!( sent(&mut subscriber.refresh()), "a later caller joins the request already sent" ); assert_eq!(demand.asked(), Some(1), "one group answers them all"); } /// A resolved refresh wakes its caller. #[test] #[cfg(feature = "async")] fn a_sent_refresh_wakes_its_caller() { let (producer, subscriber) = interval_track(); let refresh = subscriber.refresh(); let mut task = std::pin::pin!(refresh); let waker = std::task::Waker::noop(); let mut cx = std::task::Context::from_waker(waker); assert!(task.as_mut().poll(&mut cx).is_pending()); elapse(&producer, Duration::from_secs(5)); assert!(refreshing(&producer.demand())); assert!(matches!(task.as_mut().poll(&mut cx), Poll::Ready(Ok(())))); } /// Dropping the last caller's refresh before it is sent withdraws the request, but not /// one another caller still waits on, one a join also made, or one already sent. #[test] fn dropping_a_refresh_withdraws_an_unsent_request() { let (producer, subscriber) = interval_track(); let demand = producer.demand(); drop(subscriber.refresh()); elapse(&producer, Duration::from_secs(5)); assert!(!refreshing(&demand), "withdrawn while deferred"); // Due, but not yet read. drop(subscriber.refresh()); assert!(!refreshing(&demand), "withdrawn while due"); let kept = subscriber.refresh(); drop(subscriber.refresh()); assert!(refreshing(&demand), "another caller keeps it"); drop(kept); assert!(refreshing(&demand), "sent, so it stays sent"); refresh_group(&producer, 0); let refresh = subscriber.refresh(); let _join = producer.subscribe(None); drop(refresh); elapse(&producer, Duration::from_secs(5)); assert!(refreshing(&demand), "the join's request stays"); } /// A refresh resolves without sending anything when a group answers it first, and /// errors when the track closes first. #[test] fn a_refresh_ends_with_the_group_or_the_track() { let (producer, subscriber) = interval_track(); let mut refresh = subscriber.refresh(); refresh_group(&producer, 0); assert!(sent(&mut refresh), "the landed group answered it"); let mut refresh = subscriber.refresh(); producer.abort(Error::Cancel).unwrap(); assert!(matches!( refresh.poll(&kio::Waiter::noop()), Poll::Ready(Err(Error::Cancel)) )); } /// A front reader's pending refresh outlives the copy it asked: once that copy closes /// while the logical track is open, it asks the copy the route serves next, and resolves /// when that one is sent. Only the logical track closing rejects it. The same holds /// through a chain of fronts, where the switch happens on the inner one. #[test] fn a_front_refresh_moves_to_the_next_copy() { let info = Info::default().with_refresh(Refresh::default().with_interval(Duration::from_secs(5))); let copy = || { let copy = track_producer("test", info.clone()); elapse(©, Duration::ZERO); refresh_group(©, 0); copy }; let front = || { let routes = super::super::resume::Producer::new(); let logical = Request::new(Arc::new(broadcast::Info::default()), "test") .routes(routes.consume()) .accept(info.clone()); (routes, logical) }; for chained in [false, true] { let (routes, logical) = front(); // The front whose route switches: the outer one's own, or the one it serves from. let (routes, _inner) = match chained { false => (routes, None), true => { let (inner_routes, inner) = front(); routes.serve(inner.consume()); (inner_routes, Some((routes, inner))) } }; let first = copy(); routes.serve(first.consume()); let subscriber = logical.consume().subscribe(None).now_or_never().unwrap().unwrap(); let (woken, waiter) = Flag::waiter(); let mut refresh = subscriber.refresh(); assert!(refresh.poll(&waiter).is_pending(), "deferred on the first copy"); first.abort(Error::Cancel).unwrap(); assert!(woken.take()); assert!(refresh.poll(&waiter).is_pending(), "the logical track is still open"); let second = copy(); // Something reading the copy, so its publisher sees demand: a chained front's // reader subscribes only once it reads. let _reader = second.subscribe(None); routes.serve(second.consume()); assert!(woken.take(), "the new copy wakes it"); assert!(refresh.poll(&waiter).is_pending(), "deferred on the new copy"); elapse(&second, Duration::from_secs(5)); assert!(refreshing(&second.demand()), "asked of the new copy"); assert!(sent(&mut refresh)); refresh_group(&second, 0); let mut refresh = subscriber.refresh(); assert!(refresh.poll(&waiter).is_pending()); logical.abort(Error::Cancel).unwrap(); assert!(matches!(refresh.poll(&waiter), Poll::Ready(Err(Error::Cancel)))); assert!( matches!(subscriber.refresh().poll(&waiter), Poll::Ready(Err(Error::Cancel))), "asked after the logical track closed" ); } } #[test] #[cfg(feature = "async")] fn request_and_dynamic_share_weak_track_demand() { let request = Request::new(Arc::new(broadcast::Info::default()), "demand"); let dynamic = request.requests(); let demand = request.demand(); let consumer = request.consume(); assert!(!demand.is_wanted(), "a bare handle is not demand"); let query = consumer.info(); assert!(demand.is_wanted()); assert!(dynamic.demand().is_wanted()); drop(query); assert!(!demand.is_wanted()); let producer = request.accept(Info::timed()); assert_eq!(producer.demand().name(), demand.name()); drop(producer); drop(dynamic); assert!(matches!( demand.clone().wanted_changed().now_or_never(), Some(Err(Error::Dropped)) )); } /// Every fetch that joined a fill counts as its demand, and the last to leave /// withdraws it. #[test] fn fill_demand_counts_every_joined_caller() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let first = consumer.fetch_group(3, None); let second = consumer.fetch_group(3, None); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("one fill for both fetches"); }; assert!(dynamic.poll_requested_groups(&waiter).is_pending(), "the second joined"); assert_eq!((fill.start(), fill.end()), (Position::group(3), 4)); assert_eq!(fill.wanted(), Some(3)); drop(first); assert!(fill.poll_withdrawn(&waiter).is_pending()); drop(second); assert!(fill.poll_withdrawn(&waiter).is_ready()); assert_eq!(fill.wanted(), None); drop(fill); assert!( !producer.state.read().drops.contains_key(&3), "a withdrawn fill records nothing" ); } /// A fill accepting a group resolves the fetch from the cache. #[test] fn accepted_fill_resolves_its_fetch() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut fetch = consumer.fetch_group(3, None); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the fill is ready"); }; let group = fill.accept(3, Info::timed()).unwrap().expect("a fresh group"); assert!(matches!(fill.accept(3, None), Ok(None))); assert!(matches!(fill.accept(4, None), Err(Error::InvalidRange))); assert!( matches!(kio::Task::poll(&mut *fetch, &waiter), Poll::Ready(Ok(_))), "the fetch gets the accepted group" ); group.finish().unwrap(); } /// A feed declared to start at group 2, with groups 2 and 3 written, read from group 0. #[cfg(feature = "async")] fn started_late() -> (Producer, Requests, Subscriber) { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let subscriber = producer.subscribe(replay().with_range(0..)); producer.start_at(2).unwrap(); write_group(&producer, 2); write_group(&producer, 3); (producer, dynamic, subscriber) } fn write_filled(group: Result>, millis: u64) { let mut group = group.unwrap().expect("a fresh group"); group .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } #[cfg(feature = "async")] fn drained_drops(subscriber: &mut Subscriber) -> Vec<(u64, u64)> { let mut dropped = Vec::new(); while let Poll::Ready((range, _)) = subscriber.poll_drop(&kio::Waiter::noop()) { dropped.push((range.start, range.end)); } dropped } /// What a reader asks for under the feed's start is a miss, asked at once: the reader /// reads what the handler accepts, and what it refuses is named dropped only once the /// feed gives up on it too, since it may still arrive late. #[test] #[cfg(feature = "async")] fn a_miss_below_the_start_is_filled() { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); assert!(drained_drops(&mut subscriber).is_empty()); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { read.push(group.start); } assert_eq!(read, [2, 3]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; assert_eq!((fill.start().group, fill.end()), (0, 2)); write_filled(fill.accept(0, None), 0); fill.drop_groups(1..2); let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the filled group is read"); }; assert_eq!(group.start, 0); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(drained_drops(&mut subscriber).is_empty(), "group 1 may still arrive"); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(1, 2)]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "a refusal is not asked again" ); } /// A copy whose fills cannot carry a reader's budget asks none below the start its /// publisher resolved from that budget, and names nothing there dropped: what a wider /// budget reaches comes through the feed's re-offer, which a reader takes as it arrives. #[test] #[cfg(feature = "async")] fn no_fill_below_a_start_resolved_from_the_budget() { let write = |producer: &Producer, key| { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); }; let mut producer = track_producer("test", None); producer.fills_from_start(); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..)); producer.request_start(None).unwrap(); producer.start_at(2).unwrap(); write(&producer, 2); write(&producer, 3); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), [2, 3]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "below the start is no miss" ); assert!(drained_drops(&mut subscriber).is_empty()); // The re-offer of a widened budget. write(&producer, 1); assert_eq!(drain_groups(&mut subscriber), [1]); assert!(dynamic.poll_requested_groups(&waiter).is_pending()); } /// Below a start its publisher resolved, a reader asks for the history its budget /// reaches: an update may have crossed the start, so the copy cannot tell which budget /// skipped those groups, and a fill that finds none ends empty. #[test] #[cfg(feature = "async")] fn below_a_resolved_start_a_reader_asks() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut reader = producer.subscribe(replay()); producer.start_resolved(2).unwrap(); write_group(&producer, 2); write_group(&producer, 3); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut reader), [2, 3]); assert!( drained_drops(&mut reader).is_empty(), "nothing below the start is given up unasked" ); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asks below the start"); }; assert_eq!((fill.start().group, fill.end()), (0, 2)); } /// A received copy's start the publisher later resolves from a budget, at the same group, /// turns the history below it from unasked into fillable: a reader asks for it then. #[test] #[cfg(feature = "async")] fn a_start_resolved_in_place_asks_the_history_below_it() { let mut producer = track_producer("test", Info::timed()); producer.received(); let dynamic = producer.requests(); let mut reader = producer.subscribe(replay()); producer.start_at(2).unwrap(); write_group(&producer, 2); write_group(&producer, 3); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut reader), [2, 3]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "below a declared start is no miss" ); producer.start_resolved(2).unwrap(); assert!(reader.poll_recv_group(&waiter).is_pending()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asks below the resolved start"); }; assert_eq!((fill.start().group, fill.end()), (0, 2)); } /// A reader arriving after another's fill cached the groups under the start reads them /// from the cache: the arrival walk never reaches a backfill, and no fill is asked again. #[test] #[cfg(feature = "async")] fn a_newcomer_reads_the_backfill_another_fill_cached() { let (producer, dynamic, mut first) = started_late(); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = first.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; while let Some(key) = fill.wanted() { write_filled(fill.accept(key, None), key); } let mut newcomer = producer.subscribe(replay().with_range(0..)); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = newcomer.poll_recv_group(&waiter) { read.push(group.start); } read.sort(); assert_eq!(read, [0, 1, 2, 3]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "the cache answers it" ); } /// Groups a fill answered before the reader came back are handed out in the reader's /// order, so a warm cache keeps a descending reader newest first. #[test] #[cfg(feature = "async")] fn filled_groups_are_read_in_the_readers_order() { for order in [Order::Descending, Order::Ascending] { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(order)); producer.start_at(2).unwrap(); write_group(&producer, 2); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; while let Some(key) = fill.wanted() { write_filled(fill.accept(key, None), key); } let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { read.push(group.start); } let expected = match order { Order::Descending => [1, 0], Order::Ascending => [0, 1], }; assert_eq!(read, expected, "{order:?}"); } } /// An ordered reader waiting on a stalled fill gives up the groups it owes once they are /// stale under its budget, as it skips a stale cached group, and reads on. #[test] #[cfg(feature = "async")] fn a_stalled_fill_goes_stale_under_the_readers_budget() { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&waiter) { Poll::Ready(Ok(Some(group))) => Some(group.start), Poll::Pending => None, _ => panic!("the read ended"), }; assert_eq!(next(&mut subscriber), None, "groups 0 and 1 are being filled"); // The handler takes the fill and never answers it. let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; elapse(&producer, Duration::from_secs(29)); assert_eq!(next(&mut subscriber), None, "still within the budget"); elapse(&producer, Duration::from_secs(2)); // Group 2 is as stale as the groups below it, superseded by group 3 as long ago. assert_eq!(next(&mut subscriber), Some(3)); assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); assert_eq!(fill.wanted(), None, "nobody waits on the stale fill"); } /// Groups arriving out of order inside a stalled fill judge each stretch between them by /// its own successor: a fresh late group keeps the groups below it owed while the stretches /// around it go stale. #[test] #[cfg(feature = "async")] fn a_late_fresh_group_keeps_its_stretch_of_a_stalled_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..)); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 0 is being filled" ); // The handler takes the fill and never answers it. let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; assert_eq!((fill.start().group, fill.end()), (0, 10)); write_group(&producer, 2); write_group(&producer, 8); elapse(&producer, Duration::from_secs(20)); write_group(&producer, 5); elapse(&producer, Duration::from_secs(11)); let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("groups 0 and 1 went stale behind group 2"); }; assert_eq!(group.start, 2, "superseded by group 5 only 11s ago"); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is still owed" ); for key in [3, 4] { assert!(fill.is_wanted(key), "group {key} is superseded by group 5"); } for key in [0, 1, 6, 7, 9] { assert!(!fill.is_wanted(key), "group {key} went stale"); } } /// A fetched group aborted once its readers left is a hole, not an arrival: the fill asked /// to replace it keeps it wanted. #[test] #[cfg(feature = "async")] fn an_aborted_fetched_group_stays_wanted_by_its_replacement_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); let fetch = producer.consume().fetch_group(3, None); let Poll::Ready(Ok(first)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; let group = first.accept(3, None).unwrap().expect("a fresh group"); drop(fetch); group.abort(Error::Cancel).unwrap(); drop(first); let mut subscriber = producer.subscribe(replay().with_range(0..)); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 0 is being filled" ); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the aborted group is a miss again"); }; assert_eq!((fill.start().group, fill.end()), (0, 10)); assert!(fill.is_wanted(3), "the aborted group never arrived"); } /// Refusing an evicted group the feed already passed fails the readers waiting on it /// without naming it dropped, so a later reader of the whole group still gets a fill. #[test] #[cfg(feature = "async")] fn a_refused_group_is_asked_again_by_a_later_reader() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); for key in 0..5 { write_group(&producer, key); } evict(&producer, 3); let waiter = kio::Waiter::noop(); let tail = producer.consume().fetch_group( 3, group::Fetch { frame_start: 2, ..Default::default() }, ); let Poll::Ready(Ok(first)) = dynamic.poll_requested_groups(&waiter) else { panic!("an evicted group is a miss"); }; first.refuse_groups(3..4, Error::Unsupported); assert!(matches!(tail.now_or_never(), Some(Err(Error::Unsupported)))); drop(first); // From group 2, which names the start, so group 3 is owed rather than looked up. let mut subscriber = producer.subscribe(replay().with_range(2..4)); assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(_))))); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is being filled" ); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the source may still have group 3"); }; assert!(fill.is_wanted(3)); } /// A source refusing a fetch as not found answers only that request, such as a head that /// expired while the tail is kept, so a later reader still gets a fill for the group. #[test] #[cfg(feature = "async")] fn a_source_miss_is_asked_again_by_a_later_reader() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); for key in 0..5 { write_group(&producer, key); } evict(&producer, 3); let waiter = kio::Waiter::noop(); let fetch = producer.consume().fetch_group(3, None); let Poll::Ready(Ok(first)) = dynamic.poll_requested_groups(&waiter) else { panic!("an evicted group is a miss"); }; first.group(3).reject(Error::NotFound); assert!(matches!(fetch.now_or_never(), Some(Err(Error::NotFound)))); drop(first); // From group 2, which names the start, so group 3 is owed rather than looked up. let mut subscriber = producer.subscribe(replay().with_range(2..4)); assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(_))))); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is being filled" ); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the source may still have part of group 3"); }; assert!(fill.is_wanted(3)); } /// A backfill cached before its fill publishes the answer is not a live arrival: the reader /// keeps it owed, so the answer still reaches it. #[test] #[cfg(feature = "async")] fn a_backfill_cached_before_its_answer_stays_owed() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; // The window inside `FillGroups::accept` between caching the group and publishing the answer. let _writing = producer .state .write() .ok() .unwrap() .insert_group_request(3, group::Bounds::default(), None) .unwrap() .expect("a fresh group"); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(fill.is_wanted(3), "the answer has not reached the reader yet"); } /// An ordered reader taking a filled group leaves the fill at once, so a reader that drops /// the group without reading on stops holding the fill's source. #[test] #[cfg(feature = "async")] fn an_ordered_reader_leaves_a_filled_group_it_takes() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..1)); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 0 is being filled" ); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; let _writing = fill.accept(0, None).unwrap().expect("a fresh group"); let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("the filled group is read"); }; assert_eq!(group.start, 0); drop(group); assert!(fill.poll_withdrawn(&waiter).is_ready(), "nobody waits on the fill now"); } /// A parked [`FillGroups::poll_wanted_changed`] wakes on every move of the group owed: a /// reader narrowing past it, a reader picking up an answer, and the last reader leaving, /// which ends at `None`. #[test] fn fill_wanted_changed_follows_the_group_owed() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); // Descending, so the reader narrows with its end, which settles at once. let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let noop = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&noop) {} let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&noop) else { panic!("below the start is a miss"); }; let (flag, waiter) = Flag::waiter(); assert_eq!( fill.poll_wanted_changed(&waiter), Poll::Ready(Some(9)), "the first poll reports" ); assert!(fill.poll_wanted_changed(&waiter).is_pending()); subscriber.update(descending(Range::groups(0..9))).unwrap(); assert!(subscriber.poll_recv_group(&noop).is_pending()); assert!(flag.take(), "narrowing woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(8))); assert!(fill.poll_wanted_changed(&waiter).is_pending()); write_filled(fill.accept(8, None), 8); flag.take(); assert!( fill.poll_wanted_changed(&waiter).is_pending(), "group 7 waits for the reader to pick up 8" ); assert!(matches!(subscriber.poll_recv_group(&noop), Poll::Ready(Ok(Some(_))))); assert!(flag.take(), "the reader picking it up woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(7))); assert!(fill.poll_wanted_changed(&waiter).is_pending()); drop(subscriber); assert!(flag.take(), "the last reader leaving woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(None)); assert_eq!(fill.wanted(), None); } /// The owed group arriving through the live feed wakes a parked /// [`FillGroups::poll_wanted_changed`]: the cache answers it, so the fill no longer owes it. #[test] #[cfg(feature = "async")] fn fill_wanted_changed_follows_a_live_arrival() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); producer.start_at(1).unwrap(); write_group(&producer, 1); let noop = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&noop) {} let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&noop) else { panic!("below the start is a miss"); }; let (flag, waiter) = Flag::waiter(); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(0))); assert!(fill.poll_wanted_changed(&waiter).is_pending()); write_group(&producer, 0); assert!(flag.take(), "the owed group arriving live woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(None)); assert_eq!(fill.wanted(), None); drop(subscriber); } /// A reader narrowing its ranges leaves the part of its fill it no longer reads: the /// handler is not asked for it, a fetch of it in flight is cancelled, and neither a bulk /// refusal nor dropping the fill names it dropped. #[test] #[cfg(feature = "async")] fn a_narrowed_reader_leaves_the_rest_of_its_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); // Descending, so the reader narrows with its end, which settles at once. let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; assert_eq!(fill.wanted(), Some(9)); assert!(fill.is_wanted(9)); assert!(fill.poll_unwanted(9, &waiter).is_pending()); subscriber.update(descending(Range::groups(0..8))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(fill.poll_unwanted(9, &waiter).is_ready(), "nobody reads group 9 now"); assert!(!fill.is_wanted(9)); assert!(fill.is_wanted(7)); assert!(fill.poll_unwanted(7, &waiter).is_pending()); assert_eq!(fill.wanted(), Some(7), "nobody reads above 7 any more"); write_filled(fill.accept(7, None), 7); assert_eq!(fill.wanted(), Some(6)); // Over, so what the fill refuses is a drop rather than a group still to come. producer.finish_shared().unwrap(); fill.drop_groups(7..10); drop(fill); let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the filled group is read"); }; assert_eq!(group.start, 7); assert_eq!(drained_drops(&mut subscriber), [(0, 7)], "only what it still read"); assert!(producer.state.read().drops.overlapping(&(8..10)).next().is_none()); } /// A group the fill accepted stays wanted until its reader takes it, even once the rest of /// the fill is answered, so its source keeps writing it meanwhile; narrowing past it /// before then leaves it. #[test] #[cfg(feature = "async")] fn a_filled_group_is_wanted_until_taken() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; let _writing = [ fill.accept(0, None).unwrap().expect("a fresh group"), fill.accept(1, None).unwrap().expect("a fresh group"), ]; for key in 2..10 { fill.group(key).reject(Error::NotFound); } let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the first filled group is read"); }; assert_eq!(group.start, 0); assert!(!fill.is_wanted(0), "taken: its reader holds the group itself"); assert!(fill.is_wanted(1), "filled, not taken yet"); subscriber.update(ascending(8..)).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(!fill.is_wanted(1), "nobody reads group 1 now"); assert!(fill.poll_unwanted(1, &waiter).is_ready()); } /// A session still writing an accepted group stops once every reader narrowed past it, /// though the fill still owes them later groups. #[test] #[cfg(feature = "async")] fn an_accepted_groups_readers_leave_before_its_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); // Descending, so the reader narrows with its end, which settles at once. let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; let group = fill.group(9); let readers = group.readers(); let _writing = group.accept(None).unwrap().expect("a fresh group"); assert!(readers.poll_unwanted(&waiter).is_pending()); subscriber.update(descending(Range::groups(0..9))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(readers.poll_unwanted(&waiter).is_ready(), "nobody reads group 9 now"); assert!( fill.poll_withdrawn(&waiter).is_pending(), "the fill still owes 8 and down" ); } /// A time start before every group held looks up its group through a fill that owes /// none: a handler that can't resolve times drops it, and the start reaches back to the /// group containing its time, which is unknown, so it asks no fill below the time itself: /// every key from there is in range. Another range still asks for its own misses. #[test] #[cfg(feature = "async")] fn a_time_start_before_the_cache_asks_nothing_below_it() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let since = Range { start: Anchor::Time(5), end: None, }; let mut subscriber = producer.subscribe( replay() .with_ranges([Range::groups(0..2), since]) .with_order(Order::Ascending), ); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); assert_eq!( drain_groups(&mut subscriber), Vec::::new(), "the time start waits on its lookup" ); let Poll::Ready(Ok(Fill::Lookup(lookup))) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start is looked up"); }; assert_eq!((lookup.time(), lookup.span()), (5, 0..10)); drop(lookup); let mut read = drain_groups(&mut subscriber); let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { asked.push((fill.start().group, fill.end())); while let Some(key) = fill.wanted() { write_filled(fill.accept(key, None), key); } drop(fill); read.extend(drain_groups(&mut subscriber)); } assert_eq!(asked, [(0, 2), (5, 10)], "nothing below 5 for the time start"); assert_eq!(read, [10, 0, 1, 5, 6, 7, 8, 9]); } /// A time start before every group held starts at the group its lookup answers, whose key /// says where it starts, asking nothing below it; the misses past it through the time end /// are filled. #[test] #[cfg(feature = "async")] fn a_looked_up_time_range_reads_the_history() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(5), end: Some(Anchor::Time(7)), }])); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); assert_eq!( drain_groups(&mut subscriber), Vec::::new(), "nothing reads until it is found" ); let Poll::Ready(Ok(Fill::Lookup(lookup))) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start is looked up"); }; assert_eq!(lookup.time(), 5); let found = lookup.accept(5, None); drop(lookup); let mut read = drain_groups(&mut subscriber); assert_eq!(read, [5], "the group found is read at once: its key is its start"); write_filled(found, 5); // The misses past the start, through the end. while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { while let Some(key) = fill.wanted() { assert!(key >= 5, "group {key} is below the looked-up start"); write_filled(fill.accept(key, None), key); } drop(fill); read.extend(drain_groups(&mut subscriber)); } // The misses come newest first, as the reader orders them. assert_eq!(read, [5, 7, 6]); } /// A time end walking the history a handler serves takes no group the live feed /// delivered and the cache evicted as gone: the history still holds it, and it may start /// inside the end, so the end settles on what the history answers. #[test] #[cfg(feature = "async")] fn a_time_end_walks_evicted_arrivals_through_the_history() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); for key in 0..10 { write_group(&producer, key); } for key in [0, 1, 2, 3, 5, 6, 7, 8] { evict(&producer, key); } let mut subscriber = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(5), end: Some(Anchor::Time(6)), }])); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), Vec::::new()); let Poll::Ready(Ok(Fill::Lookup(lookup))) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start in the evicted gap is looked up"); }; assert_eq!(lookup.time(), 5); write_filled(lookup.accept(5, None), 5); drop(lookup); let mut read = drain_groups(&mut subscriber); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { while let Some(key) = fill.wanted() { write_filled(fill.accept(key, None), key); } drop(fill); read.extend(drain_groups(&mut subscriber)); } assert_eq!(read, [5, 6], "groups 7 and 8 start past the end"); } /// A time end parked on a gap the history may still fill moves on once the reader's own /// fill refuses it: what its fills give up counts gone for its walks, which wake on it. #[test] #[cfg(feature = "async")] fn a_time_end_parked_on_a_gap_wakes_when_its_fill_refuses_it() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); for key in 0..7 { write_group(&producer, key); } for key in [3, 4] { evict(&producer, key); } let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending).with_ranges([Range { start: Anchor::Time(0), end: Some(Anchor::Time(4)), }])); // Polled as an executor would, again whenever the read wakes itself. let (flag, waiter) = Flag::waiter(); let mut poll = || loop { match subscriber.poll_group(true, &waiter) { Poll::Ready(res) => return Some(res.expect("no error").map(|group| group.start)), Poll::Pending if flag.take() => continue, Poll::Pending => return None, } }; let (mut read, mut ended) = (Vec::new(), false); loop { while !ended && let Some(next) = poll() { match next { Some(key) => read.push(key), None => ended = true, } } let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&kio::Waiter::noop()) else { break; }; while let Some(key) = fill.wanted() { fill.group(key).reject(Error::App(key as u16)); } } assert_eq!(read, [0, 1, 2]); assert!(ended, "the end settles at group 5, past the refused 3 and 4"); } /// A wide fill refused group by group, each with its own error, holds only the answers its /// readers have yet to take, not one per refusal. #[test] #[cfg(feature = "async")] fn a_wide_fill_forgets_refusals_its_readers_took() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); producer.start_at(100).unwrap(); write_group(&producer, 100); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; assert_eq!((fill.start().group, fill.end()), (0, 100)); for key in 0..50 { assert_eq!(fill.wanted(), Some(key)); fill.group(key).reject(Error::App(key as u16)); let _ = subscriber.poll_recv_group(&waiter); assert!(fill.answers() <= 1, "group {key}: {} answers held", fill.answers()); } } /// What a fill stopped owing is asked afresh by a reader that wants it later, rather than /// joining a fetch its handler is cancelling. #[test] #[cfg(feature = "async")] fn a_later_reader_asks_afresh_for_what_a_fill_stopped_owing() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); // Descending, so the first reader narrows with its end, which settles at once. let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); let mut narrowed = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = narrowed.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; narrowed.update(descending(Range::groups(0..8))).unwrap(); assert!(narrowed.poll_recv_group(&waiter).is_pending()); let mut later = producer.subscribe(descending(Range::groups(0..))); while let Poll::Ready(Ok(Some(_))) = later.poll_recv_group(&waiter) {} let Poll::Ready(Ok(again)) = dynamic.poll_requested_groups(&waiter) else { panic!("what the first fill stopped owing is asked afresh"); }; assert_eq!((again.start().group, again.end()), (8, 10)); assert_eq!(fill.wanted(), Some(7), "the first fill still owes only 7 and down"); assert!(!fill.is_wanted(9), "a group stops being wanted for good"); } /// A descending reader's disjoint misses queue newest first, so a handler serving one fill /// at a time reaches the newer history first. #[test] #[cfg(feature = "async")] fn a_descending_reader_asks_its_newest_miss_first() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_ranges([Range::groups(0..2), Range::groups(4..6), Range::groups(8..)])); producer.start_at(8).unwrap(); write_group(&producer, 8); // Held, so the middle range's time start names it and only 5 is a miss there. write_group(&producer, 4); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { asked.push((fill.start().group, fill.end())); } assert_eq!(asked, [(5, 6), (0, 2)]); } /// A group evicted after an arrival-order reader judged it cached, before it took it, is /// asked of a fill like one evicted before: the reader still receives it, and a group it /// took is never asked again. #[test] #[cfg(feature = "async")] fn a_group_evicted_before_its_reader_took_it_is_refilled() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay()); for key in 0..3 { write_group(&producer, key); } let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { panic!("group 0 is cached"); }; assert_eq!(first.start, 0); evict(&producer, 0); evict(&producer, 1); assert_eq!(drain_groups(&mut subscriber), [2]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("group 1 is a miss now"); }; assert_eq!((fill.start().group, fill.end()), (1, 2), "group 0 was taken"); assert_eq!(fill.wanted(), Some(1)); write_filled(fill.accept(1, None), 1); drop(fill); assert_eq!(drain_groups(&mut subscriber), [1]); assert!(dynamic.poll_requested_groups(&waiter).is_pending()); assert!(drained_drops(&mut subscriber).is_empty(), "nothing was given up"); let Inner::Plain(cursor) = &subscriber.inner else { unreachable!("a plain subscription"); }; let misses = cursor.misses.as_ref().expect("the reader judged its misses"); assert!(misses.refilled().is_empty(), "a refill handed out is forgotten"); } /// A live arrival judged on its own keeps the history a resumed copy's live floor sits /// above: a cached group the reader hasn't taken yet, evicted after, is still refilled. #[test] #[cfg(feature = "async")] fn history_below_a_resumed_live_floor_is_refilled_after_a_live_arrival() { let mut producer = track_producer("test", None); let dynamic = producer.requests(); producer.received(); for key in 0..5 { framed(&producer, key, 1).finish().unwrap(); } producer.set_idle(); // The feed resumes at 8, above the cached 0..=4: its live floor is 5. producer.set_live(Some(Position::group(8)), None).unwrap(); framed(&producer, 8, 1).finish().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_range(0..) .with_order(Order::Ascending) .with_delay(Delay::default().with_max(None)), ); let waiter = kio::Waiter::noop(); let next = |reader: &mut Subscriber| match reader.poll_next_group(&waiter) { Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut reader), Some(0)); // The gap the feed may still deliver arrives late, so nothing is deferred after. for key in 5..8 { framed(&producer, key, 1).finish().unwrap(); } assert_eq!(next(&mut reader), Some(1)); // Only the newest group moves before this read: judged on its own. framed(&producer, 9, 1).finish().unwrap(); assert_eq!(next(&mut reader), Some(2)); evict(&producer, 3); let _ = next(&mut reader); let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { asked.push((fill.start().group, fill.end())); } assert!( asked.iter().any(|&(start, end)| start <= 3 && 3 < end), "group 3 was evicted before the reader took it: asked {asked:?}" ); } /// An eviction waits only for a reader that judged the evicted group cached: a reader /// that judged once and stalled holds no more than what was cached then, however many /// groups the feed caches and evicts after, late lower ones included. A reader that /// took every group up to it holds none, since a group it took is never a miss again. #[test] #[cfg(feature = "async")] fn an_eviction_waits_only_for_readers_that_judged_it_cached() { let producer = track_producer("test", Info::timed()); let _dynamic = producer.requests(); write_group(&producer, 0); write_group(&producer, 1); write_group(&producer, 1 << 40); let held = || producer.state.read().inboxes.lock().unwrap().held(); evict(&producer, 0); assert!(held().is_empty(), "no reader judged group 0"); let mut stalled = producer.subscribe(replay()); let mut active = producer.subscribe(replay()); drain_groups(&mut stalled); drain_groups(&mut active); // Late groups below the stalled reader's newest, cached after it judged. for key in (2..200).step_by(2) { write_group(&producer, key); drain_groups(&mut active); evict(&producer, key); } // The active reader judged each one cached, so it holds the last until it judges again. assert_eq!(held(), [vec![], vec![(198, 199)]], "the stalled reader judged none"); drain_groups(&mut active); evict(&producer, 1); assert_eq!( held(), [vec![(1, 2)], vec![]], "the active reader judged after taking every group up to 2; the stalled one judged before it took group 1" ); drain_groups(&mut active); assert_eq!( held(), [vec![(1, 2)], vec![]], "the active reader has nothing to take in" ); drop(stalled); assert_eq!(held(), [vec![]], "a dropped reader holds nothing"); write_group(&producer, 3); evict(&producer, 3); assert_eq!( producer.state.read().inboxes.lock().unwrap().len(), 1, "an eviction prunes a dropped reader's inbox" ); } /// An ordered reader forwarding drops asks nothing below what it took, though its drop /// cursor reports from the window's floor. #[test] #[cfg(feature = "async")] fn an_ordered_readers_drop_poll_asks_nothing_it_took() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); for key in 0..3 { write_group(&producer, key); } let waiter = kio::Waiter::noop(); for expected in 0..2 { let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("group {expected} is cached"); }; assert_eq!(group.start, expected); } evict(&producer, 0); evict(&producer, 1); assert!(drained_drops(&mut subscriber).is_empty()); assert!(dynamic.poll_requested_groups(&waiter).is_pending(), "both were taken"); } /// An ordered reader waits on the refill of a group evicted at or above its next /// key, and reads it in order. #[test] #[cfg(feature = "async")] fn an_ordered_reader_refills_an_evicted_group_in_order() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); for key in 0..4 { write_group(&producer, key); } let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut subscriber), Some(0)); evict(&producer, 1); evict(&producer, 2); assert_eq!(next(&mut subscriber), None, "group 1 comes before 3"); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("groups 1 and 2 are misses now"); }; assert_eq!((fill.start().group, fill.end()), (1, 3)); while let Some(key) = fill.wanted() { write_filled(fill.accept(key, None), key); } drop(fill); assert_eq!([next(&mut subscriber), next(&mut subscriber)], [Some(1), Some(2)]); assert_eq!(next(&mut subscriber), Some(3)); } /// An ordered reader gives up a refill evicted again while it waits on a lower one, /// naming it dropped and releasing its fill, rather than reading past it unnamed. #[test] #[cfg(feature = "async")] fn an_ordered_reader_gives_up_a_refill_evicted_while_it_waits_below() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); for key in 0..4 { write_group(&producer, key); } let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut subscriber), Some(0)); evict(&producer, 1); evict(&producer, 2); assert_eq!(next(&mut subscriber), None, "group 1 comes before 3"); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("groups 1 and 2 are misses now"); }; write_filled(fill.accept(2, None), 2); assert_eq!(next(&mut subscriber), None, "still waiting on group 1"); evict(&producer, 2); assert_eq!(next(&mut subscriber), None, "still waiting on group 1"); assert_eq!(drained_drops(&mut subscriber), [(2, 3)]); write_filled(fill.accept(1, None), 1); drop(fill); assert_eq!([next(&mut subscriber), next(&mut subscriber)], [Some(1), Some(3)]); assert!(dynamic.poll_requested_groups(&waiter).is_pending(), "refilled once"); } /// A refill evicted again before its reader took it is given up rather than asked a /// second time, so refills that evict each other cannot loop. #[test] #[cfg(feature = "async")] fn a_refill_evicted_again_is_given_up() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay()); for key in 0..3 { write_group(&producer, key); } let waiter = kio::Waiter::noop(); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(Some(_))))); evict(&producer, 1); assert_eq!(drain_groups(&mut subscriber), [2]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("group 1 is a miss now"); }; assert_eq!(fill.wanted(), Some(1)); write_filled(fill.accept(1, None), 1); drop(fill); evict(&producer, 1); assert!(drain_groups(&mut subscriber).is_empty()); assert!(dynamic.poll_requested_groups(&waiter).is_pending(), "refilled once"); assert_eq!(drained_drops(&mut subscriber), [(1, 2)]); } /// A slow ascending reader on a pooled track holding `cached` groups, taking `pace` of /// every four its feed writes, as behind a bufferbloated link. Returns what it read, how /// many groups the fills wrote, and what it gave up. #[cfg(feature = "async")] fn read_slowly(cached: usize, pace: u64) -> (Vec, u64, Vec<(u64, u64)>) { let size = 10_000; let (producer, _pool) = pooled_producer((cached * size) as u64); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); let write = |group: Result>, key: u64| { let mut group = group.unwrap().expect("a fresh group"); group .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; size]) .unwrap(); group.finish().unwrap(); }; let waiter = kio::Waiter::noop(); let mut refilled = 0; let mut fills = Vec::new(); let mut serve = || { while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { fills.push(fill); } fills.retain_mut(|fill| { loop { match fill.poll_wanted_changed(&waiter) { Poll::Ready(Some(key)) => { write(fill.accept(key, None), key); refilled += 1; } Poll::Ready(None) => break false, Poll::Pending => break true, } } }); }; let (mut read, mut drops) = (Vec::new(), Vec::new()); // A wire publisher forwards what the reader gives up as it reads. let mut take = |subscriber: &mut Subscriber| { let taken = match subscriber.poll_next_group(&kio::Waiter::noop()) { Poll::Ready(Ok(Some(group))) => { read.push(group.start); true } _ => false, }; drops.extend(drained_drops(subscriber)); taken }; for key in 0..GROUPS { write(producer.create_group(group::Info { start: key }).map(Some), key); if key % 4 < pace { take(&mut subscriber); } serve(); } // The feed pauses and the reader drains what it can, refills included. loop { serve(); if !take(&mut subscriber) { serve(); if !take(&mut subscriber) { break; } } } (read, refilled, drops) } #[cfg(feature = "async")] const GROUPS: u64 = 40; /// A backlog a little past the cache: what the reader judged and lost to eviction is /// refilled, so it receives every group once, in order, refilling no more than it reads. #[test] #[cfg(feature = "async")] fn a_slow_reader_on_a_pooled_track_receives_every_group() { let (read, refilled, drops) = read_slowly(8, 3); assert_eq!(read, (0..GROUPS).collect::>(), "every group once, in order"); assert!(drops.is_empty(), "nothing given up: {drops:?}"); assert!(refilled > 0, "the backlog outgrew the cache"); assert!(refilled <= GROUPS, "{refilled} refills for {GROUPS} groups read"); } /// A backlog far past the cache, where refills evict the groups the reader reads next: /// each group is refilled at most once, and one lost again is given up and named, never /// handed out twice. #[test] #[cfg(feature = "async")] fn a_slow_reader_past_the_cache_refills_each_group_once() { let (read, refilled, drops) = read_slowly(4, 2); let mut seen = read.clone(); seen.dedup(); assert_eq!(seen, read, "none twice"); assert!(read.is_sorted(), "in order"); assert!(refilled <= GROUPS, "{refilled} refills for {GROUPS} groups"); let lost: Vec = (0..GROUPS).filter(|key| !read.contains(key)).collect(); let named: Vec = drops.iter().flat_map(|&(start, end)| start..end).collect(); assert_eq!(lost, named, "every group lost is named dropped"); } /// A batch far past the cache, asked by one slow reader: served through `wanted_changed`, /// the fill writes one group at a time as the reader takes them, so nothing it wrote is /// evicted before the reader gets it. #[test] #[cfg(feature = "async")] fn a_batch_fill_waits_for_its_reader() { let size = 10_000; let (mut producer, _pool) = pooled_producer(4 * size as u64); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); let write = |group: Result>, key: u64| { let mut group = group.unwrap().expect("a fresh group"); group .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; size]) .unwrap(); group.finish().unwrap(); }; producer.start_at(GROUPS).unwrap(); write(producer.create_group(group::Info { start: GROUPS }).map(Some), GROUPS); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_next_group(&waiter).is_pending(), "group 0 is a miss"); let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asked for the batch"); }; assert_eq!((fill.start().group, fill.end()), (0, GROUPS)); let (mut read, mut ahead) = (Vec::new(), 0); loop { let mut served = 0; while let Poll::Ready(Some(key)) = fill.poll_wanted_changed(&waiter) { write(fill.accept(key, None), key); served += 1; } ahead = ahead.max(served); match subscriber.poll_next_group(&waiter) { Poll::Ready(Ok(Some(group))) => read.push(group.start), _ => break, } } assert_eq!(read, (0..=GROUPS).collect::>(), "every group once, in order"); assert!(drained_drops(&mut subscriber).is_empty()); assert_eq!(ahead, 1, "one group served at a time"); } /// A reader asking newest first but reading in key picks up each group the fill writes /// while it waits on the oldest, so the fill never waits on a group the reader cannot take /// yet. #[test] #[cfg(feature = "async")] fn a_fill_against_its_readers_order_still_finishes() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Descending)); let mut ordered = subscriber.ordered(); producer.start_at(4).unwrap(); write_group(&producer, 4); let waiter = kio::Waiter::noop(); assert!(ordered.poll_next_group(&waiter).is_pending(), "group 0 is a miss"); let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asked for the batch"); }; assert_eq!(fill.order(), Order::Descending); let (mut served, mut read) = (Vec::new(), Vec::new()); while let Poll::Ready(Some(key)) = fill.poll_wanted_changed(&waiter) { write_filled(fill.accept(key, None), key); served.push(key); while let Poll::Ready(Ok(Some(item))) = ordered.poll_next_group(&waiter) { read.push(item.group().expect("a group").start); } } assert_eq!(served, [3, 2, 1, 0]); assert_eq!(read, [0, 1, 2, 3, 4]); } /// A publisher out of stream credit still forwards drops, whose probe takes in a fill's /// answers: only a read picks a group up, so the fill never runs ahead of a publisher /// that cannot send what it wrote. #[test] #[cfg(feature = "async")] fn a_drop_poll_does_not_pick_up_a_filled_group() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); producer.start_at(4).unwrap(); write_group(&producer, 4); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_next_group(&waiter).is_pending(), "group 0 is a miss"); let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asked for the batch"); }; assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(0))); write_filled(fill.accept(0, None), 0); for _ in 0..3 { assert!(subscriber.poll_drop(&waiter).is_pending()); assert!( fill.poll_wanted_changed(&waiter).is_pending(), "group 1 waits for a read" ); } assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(group))) if group.start == 0)); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(1))); } /// A fetch that joined a batch fill picks up its group as it resolves, so the batch moves /// on though the caller keeps the fetch and the reader that asked the batch narrowed past /// that group. #[test] #[cfg(feature = "async")] fn a_fetch_picks_up_its_group_in_a_batch_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); // Descending, so the reader narrows past group 2 with its end, which settles at once. let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(4).unwrap(); write_group(&producer, 4); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_recv_group(&waiter).is_ready(), "the live group"); assert!( subscriber.poll_recv_group(&waiter).is_pending(), "groups 0..4 are a miss" ); let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the reader asked for the batch"); }; assert_eq!((fill.start().group, fill.end()), (0, 4)); let mut fetch = producer.consume().fetch_group(2, None); assert!(kio::Task::poll(&mut *fetch, &waiter).is_pending()); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "the fetch joined the batch" ); subscriber.update(descending(Range::groups(0..2))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert_eq!( fill.poll_wanted_changed(&waiter), Poll::Ready(Some(2)), "only the fetch wants 2" ); write_filled(fill.accept(2, None), 2); assert!( fill.poll_wanted_changed(&waiter).is_pending(), "1 waits for 2 to be picked up" ); assert!(matches!(kio::Task::poll(&mut *fetch, &waiter), Poll::Ready(Ok(_)))); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(1))); } /// A reader forgets the misses it gave up once its window moves past them, as it forgets /// what it judged, so a feed skipping every other group costs it only the gaps it can /// still read. #[test] #[cfg(feature = "async")] fn a_reader_forgets_misses_behind_its_window() { let mut producer = track_producer("test", Info::timed()); // A received copy, whose missing groups may still arrive. producer.received(); let _dynamic = producer.requests(); let ascending = |groups| Subscription::default().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); producer.start_at(0).unwrap(); for key in (0..20).step_by(2) { write_group(&producer, key); drain_groups(&mut subscriber); elapse(&producer, Duration::from_secs(2)); drain_groups(&mut subscriber); } let given_up = |subscriber: &Subscriber| -> Vec<(u64, u64)> { let Inner::Plain(cursor) = &subscriber.inner else { unreachable!("a plain subscription"); }; let misses = cursor.misses.as_ref().expect("the reader judged its misses"); misses.given_up().iter().map(|run| (run.start, run.end)).collect() }; assert_eq!(given_up(&subscriber).len(), 9, "each skipped group is given up"); subscriber.update(ascending(15..)).unwrap(); write_group(&producer, 20); drain_groups(&mut subscriber); assert_eq!(given_up(&subscriber), [(15, 16), (17, 18)]); } /// A descending reader's miss split by a fill already in flight queues its newer fragment /// first too. #[test] #[cfg(feature = "async")] fn a_descending_reader_asks_its_newest_fragment_first() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut middle = producer.subscribe(replay().with_range(4..6)); producer.start_at(8).unwrap(); write_group(&producer, 8); // Held, so the middle reader's time start names it and only 5 is a miss. write_group(&producer, 4); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = middle.poll_recv_group(&waiter) {} let Poll::Ready(Ok(taken)) = dynamic.poll_requested_groups(&waiter) else { panic!("group 5 is a miss"); }; assert_eq!((taken.start().group, taken.end()), (5, 6)); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Descending)); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { asked.push((fill.start().group, fill.end())); } assert_eq!(asked, [(6, 8), (0, 4)]); } /// A group the handler refused that still arrives live is delivered. #[test] #[cfg(feature = "async")] fn a_refused_miss_that_arrives_late_is_read() { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); assert!(drained_drops(&mut subscriber).is_empty()); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; fill.drop_groups(0..2); write_group(&producer, 1); let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the late group is read"); }; assert_eq!(group.start, 1); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, 1)]); } /// A refused miss is named dropped as soon as the feed ends, since nothing more arrives, /// and the read ends with it. #[test] #[cfg(feature = "async")] fn a_refused_miss_is_dropped_once_the_feed_ends() { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; fill.drop_groups(0..2); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!( drained_drops(&mut subscriber).is_empty(), "the feed may still deliver them" ); let _producer = producer.finish().unwrap(); assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); } /// An end declared ahead of the live edge does not end the feed yet: a group missing /// below the edge may still arrive, so it is not named dropped. #[test] #[cfg(feature = "async")] fn an_end_ahead_of_the_edge_keeps_a_late_group() { let mut producer = track_producer("test", Info::timed()); // A received copy, whose missing groups may still arrive. producer.received(); let mut subscriber = producer.subscribe(replay().with_range(0..)); for key in [0, 1, 2, 4, 5] { write_group(&producer, key); } producer.finish_at(Position::group(10)).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0, 1, 2, 4, 5]); assert!(drained_drops(&mut subscriber).is_empty(), "group 3 may still arrive"); write_group(&producer, 3); assert_eq!(drain_groups(&mut subscriber), [3]); assert!(drained_drops(&mut subscriber).is_empty()); } /// With no handler to fill it, a miss is named dropped once the feed gives up on it. #[test] #[cfg(feature = "async")] fn a_miss_with_no_handler_is_named_dropped() { let (producer, dynamic, mut subscriber) = started_late(); drop(dynamic); let waiter = kio::Waiter::noop(); assert!(drained_drops(&mut subscriber).is_empty()); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { read.push(group.start); } assert_eq!(read, [2, 3]); assert!(drained_drops(&mut subscriber).is_empty()); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); } /// A handler that takes only fetches leaves a reader's miss to settle as with none, and /// still takes a fetch. #[test] #[cfg(feature = "async")] fn a_fetch_only_handler_takes_fetches_alone() { let (producer, dynamic, mut subscriber) = started_late(); let dynamic = dynamic.fetches_only(); let waiter = kio::Waiter::noop(); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { read.push(group.start); } assert_eq!(read, [2, 3]); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "the miss asked nothing" ); let mut fetch = producer.consume().fetch_group(1, None); assert!(kio::Task::poll(&mut *fetch, &waiter).is_pending()); let Poll::Ready(Ok(fill)) = dynamic.clone().poll_requested_groups(&waiter) else { panic!("the fetch is taken"); }; assert_eq!((fill.start().group, fill.end()), (1, 2)); assert!(fill.is_fetch()); } /// A fill whose handler leaves before taking it settles by run: a peer that starts its /// feed far past the reader's floor makes the miss wide, and walking it group by group /// would never finish. #[test] #[cfg(feature = "async")] fn a_wide_fill_its_handler_left_settles_by_run() { const WIDE: u64 = 1 << 40; let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..)); producer.start_at(WIDE).unwrap(); write_group(&producer, WIDE); assert_eq!(drain_groups(&mut subscriber), [WIDE]); drop(dynamic); assert!(subscriber.poll_recv_group(&kio::Waiter::noop()).is_pending()); assert!( drained_drops(&mut subscriber).is_empty(), "the feed may still deliver them" ); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, WIDE)]); } /// A handler refusing a wide fill one group at a time, as a FETCH-capable session does, /// advances after each refusal and coalesces the resolved run, forgetting consumed answers. /// `track_fill_refusal` benchmarks the cost over refusals and readers. #[test] #[cfg(feature = "async")] fn a_fill_refused_group_by_group_stays_linear() { const WIDE: u64 = 1 << 40; const REFUSED: u64 = 32; let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..)); producer.start_at(WIDE).unwrap(); write_group(&producer, WIDE); assert_eq!(drain_groups(&mut subscriber), [WIDE]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&kio::Waiter::noop()) else { panic!("the miss below the start is asked"); }; for refused in 1..=REFUSED { let key = fill.wanted().unwrap(); fill.drop_groups(key..key + 1); assert!(subscriber.poll_recv_group(&kio::Waiter::noop()).is_pending()); assert_eq!(fill.wanted(), Some(WIDE - refused - 1)); assert_eq!(fill.answers(), 0, "the reader consumed the refusal"); assert_eq!( fill.resolved_runs(), vec![WIDE - refused..WIDE], "adjacent refusals remain one resolved run" ); } } /// A descending reader joining a queued fill makes it newest first, as any descending /// subscriber does a subscription. #[test] fn a_descending_reader_turns_a_queued_fill_newest_first() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut ascending = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); let mut descending = producer.subscribe(replay().with_range(0..)); producer.start_at(4).unwrap(); write_group(&producer, 4); let _ = ascending.poll_recv_group(&kio::Waiter::noop()); let _ = descending.poll_recv_group(&kio::Waiter::noop()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&kio::Waiter::noop()) else { panic!("the miss below the start is asked"); }; assert_eq!((fill.start().group, fill.end()), (0, 4)); assert_eq!(fill.order(), Order::Descending); assert!( dynamic.poll_requested_groups(&kio::Waiter::noop()).is_pending(), "both readers share the fill" ); } /// A run that passes over a group another request is bringing serves the groups around /// it, across the runs that answered groups split the fill into, and leaves that group /// to [`FillGroups::next_run`] once nothing else is owed. #[test] #[cfg(feature = "async")] fn a_run_passes_over_a_skipped_group() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut descending = producer.subscribe(replay().with_range(0..)); producer.start_at(6).unwrap(); write_group(&producer, 6); let _ = descending.poll_recv_group(&kio::Waiter::noop()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&kio::Waiter::noop()) else { panic!("the miss below the start is asked"); }; assert_eq!(fill.order(), Order::Descending); assert_eq!(fill.next_run(), Some(0..6)); assert_eq!(fill.next_run_except(5), Some(0..5)); assert_eq!(fill.next_run_except(2), Some(3..6)); fill.drop_groups(3..4); assert_eq!(fill.next_run(), Some(4..6)); assert_eq!(fill.next_run_except(5), Some(4..5)); assert_eq!(fill.next_run_except(4), Some(5..6)); fill.drop_groups(4..5); assert_eq!( fill.next_run_except(5), Some(0..3), "older runs are served while 5 waits" ); fill.drop_groups(0..3); assert_eq!(fill.next_run_except(5), None); assert_eq!(fill.next_run(), Some(5..6)); } /// A reader changing its priority, budget, or order moves the fill it waits on, queued /// or taken, so the handler serves the rest under the new values, even ones lower than /// it was asked with. #[test] #[cfg(feature = "async")] fn a_reader_update_reaches_its_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); producer.start_at(4).unwrap(); write_group(&producer, 4); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), [4]); subscriber.update(subscriber.subscription().with_priority(3)).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the miss below the start is asked"); }; assert_eq!((fill.start().group, fill.end()), (0, 4)); assert_eq!( (fill.priority(), fill.order()), (3, Order::Ascending), "refreshed while queued" ); assert_eq!(fill.wanted(), Some(0)); write_filled(fill.accept(0, None), 0); let budget = Duration::from_secs(20); let update = subscriber .subscription() .with_priority(-2) .with_delay(Delay::default().with_max(budget)) .with_order(Order::Descending); subscriber.update(update).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0]); assert_eq!(fill.priority(), -2); assert_eq!(fill.max_delay(), Some(budget)); assert_eq!(fill.order(), Order::Descending); assert_eq!(fill.wanted(), Some(3), "the rest is served newest first"); } /// An ordered reader waits on a fill for its miss rather than skipping past it. #[test] #[cfg(feature = "async")] fn an_ordered_read_waits_on_a_fill() { let (_producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); let mut next = || match subscriber.poll_next_group(&waiter) { Poll::Ready(Ok(Some(group))) => Some(group.start), Poll::Pending => None, _ => panic!("the read ended"), }; assert_eq!(next(), None, "group 0 is being filled"); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("below the start is a miss"); }; write_filled(fill.accept(0, None), 0); assert_eq!(next(), Some(0)); assert_eq!(next(), None, "group 1 is being filled"); fill.drop_groups(1..2); assert_eq!(next(), Some(2)); assert_eq!(next(), Some(3)); } /// A fill nobody waits on any more is withdrawn: it names nothing dropped, so a later /// reader can still ask for the same history. #[test] #[cfg(feature = "async")] fn a_withdrawn_fill_names_nothing_dropped() { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the gap is a miss"); }; drop(subscriber); assert_eq!(fill.wanted(), None); drop(fill); assert!(drops(&producer).is_empty()); } /// Fetch group 3 and drop it through the fill it queues, returning how the fetch ended. fn drop_through_fill(producer: &Producer, dynamic: &Requests, drop_all: bool) -> Error { let consumer = producer.consume(); let mut fetch = consumer.fetch_group(3, None); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the fill is ready"); }; match drop_all { true => fill.drop_groups(0..10), false => drop(fill), } match kio::Task::poll(&mut *fetch, &waiter) { Poll::Ready(Err(err)) => err, _ => panic!("the fetch fails"), } } fn drops(producer: &Producer) -> Vec> { producer .state .read() .drops .iter() .map(|(range, _)| range.clone()) .collect() } /// A group the source no longer has fails its fetch, and is named dropped for good only /// once the live feed gave up on it too: a fetch answered before a late group header /// arrives says nothing about the feed. #[kio::test] async fn dropped_fill_group_fails_its_fetch() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); write_group(&producer, 5); assert!(matches!(drop_through_fill(&producer, &dynamic, true), Error::NotFound)); assert!(drops(&producer).is_empty(), "the feed may still deliver 3"); kio::clock::sim::advance(Duration::from_secs(60)).await; assert!(matches!(drop_through_fill(&producer, &dynamic, true), Error::NotFound)); assert_eq!(drops(&producer), vec![3..4], "clipped to the fill"); } /// Dropping a fill a reader still waits on names what it left as dropped. #[kio::test] async fn fill_dropped_unanswered_names_the_rest_dropped() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); write_group(&producer, 5); kio::clock::sim::advance(Duration::from_secs(60)).await; assert!(matches!(drop_through_fill(&producer, &dynamic, false), Error::Dropped)); assert_eq!(drops(&producer), vec![3..4]); } #[kio::test] #[cfg(feature = "async")] async fn fetch_aborts_with_track() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut pending = consumer.fetch_group(3, None); assert!(kio::Task::poll(&mut *pending, &kio::Waiter::noop()).is_pending()); producer.abort(Error::Cancel).unwrap(); assert!(pending.await.is_err()); drop(dynamic); } /// Ranges edited one key at a time resolve and cover what the same ranges given whole do, /// so an edit leaves the subscriber as a fresh subscription to the ranges it keeps would be. #[test] #[cfg(feature = "async")] fn edited_ranges_match_the_same_ranges_given_whole() { use rand::{RngExt, SeedableRng}; let mut rng = rand::rngs::StdRng::seed_from_u64(0x65646974); let producer = track_producer("test", Info::timed()); for key in (0..20_000).step_by(1000) { write_group(&producer, key); } let range = |rng: &mut rand::rngs::StdRng| { let start = match rng.random_range(0..4) { 0 => Anchor::Relative(rng.random_range(0..5)), _ => Anchor::Time(rng.random_range(0..24_000)), }; let end = rng.random_bool(0.6).then(|| Anchor::Time(rng.random_range(0..24_000))); Range { start, end } }; // Key 0 is the default subscription's one range. let mut live = BTreeMap::from([(0, Range::oldest())]); let mut next = 0; let mut edited = producer.subscribe(Subscription::default()); for step in 0..400 { let retire: Vec = live.keys().copied().filter(|_| rng.random_bool(0.2)).collect(); let add: Vec<(u64, Range)> = (0..rng.random_range(0..3)) .map(|_| { next += 1; (next, range(&mut rng)) }) .collect(); for key in &retire { live.remove(key); } live.extend(add.iter().copied()); edited .edit(Edit { retire, add, priority: 0, delay: Delay::default(), order: Order::default(), }) .unwrap(); let mut whole = producer.subscribe(Subscription::default().with_ranges(live.values().copied())); for (index, key) in live.keys().enumerate() { assert_eq!( edited.resolved_range(*key), whole.resolved_range(index as u64), "step {step}, key {key}" ); } assert_eq!( edited.runs_in_range(0, 30_000), whole.runs_in_range(0, 30_000), "step {step}" ); } } /// Write a finished one-frame group, presented at its key in milliseconds. fn write_group(producer: &Producer, key: u64) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } /// Every group the subscriber hands out right now, in arrival order. fn drain_groups(subscriber: &mut Subscriber) -> Vec { let mut keys = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&kio::Waiter::noop()) { keys.push(group.start); } keys } /// A key the publisher aborts and re-creates is handed out once. #[test] fn a_recreated_group_is_handed_out_once() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); write_group(&producer, 0); let open = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0, 1]); open.abort(Error::Cancel).unwrap(); write_group(&producer, 1); write_group(&producer, 2); assert_eq!(drain_groups(&mut subscriber), [2]); } /// A gap is remembered until a group above it is past the budget, max delay or max /// age alike. Then every group below that one counts as handed out. #[test] fn a_gap_is_forgotten_once_a_group_above_it_is_past_the_budget() { let budget = Duration::from_millis(10); let readers = [ ( Info::timed(), Subscription::default().with_delay(Delay::default().with_max(budget)), ), ( Info::timed().with_max_age(budget), Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(3600))), ), ]; for (info, subscription) in readers { let producer = track_producer("test", info); let mut subscriber = producer.subscribe(subscription); // Group 3 never arrives. for key in [0, 1, 2, 4, 5, 6] { write_group(&producer, key); } assert_eq!(drain_groups(&mut subscriber), [0, 1, 2, 4, 5, 6]); // Group 4 reaches until group 5 presents at 5ms, so it is past a 10ms budget once // the edge presents at 15ms. for key in 7..=14 { write_group(&producer, key); assert_eq!(drain_groups(&mut subscriber), [key]); assert_eq!(subscriber.delivered_runs(), 2, "the gap at 3 is kept at {key}"); } write_group(&producer, 15); assert_eq!(drain_groups(&mut subscriber), [15]); assert_eq!(subscriber.delivered_runs(), 1, "the gap at 3 is forgotten"); write_group(&producer, 3); assert_eq!(drain_groups(&mut subscriber), Vec::::new()); } } /// An endless track with groups missing remembers only the gaps within the budget, and /// still delivers an unseen group that arrives late within it. #[test] fn an_endless_track_with_gaps_stays_bounded() { let producer = track_producer("test", Info::timed()); // Groups present 1ms apart, so the budget spans ten of them. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; for key in 0..3000u64 { // Every third group never arrives. if key % 3 == 1 { continue; } write_group(&producer, key); assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); // 2998 is within the budget, 2500 far past it. write_group(&producer, 2998); write_group(&producer, 2500); assert_eq!(drain_groups(&mut subscriber), [2998]); } /// A group of one untimed frame at `key`. fn write_untimed_group(producer: &Producer, key: u64) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); } /// The wall clock forgets the gaps on an untimed track, which has no media time to /// measure them by. #[test] fn an_endless_untimed_track_with_gaps_stays_bounded() { let producer = track_producer("test", None); // Groups arrive 1ms apart, so the budget spans ten of them. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; for key in 0..3000u64 { elapse(&producer, Duration::from_millis(1)); // Every third group never arrives. if key % 3 == 1 { continue; } write_untimed_group(&producer, key); assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); // 2998 is within the budget, 2500 far past it. write_untimed_group(&producer, 2998); write_untimed_group(&producer, 2500); assert_eq!(drain_groups(&mut subscriber), [2998]); } /// A scale-less track's max age counts bare key units as milliseconds over the whole key /// range, so keys past a scaled timestamp's bound age by their distance as low ones do. #[test] fn a_scale_less_track_ages_wide_keys_by_their_distance() { for base in [0, 1u64 << 61, u64::MAX - 30_000] { let producer = track_producer("test", None); for offset in [0, 10_000, 20_000] { write_untimed_group(&producer, base + offset); } // All three arrived at once, so only the keys' distance makes the oldest stale: its // successor starts 10 s behind the edge. The middle one reaches the edge. let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(5)))); assert_eq!( drain_groups(&mut subscriber), [base + 10_000, base + 20_000], "from {base}" ); } } /// An empty successor gives the lowest group above a gap no reach, so media time never /// forgets that gap or any after it. The wall clock still does. #[test] fn an_empty_successor_still_lets_the_gaps_go() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; for key in 0..3000u64 { elapse(&producer, Duration::from_millis(1)); if key % 3 == 1 { continue; } // Group 2 sits above the first gap, and its successor never presents a frame. match key { 3 => { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } _ => write_group(&producer, key), } assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); } /// A gap within the budget below a run of frameless groups is forgotten once the group /// above it is superseded for a full budget, so the run stops being rescanned. #[test] fn a_gap_below_a_frameless_run_lets_the_floor_advance() { const RUN: u64 = 100; let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); write_group(&producer, 0); // Group 1 never arrives. let open: Vec<_> = (2..2 + RUN) .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); assert_eq!( drain_groups(&mut subscriber), [0].into_iter().chain(2..2 + RUN).collect::>() ); assert_eq!(subscriber.delivered_runs(), 2, "the floor and the run above the gap"); elapse(&producer, Duration::from_secs(1)); write_group(&producer, 2 + RUN); assert_eq!(drain_groups(&mut subscriber), [2 + RUN]); assert_eq!(subscriber.delivered_runs(), 1, "the gap is forgotten"); drop(open); } /// Gaps below groups aborted since, but still cached, share the successor past them, so /// one hand-out that puts them all past the budget forgets every one. The gap below a /// group still within it is kept. `track_gap_flush` measures the cost. #[test] fn gaps_below_aborted_groups_are_forgotten_together() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); let mut handed = Vec::new(); for key in (0..20).step_by(2) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); handed.push(group); } assert_eq!(drain_groups(&mut subscriber).len(), 10); for group in handed.drain(..5) { group.abort(Error::Cancel).unwrap(); } // Groups 0 to 8 reach until group 10 at 10ms, which a group at 1.01s puts past the // budget. Group 10 reaches until 12ms, so the gap at 9 is kept. write_group(&producer, 1010); assert_eq!(drain_groups(&mut subscriber), [1010]); assert_eq!( subscriber.delivered_runs(), 7, "the floor at 8, then 10 to 18 and 1010 apart" ); } /// A common wall-clock successor retires the gaps below an aborted prefix, but a /// later arrival keeps the next gap within its budget. #[test] fn wall_time_batches_aborted_gaps_without_forgetting_a_fresh_successor() { let budget = Duration::from_secs(1); let producer = track_producer("test", None); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let mut open = Vec::new(); for key in (0..=10).step_by(2) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); open.push(group); } assert_eq!(drain_groups(&mut subscriber), [0, 2, 4, 6, 8, 10]); for group in open.drain(..5) { group.abort(Error::Cancel).unwrap(); } elapse(&producer, budget / 2); write_untimed_group(&producer, 12); assert_eq!(drain_groups(&mut subscriber), [12]); elapse(&producer, budget / 2); write_untimed_group(&producer, 14); assert_eq!(drain_groups(&mut subscriber), [14]); assert_eq!(subscriber.delivered_runs(), 4, "floor through 8, then 10, 12 and 14"); assert!( !subscriber.record().delivered.contains(9), "the next gap is still remembered" ); write_untimed_group(&producer, 11); assert_eq!( drain_groups(&mut subscriber), [11], "the gap below fresh group 12 remains" ); } /// An untimed sparse feed whose groups all arrive within the budget keeps every gap /// and still hands out every group. The hand-out cost stays flat over the retained /// gaps; `track_sparse_read` measures it. #[test] fn an_untimed_sparse_feed_keeps_its_gaps() { let producer = untimed_producer(); let mut subscriber = producer.subscribe(replay()); for key in (0..2000u64).step_by(2) { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); assert_eq!(drain_groups(&mut subscriber), [key]); } assert_eq!(subscriber.delivered_runs(), 1000); } /// A track whose newest group is `newest`, every group fresh to [`replay`]. #[cfg(feature = "async")] fn quiet_track(newest: u64) -> Producer { let mut producer = track_producer("test", Info::timed()); for millis in 0..=newest { append_at(&mut producer, millis); } producer } /// A live range merged with one starting past the live edge still starts at the newest /// group, and the aggregate keeps both rather than flooring the live one at the higher /// start. The later start is a time the newest group contains, so both read it. #[test] #[cfg(feature = "async")] fn a_live_range_merged_with_one_above_the_live_edge_gets_the_newest_group() { let mut producer = quiet_track(3); let mut ahead = producer.subscribe(replay().with_range(4..)); let mut live = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!( producer.subscription().unwrap().ranges, vec![Range::live(), Range::groups(4..)] ); assert_eq!(drain(&mut ahead), vec![3]); assert_eq!(drain(&mut live), vec![3]); append_at(&mut producer, 4); assert_eq!(drain(&mut ahead), vec![4]); assert_eq!(drain(&mut live), vec![4]); } /// A live range on a track with no groups yet starts at the first one to arrive. #[test] #[cfg(feature = "async")] fn a_live_range_on_an_empty_track_starts_at_the_first_group() { let mut producer = track_producer("test", Info::timed()); let mut live = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!(drain(&mut live), Vec::::new()); append_at(&mut producer, 0); append_at(&mut producer, 1); assert_eq!(drain(&mut live), vec![0, 1]); } /// Fetched backfill past the newest produced group moves no relative start, whether the /// track had a produced group at subscribe or not. #[kio::test] #[cfg(feature = "async")] async fn a_live_range_ignores_a_fetched_group_past_the_newest_group() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut early = producer.subscribe(replay().with_ranges([Range::live()])); let _fetching = consumer.fetch_group(5, None); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); req.accept(None).unwrap().expect("a fresh group").finish().unwrap(); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); let mut late = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!(drain(&mut early), vec![0]); assert_eq!(drain(&mut late), vec![0]); } /// A relative start on an empty track counts back from the first group to arrive, even /// once that group is evicted before the subscriber reads. #[test] #[cfg(feature = "async")] fn a_relative_start_on_an_empty_track_survives_its_first_group_leaving() { let producer = untimed_producer(); let buffering = Range { start: Anchor::Relative(2), end: None, }; let mut behind = producer.subscribe(replay().with_ranges([buffering])); let write = |key| { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); group.finish().unwrap(); }; write(10); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); write(11); assert!(!producer.state.read().lookup.contains_key(&10), "group 10 evicted"); write(8); let mut delivered = drain(&mut behind); delivered.sort_unstable(); assert_eq!(delivered, vec![8, 11]); } /// A relative start buffers that many groups behind the live edge, whatever a range /// starting past the edge merged beside it asks for: that one reads from the newest /// group, which contains its start. #[test] #[cfg(feature = "async")] fn a_buffering_range_merged_with_one_above_the_live_edge_keeps_its_backlog() { let mut producer = quiet_track(5); let mut ahead = producer.subscribe(replay().with_range(8..)); let buffering = Range { start: Anchor::Relative(2), end: None, }; let mut behind = producer.subscribe(replay().with_ranges([buffering])); assert_eq!( producer.subscription().unwrap().ranges, vec![buffering, Range::groups(8..)] ); assert_eq!(drain(&mut behind), vec![3, 4, 5]); for millis in 6..=8 { append_at(&mut producer, millis); } assert_eq!(drain(&mut behind), vec![6, 7, 8]); assert_eq!(drain(&mut ahead), vec![5, 6, 7, 8]); } /// An untimed track replays from group 0 only what the wall clock hasn't made stale. #[test] #[cfg(feature = "async")] fn an_untimed_range_from_group_zero_replays_what_is_still_fresh() { let mut producer = untimed_producer(); for _ in 0..5 { append_untimed(&mut producer); elapse(&producer, Duration::from_secs(1)); } let budget = Duration::from_millis(1500); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_range(0..), ); assert_eq!(drain(&mut subscriber), vec![3, 4]); } /// A group `key` whose first frame lands at `millis`, or the write's refusal. A /// refused group is aborted, as a caller would. fn start_at_millis(producer: &Producer, key: u64, millis: u64) -> Result { let mut group = producer.create_group(group::Info { start: key })?; match group.write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) { Ok(()) => Ok(group), Err(err) => { group.abort(err.clone()).unwrap(); Err(err) } } } /// Take group `key` out of the cache as eviction does. fn evict(producer: &Producer, key: u64) { producer.modify().unwrap().evict_group(key, Error::Evicted); } /// A rewound first frame is refused and leaves the group as it was, so the publisher can /// still write a start at or after the group before it. #[test] fn an_appended_rewind_is_refused() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 1000); let mut group = producer.append_next().unwrap(); let refused = group.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); assert_eq!(group.frame_count(), 0); assert_eq!(group.handle().timestamp(), None); // Starts strictly increase, so an equal start is refused like a rewind. let tie = group.write_frame(Timestamp::from_millis(1000).unwrap(), &b"x"[..]); assert!(matches!(tie, Err(Error::TimestampMismatch)), "{tie:?}"); group .write_frame(Timestamp::from_millis(1001).unwrap(), &b"x"[..]) .unwrap(); } /// The one-shot write ends the group it appended rather than leaving it open. #[test] fn a_rewound_single_frame_group_is_aborted() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 1000); let refused = producer.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); let state = producer.state.read(); assert!(state.lookup.get(&500).unwrap().group.is_aborted()); } /// Groups land out of key order, so a start is bounded from above as well as below. #[test] fn a_rewind_is_refused_whichever_group_lands_first() { let producer = track_producer("test", Info::timed()); start_at_millis(&producer, 5, 1000).unwrap().finish().unwrap(); assert!(matches!( start_at_millis(&producer, 3, 2000), Err(Error::TimestampMismatch) )); start_at_millis(&producer, 3, 500).unwrap().finish().unwrap(); for millis in [400, 500, 1000] { assert!(matches!( start_at_millis(&producer, 4, millis), Err(Error::TimestampMismatch) )); } start_at_millis(&producer, 4, 800).unwrap().finish().unwrap(); } /// The batched and streamed writes check frame 0 the same way, before writing anything. #[test] fn every_write_path_refuses_a_rewind() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 1000); let rewound = Timestamp::from_millis(500).unwrap(); let mut batched = producer.append_next().unwrap(); let mut frames = frame::Buffer::<4>::new(); frames .push(frame::Frame::new(rewound, bytes::Bytes::from_static(b"x"))) .unwrap(); assert!(matches!( batched.write_frames(&mut frames), Err(Error::TimestampMismatch) )); assert_eq!(frames.len(), 1, "the batch is left to retry"); assert_eq!(batched.frame_count(), 0); let streamed = producer.append_next().unwrap(); let handle = streamed.handle(); let refused = streamed .create_frame(frame::Info { size: 1, timestamp: rewound, }) .err(); assert!(matches!(refused, Some(Error::TimestampMismatch)), "{refused:?}"); assert_eq!(handle.frame_count(), 0); assert!(handle.is_aborted(), "a refused frame aborts its group"); } /// Eviction is not a key prefix: a group stamped after its predecessor was evicted /// is still held above it, though an older group survives below. #[test] fn an_evicted_predecessor_still_bounds_a_delayed_stamp() { let producer = track_producer("test", Info::timed()); start_at_millis(&producer, 0, 0).unwrap().finish().unwrap(); start_at_millis(&producer, 1, 1000).unwrap().finish().unwrap(); let mut delayed = producer.create_group(group::Info { start: 2 }).unwrap(); evict(&producer, 1); let refused = delayed.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); delayed .write_frame(Timestamp::from_millis(1500).unwrap(), &b"x"[..]) .unwrap(); } /// A group at or below the eviction watermark is checked against what survives, and /// otherwise accepted unverified: it bounds no later group, so a refetch of evicted /// history is never refused on its account, and it drives no media-time expiry. #[test] fn a_late_group_below_the_watermark_is_unverified() { let producer = track_producer("test", Info::timed()); for (key, millis) in [(0, 1000), (1, 2000), (2, 3000), (3, 4000)] { start_at_millis(&producer, key, millis).unwrap().finish().unwrap(); } evict(&producer, 1); evict(&producer, 2); // Below the surviving group 0, or above the watermark at group 2. for millis in [500, 3500] { assert!(matches!( start_at_millis(&producer, 1, millis), Err(Error::TimestampMismatch) )); } let late = start_at_millis(&producer, 1, 2900).unwrap(); assert_eq!(late.handle().bound(), None, "a late group bounds nothing"); assert_eq!(late.handle().timestamp(), None, "nor drives media-time expiry"); // Group 2 really started before late group 1 claimed to. start_at_millis(&producer, 2, 2500).unwrap().finish().unwrap(); } /// Subscribers may have seen an aborted group's start, so a replacement at its key /// leaves that start as the watermark, and no later group can start before it. #[test] fn a_replaced_aborted_group_still_bounds_later_ones() { let producer = track_producer("test", Info::timed()); start_at_millis(&producer, 0, 1000) .unwrap() .abort(Error::Cancel) .unwrap(); let replacement = start_at_millis(&producer, 0, 500).unwrap(); assert_eq!(replacement.handle().bound(), None, "a late replacement bounds nothing"); assert!(matches!( start_at_millis(&producer, 1, 600), Err(Error::TimestampMismatch) )); start_at_millis(&producer, 1, 1500).unwrap().finish().unwrap(); } /// A group received from past its first frame stamps a later frame, which may overlap /// the next group's start. It has no known start, so it is neither checked nor a bound. #[test] fn a_resumed_partial_group_is_neither_checked_nor_a_bound() { let producer = track_producer("test", Info::timed()); start_at_millis(&producer, 0, 1000).unwrap().finish().unwrap(); let mut partial = producer.create_group(group::Info { start: 1 }).unwrap(); partial.start_at(3).unwrap(); partial .write_frame(Timestamp::from_millis(2100).unwrap(), &b"x"[..]) .unwrap(); start_at_millis(&producer, 2, 2000).unwrap().finish().unwrap(); } /// An untimed track has no timeline, so its groups land in any order. #[test] fn an_untimed_track_takes_groups_in_any_order() { let producer = untimed_producer(); for key in [5, 3, 4] { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) .unwrap(); } } /// A subscriber reads only its own ranges, whatever another subscriber's widen the /// aggregate to, in arrival order and in key order alike. #[test] #[cfg(feature = "async")] fn a_subscriber_reads_only_its_own_ranges() { let mut producer = track_producer("test", Info::timed()); let mut everything = producer.subscribe(replay()); let ranges = [Range::groups(1..3), Range::groups(5..)]; let mut mine = producer.subscribe(replay().with_ranges(ranges)); let mut ordered = producer.subscribe(replay().with_ranges(ranges)).ordered(); for second in 0..7 { append_at(&mut producer, second * 1000); } assert_eq!(drain(&mut everything), vec![0, 1, 2, 3, 4, 5, 6]); assert_eq!(drain(&mut mine), vec![1, 2, 5, 6]); assert_eq!(drain_ordered(&mut ordered), vec![1, 2, 5, 6]); } /// Replacing the ranges re-offers what the old ones held back, and a relative anchor /// counts back from the latest group as of the update, not the subscribe. #[test] #[cfg(feature = "async")] fn replaced_ranges_resolve_again() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..2)); for second in 0..4 { append_at(&mut producer, second * 1000); } assert_eq!(drain(&mut subscriber), vec![0, 1]); subscriber.update(replay().with_range(0..)).unwrap(); assert_eq!( drain(&mut subscriber), vec![3, 2], "held back, now in range, newest first" ); // The latest group is 3 now, so one back is 2, already read: only new groups follow. let mut late = producer.subscribe(replay().with_range(..1)); assert_eq!(drain(&mut late), vec![0]); late.update(replay().with_ranges([Range { start: Anchor::Relative(1), end: None, }])) .unwrap(); assert_eq!(drain(&mut late), vec![3, 2]); append_at(&mut producer, 4000); assert_eq!(drain(&mut late), vec![4]); } /// A cap moving beside fixed ranges owes, of every cached group, exactly what was owed and /// is still admitted, and what is newly admitted: what the cap gains is owed where the /// cache holds a group now, and what it loses is cut from what is owed. #[test] fn a_moving_cap_owes_what_it_newly_admits() { let seed = std::cell::Cell::new(0x2545_f491_4f6c_dd1d_u64); let next = |bound: u64| { let mut x = seed.get(); x ^= x << 13; x ^= x >> 7; x ^= x << 17; seed.set(x); x % bound }; let at = |millis| Timestamp::from_millis(millis).unwrap(); const KEYS: u64 = 40; let mut owed = 0; for round in 0..150 { let producer = track_producer("test", Info::timed()); // One open range, and fixed ranges around and above where the feed starts. let ranges = std::iter::once(Range { start: Anchor::Time(next(KEYS)), end: None, }) .chain((0..1 + next(6)).map(|_| { let start = next(KEYS); Range::groups(start..start + 1 + next(3)) })); let mut subscriber = producer.subscribe(Subscription::default().with_ranges(ranges)); let mut groups = Vec::new(); for step in 0..30 { let key = next(KEYS); match next(5) { 0 | 1 => { if let Ok(mut group) = producer.create_group(group::Info { start: key }) { if next(2) == 0 { group .write_frame(at(key * 20), bytes::Bytes::from_static(b"x")) .unwrap(); } groups.push(group); } } 2 => { if let Some(group) = groups.iter_mut().find(|group| group.start == key) && group.handle().timestamp().is_none() { let _ = group.write_frame(at(key * 20), bytes::Bytes::from_static(b"x")); } } 3 => match next(4) { // The cap moves, and now and then lifts. 0 => subscriber.set_range(..), _ => subscriber.set_range(..next(KEYS + 1)), }, _ => { // Read now and then, so the walk passes groups and owes them again. let _ = subscriber.poll_recv_group(&kio::Waiter::noop()); } } let Inner::Plain(cursor) = &mut subscriber.inner else { unreachable!("a local subscriber reads its own cursor"); }; cursor.sync(); let (before, was, primed) = ( cursor.filter.admitted.runs(), cursor.filter.owed.clone(), cursor.filter.admitted_at.is_some(), ); cursor.admit(); let now = cursor.filter.admitted.runs(); let held = |runs: &[ops::RangeInclusive], key: u64| runs.iter().any(|run| run.contains(&key)); let cached: Vec = cursor.state.read().lookup.keys().copied().collect(); for key in cached { let expected = primed && held(&now, key) && (was.contains(&key) || !held(&before, key)); assert_eq!( cursor.filter.owed.contains(&key), expected, "round {round} step {step}: group {key}, admitted {before:?} then {now:?}, owed {was:?}" ); owed += usize::from(expected); } for run in cursor.filter.owed.iter() { assert!( cursor.filter.admitted.overlapping(run.clone()) == vec![run.clone()], "round {round} step {step}: owed {run:?} beyond {now:?}" ); } } } assert!(owed > 100, "only {owed} groups owed"); } /// A narrow reader under an open aggregate passes the groups outside its range holding /// nothing for them, however many arrive, and widening its ranges still serves the /// ones the cache holds in the new span, newest first and once each. #[test] #[cfg(feature = "async")] fn a_narrow_reader_holds_nothing_for_excluded_groups() { let producer = track_producer("test", Info::timed()); let _wide = producer.subscribe(replay()); let mut narrow = producer.subscribe(replay().with_range(0..2)); for key in 0..1000 { producer .create_group(group::Info { start: key }) .unwrap() .finish() .unwrap(); } assert_eq!(drain(&mut narrow), vec![0, 1]); let Inner::Plain(cursor) = &narrow.inner else { unreachable!() }; assert!(cursor.filter.owed.is_empty(), "nothing owed for excluded groups"); assert_eq!(cursor.filter.admitted.runs().len(), 1, "only the range is kept"); narrow .update(replay().with_ranges([Range::groups(0..2), Range::groups(5..8)])) .unwrap(); assert_eq!(drain(&mut narrow), vec![7, 6, 5]); narrow.set_range(..9); narrow.update(replay().with_range(0..)).unwrap(); assert_eq!(drain(&mut narrow), vec![8, 4, 3, 2]); narrow.set_range(..); assert_eq!(drain(&mut narrow), (9..1000).rev().collect::>()); } /// Lowering the floor re-offers the cached groups the arrival walk passed below it, /// as widening any range does. #[test] #[cfg(feature = "async")] fn a_lowered_floor_reoffers_what_arrived_below_it() { let mut producer = track_producer("test", Info::timed()); for second in 0..6 { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(replay().with_range(5..)); assert_eq!(drain(&mut subscriber), vec![5]); subscriber.update(replay().with_range(2..)).unwrap(); assert_eq!( drain(&mut subscriber), vec![4, 3, 2], "passed below the floor, now in range" ); subscriber.update(replay().with_range(0..)).unwrap(); assert_eq!(drain(&mut subscriber), vec![1, 0]); // Raising and lowering it again re-offers only what is cached in between, and // nothing already served goes out twice. subscriber.update(replay().with_range(4..)).unwrap(); assert_eq!(drain(&mut subscriber), Vec::::new()); subscriber.update(replay().with_range(0..)).unwrap(); assert_eq!(drain(&mut subscriber), Vec::::new()); append_at(&mut producer, 6000); assert_eq!(drain(&mut subscriber), vec![6]); } /// Lowering a reader's local floor offers the cached groups below the old one that it /// passed over, in its order, and never a group twice. #[test] #[cfg(feature = "async")] fn a_lowered_range_reoffers_the_unread_cached_groups() { let mut producer = track_producer("test", Info::timed()); for second in 0..6 { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(replay()).with_range(4..); assert_eq!(drain(&mut subscriber), vec![4, 5]); let mut subscriber = subscriber.with_range(1..); assert_eq!(drain(&mut subscriber), vec![3, 2, 1], "passed below the floor"); let mut subscriber = subscriber.with_range(..=1); assert_eq!(drain(&mut subscriber), vec![0], "the floor and the cap both assign"); let mut subscriber = subscriber.with_range(..); assert_eq!(drain(&mut subscriber), Vec::::new(), "nothing goes out twice"); append_at(&mut producer, 6000); assert_eq!(drain(&mut subscriber), vec![6]); } /// Only the keys a subscriber asked for are named dropped to it. #[test] #[cfg(feature = "async")] fn drops_are_named_only_within_the_ranges() { let producer = track_producer("test", Info::timed()); // Held, so each range's time start names its own group. write_group(&producer, 2); write_group(&producer, 6); let mut subscriber = producer.subscribe(replay().with_ranges([Range::groups(2..4), Range::groups(6..8)])); for keys in [0..2, 3..6, 7..10] { producer.drop_groups(keys, StreamError::NotFound); } let waiter = kio::Waiter::noop(); let mut dropped = Vec::new(); while let Poll::Ready(range) = subscriber.poll_drop(&waiter) { dropped.push(range); } assert_eq!( dropped, vec![ (3..4, Hole::drop(StreamError::NotFound)), (7..8, Hole::drop(StreamError::NotFound)) ] ); } /// A received stale run settles at once only a reader no wider than the budget it was /// judged under, whose range covers it and everything above it; any other reader treats /// it as unknown. A skip settles every reader. #[test] #[cfg(feature = "async")] fn a_received_stale_drop_settles_only_the_readers_it_covers() { let producer = track_producer("test", Info::timed()); // A received copy's serving session keeps a handler, so an unsettled run is asked // rather than given up as unfillable. let _dynamic = producer.requests(); let budget = Duration::from_millis(500); let reader = |max: Duration, groups: Range| { producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(max)) .with_ranges([groups]), ) }; let mut covering = reader(budget, Range::groups(0..)); let mut narrower = reader(Duration::from_millis(100), Range::groups(0..)); let mut wider = reader(Duration::from_secs(5), Range::groups(0..)); let mut capped = reader(budget, Range::groups(0..5)); // Held, so a time start there names it rather than reaching below. write_group(&producer, 3); let mut above = reader(budget, Range::groups(3..)); producer.give_up(1..3, Hole::stale(budget)); producer.skip_groups(4..5); let waiter = kio::Waiter::noop(); let dropped = |subscriber: &mut Subscriber| { let mut runs = Vec::new(); while let Poll::Ready(run) = subscriber.poll_drop(&waiter) { runs.push(run); } runs }; let settled = vec![(1..3, Hole::stale(budget)), (4..5, Hole::Skip)]; assert_eq!(dropped(&mut covering), settled); assert_eq!(dropped(&mut narrower), settled, "a narrower budget is settled too"); assert_eq!(dropped(&mut wider), [(4..5, Hole::Skip)], "a wider budget waits"); assert_eq!( dropped(&mut capped), [(4..5, Hole::Skip)], "a capped range may judge otherwise" ); assert_eq!( dropped(&mut above), [(4..5, Hole::Skip)], "the stale run is outside its range" ); } /// A received stale run in a reader's range settles it once a group arriving places its /// open range's time start low enough to read every group above the run, so it is /// reported then, as a scan of the whole window would, though the window grew only above /// the run. #[test] #[cfg(feature = "async")] fn a_stale_run_settles_once_the_open_range_reaches_it() { let producer = track_producer("test", Info::timed()); // An unsettled run is asked of a handler rather than given up as unfillable. let _dynamic = producer.requests(); let budget = Duration::from_millis(500); // Time 0 names group 0, so the first range reads through 5 at once; the open one waits // on a group to place its time start. let first = Range { start: Anchor::Time(0), end: Some(Anchor::Time(5)), }; let open = Range { start: Anchor::Time(8), end: None, }; let mut reader = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([first, open]), ); producer.give_up(5..6, Hole::stale(budget)); let waiter = kio::Waiter::noop(); assert!( reader.poll_drop(&waiter).is_pending(), "the groups above it are not all read" ); let _group = producer.create_group(group::Info { start: 6 }).unwrap(); assert_eq!(reader.poll_drop(&waiter), Poll::Ready((5..6, Hole::stale(budget)))); } /// A queue-only poll, which judges no drift, still reports the received stale runs the /// last probing poll found settled, rather than forgetting them until the next probe. #[test] #[cfg(feature = "async")] fn a_queued_drop_poll_keeps_the_last_drift() { let producer = track_producer("test", Info::timed()); let budget = Duration::from_millis(500); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([Range::groups(0..)]), ); producer.give_up(1..2, Hole::stale(budget)); producer.give_up(3..4, Hole::stale(budget)); let waiter = kio::Waiter::noop(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..2, Hole::stale(budget)))); assert_eq!( subscriber.poll_queued_drop(&waiter), Poll::Ready((3..4, Hole::stale(budget))) ); assert!(subscriber.poll_queued_drop(&waiter).is_pending()); } /// A stale run that settled a reader is no verdict once the reader's budget widens past /// the one it was judged under: the wider reader asks a fill for it, as one that was /// always that wide does. #[test] #[cfg(feature = "async")] fn a_widened_budget_asks_a_fill_for_a_stale_run() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); producer.start_at(3).unwrap(); write_group(&producer, 3); let budget = Duration::from_millis(500); producer.give_up(0..3, Hole::stale(budget)); let narrow = Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([Range::groups(0..)]); let mut subscriber = producer.subscribe(narrow.clone()); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), [3]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "the stale run settles the reader" ); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(60)))) .unwrap(); assert!(subscriber.poll_next_group(&waiter).is_pending()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the wider budget asks for the stale run"); }; assert_eq!((fill.start().group, fill.end()), (0, 3)); } /// A fill's OLD answer under the publisher's max age binds every reader, so the track holds /// it as a stale verdict under that age rather than a loss for good: the readers waiting now /// are refused, a later reader no wider is settled without asking, and once the verdict is /// forgotten, as a new upstream subscription or the drop set's expiry window forgets one, a /// reader asks again. Mirrors JS. #[test] #[cfg(feature = "async")] #[expect(clippy::single_range_in_vec_init, reason = "the forgotten runs are a set, here one")] fn a_fills_old_answer_under_the_max_age_is_held_until_forgotten() { let max_age = Duration::from_secs(10); let mut producer = track_producer("test", Info::timed().with_max_age(max_age)); let dynamic = producer.requests(); producer.start_at(3).unwrap(); write_group(&producer, 3); // No subscription feeds the copy, so nothing below the start arrives but what a fill does. producer.set_unfed(true); let terms = || { Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(60))) .with_ranges([Range::groups(0..)]) }; let mut subscriber = producer.subscribe(terms()); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), [3]); // A reader of the run alone, which the verdict does not settle, waits on the same fill. let mut bounded = producer.subscribe(terms().with_ranges([Range::groups(0..3)])); assert!(drain_groups(&mut bounded).is_empty()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("a fill is asked below the start"); }; fill.reject_groups(0..3, Error::Old); drop(fill); assert_eq!(drained_drops(&mut subscriber), [(0, 3)]); assert_eq!( drained_drops(&mut bounded), [(0, 3)], "the readers waiting now are refused" ); assert!(dynamic.poll_requested_groups(&waiter).is_pending()); assert_eq!( producer.state.read().drops.get(&0), Some(&Hole::stale(max_age)), "held under the max age, which a widening reverses" ); let mut later = producer.subscribe(terms()); assert_eq!(drain_groups(&mut later), [3]); assert_eq!(drained_drops(&mut later), [(0, 3)]); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "the verdict is not asked again" ); assert_eq!(producer.supersede_stale(), RangeSet::from_iter([0..3])); let mut again = producer.subscribe(terms()); assert_eq!(drain_groups(&mut again), [3]); assert!( matches!(dynamic.poll_requested_groups(&waiter), Poll::Ready(Ok(_))), "a forgotten verdict is asked again" ); } /// A received stale run settles only a reader no wider than the budget it was judged under /// that reads every group above it: a bounded reader or a wider one asks a fill for the run /// instead, since the subscription that dropped it never delivers it. Group 3 is received, /// so the readers have a newest group to judge against. Mirrors JS. #[test] #[cfg(feature = "async")] fn a_received_stale_run_settles_only_a_reader_of_everything_above_it() { let producer = track_producer("test", Info::timed()); producer.received(); let dynamic = producer.requests(); let budget = Duration::from_millis(500); producer.give_up(0..2, Hole::stale(budget)); producer.create_group(3u64.into()).unwrap().finish().unwrap(); let reader = |max: Duration, range: Range| { producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(max)) .with_ranges([range]), ) }; let waiter = kio::Waiter::noop(); let mut open = reader(budget, Range::groups(0..)); assert_eq!(open.poll_drop(&waiter), Poll::Ready((0..2, Hole::stale(budget)))); assert_eq!(drain_groups(&mut open), [3]); assert!(dynamic.poll_requested_groups(&waiter).is_pending(), "nothing is asked"); let mut bounded = reader(budget, Range::groups(0..2)); assert!( bounded.poll_recv_group(&waiter).is_pending(), "it still wants the groups" ); let mut wider = reader(Duration::from_secs(5), Range::groups(0..)); assert_eq!(drain_groups(&mut wider), [3]); assert!(wider.poll_drop(&waiter).is_pending(), "it still wants the groups"); let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) { asked.push((fill.start().group, fill.end())); } assert_eq!(asked, [(0, 2)], "one fill, which both join"); } /// A fill taken before the last handler closed still answers an unsettled stale run: a /// reader that asks afterwards joins it while it runs, and reads its groups from the cache /// once it is done. #[test] #[cfg(feature = "async")] fn a_taken_fill_answers_an_unsettled_stale_run_after_its_handler_closes() { let producer = track_producer("test", Info::timed()); producer.received(); let dynamic = producer.requests(); let budget = Duration::from_millis(500); producer.give_up(0..2, Hole::stale(budget)); producer.give_up(4..6, Hole::stale(budget)); // Stops at 2, though the upstream judged 0..2 against what is above it: the run is // unknown to it. let reader = || { producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([Range::groups(0..2)]), ) }; let waiter = kio::Waiter::noop(); let mut first = reader(); assert!(first.poll_recv_group(&waiter).is_pending()); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the unsettled stale run is asked for"); }; assert_eq!((fill.start().group, fill.end()), (0, 2)); drop(dynamic); let mut joined = reader(); assert!(joined.poll_recv_group(&waiter).is_pending(), "it joins the taken fill"); for key in 0..2 { let mut group = fill.accept(key, None).unwrap().expect("a fresh group"); group .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); } drop(fill); let mut late = reader(); for subscriber in [&mut first, &mut joined, &mut late] { let mut got = drain_groups(subscriber); got.sort_unstable(); assert_eq!(got, [0, 1]); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); } } /// Creating a group past the next key says nothing about the gap: groups may be /// created out of order, so a late one in it still reaches readers, and nothing is skipped. #[test] #[cfg(feature = "async")] fn creating_a_group_past_a_gap_skips_nothing() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); let waiter = kio::Waiter::noop(); write_group(&producer, 2); write_group(&producer, 5); assert!(producer.state.read().drops.is_empty()); assert!(subscriber.poll_drop(&waiter).is_pending()); assert_eq!(drain_groups(&mut subscriber), [2, 5]); write_group(&producer, 4); assert_eq!(drain_groups(&mut subscriber), [4]); assert!(producer.state.read().drops.is_empty()); } /// Appending says the producer will not go back: every key below the new group it /// never produced since its first group is skipped, and readers are told at once. #[test] #[cfg(feature = "async")] fn appending_skips_the_holes_below_it() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); let waiter = kio::Waiter::noop(); write_group(&producer, 2); write_group(&producer, 5); write_group(&producer, 7); producer.append_next().unwrap().finish().unwrap(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((3..5, Hole::Skip))); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((6..7, Hole::Skip))); assert_eq!(drain_groups(&mut subscriber), [2, 5, 7, 8]); // The next append only names what was left since this one. write_group(&producer, 10); producer.append_next().unwrap().finish().unwrap(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((9..10, Hole::Skip))); let skipped: Vec<_> = producer .state .read() .drops .iter() .map(|(run, hole)| (run.clone(), hole.clone())) .collect(); assert_eq!(skipped, [(3..5, Hole::Skip), (6..7, Hole::Skip), (9..10, Hole::Skip)]); } /// An append's skip replaces a drop below it, whatever its error: the key was never /// produced, which is the most authoritative verdict. #[test] #[cfg(feature = "async")] fn appending_replaces_a_drop_with_a_skip() { let producer = track_producer("test", Info::timed()); write_group(&producer, 0); write_group(&producer, 4); producer.give_up(1..2, Hole::drop(StreamError::Cancel)); producer.append_next().unwrap().finish().unwrap(); let drops: Vec<_> = producer .state .read() .drops .iter() .map(|(run, hole)| (run.clone(), hole.clone())) .collect(); assert_eq!(drops, [(1..4, Hole::Skip)]); } /// A skip is final: creating or filling a group at a key an append or `skip_groups` /// skipped is refused, while a dropped one may be created again. #[test] #[cfg(feature = "async")] fn creating_a_skipped_key_is_refused() { let producer = track_producer("test", Info::timed()); write_group(&producer, 0); write_group(&producer, 3); producer.append_next().unwrap().finish().unwrap(); assert!(matches!( producer.create_group(group::Info { start: 1 }), Err(Error::Skipped) )); producer.skip_groups(5..7); assert!(matches!( producer.create_group(group::Info { start: 6 }), Err(Error::Skipped) )); // The next key, 5, was skipped too. assert!(matches!(producer.append_next(), Err(Error::Skipped))); // A later drop leaves a skipped key skipped. producer.give_up(1..2, Hole::drop(StreamError::Cancel)); assert!(matches!( producer.create_group(group::Info { start: 1 }), Err(Error::Skipped) )); // No fill holds a skipped key either. assert!(matches!( producer .state .write() .ok() .unwrap() .insert_group_request(6, group::Bounds::default(), None), Err(Error::Skipped) )); // A dropped key may be created again. producer.give_up(9..10, Hole::drop(StreamError::Cancel)); write_group(&producer, 9); } /// A run given up longer than the pool's expiry window ago is forgotten by the sweep, even /// on a track that caches nothing, so a pool with no byte target still bounds the drop set; /// a run given up since stays. #[test] fn the_sweep_forgets_runs_past_the_expiry_window() { let producer = track_producer("test", Info::timed()); producer.received(); let pool = producer.broadcast.pool.clone(); // A local track keeps what it skipped above its newest group, which its end relies on. let local = track_producer_pooled("local", pool.clone()); local.skip_groups(5..10); let used = pool.used(); producer.drop_groups(0..1, StreamError::Old); producer.drop_groups(2..3, StreamError::Old); elapse(&producer, cache::DEFAULT_EXPIRY / 2); producer.drop_groups(4..5, StreamError::Old); assert_eq!( pool.used(), used + 3 * (RUN_OVERHEAD + BORN_OVERHEAD), "each run and its expiry are billed" ); elapse(&producer, cache::DEFAULT_EXPIRY); pool.sweep(); let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.start).collect(); assert_eq!(held, [4], "only the run given up within the window stays"); assert_eq!(pool.used(), used + RUN_OVERHEAD + BORN_OVERHEAD); assert!( producer.state.read().given_up(0).is_none(), "a forgotten run is unknown again" ); elapse(&producer, cache::DEFAULT_EXPIRY); pool.sweep(); assert!(producer.state.read().drops.is_empty()); assert_eq!(pool.used(), used); assert!(local.state.read().skipped(5), "a local skip above the edge stays"); } /// A received copy whose end a dropped gap let it reach stays ended once that run expires, /// even for a reader whose budget outlasts the run, which would otherwise wait on the gap. #[test] #[cfg(feature = "async")] fn a_reached_end_outlives_its_expired_drops() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let mut producer = track_producer_pooled("test", pool.clone()); producer.received(); let mut subscriber = producer.subscribe(patient()); write_group(&producer, 0); producer.drop_groups(1..2, StreamError::Old); write_group(&producer, 2); producer.finish_at(Position::group(3)).unwrap(); drain_groups(&mut subscriber); let waiter = kio::Waiter::noop(); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); elapse(&producer, Duration::from_secs(40)); pool.sweep(); assert!(producer.state.read().drops.is_empty(), "the run expired"); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); let mut late = producer.subscribe(patient()); drain_groups(&mut late); assert!(matches!(late.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); } /// A received copy that reached its end through a dropped gap, after that run expires. #[cfg(feature = "async")] fn reached_then_expired() -> (Producer, Subscriber, cache::Pool) { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let mut producer = track_producer_pooled("test", pool.clone()); producer.received(); let mut subscriber = producer.subscribe(patient()); write_group(&producer, 0); producer.drop_groups(1..2, StreamError::Old); write_group(&producer, 2); producer.finish_at(Position::group(3)).unwrap(); drain_groups(&mut subscriber); let waiter = kio::Waiter::noop(); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None)))); elapse(&producer, Duration::from_secs(40)); pool.sweep(); assert!(producer.state.read().drops.is_empty(), "the run expired"); (producer, subscriber, pool) } /// A subscription whose budget outlasts any run, so it would wait on a forgotten one. #[cfg(feature = "async")] fn patient() -> Subscription { Subscription::default() .with_range(0..) .with_delay(Delay::default().with_max(Duration::from_secs(3600))) } /// A reached end refined mid-group by a peer that named the boundary first stays reached /// after its dropped gap is forgotten, since the forgotten run stays accounted for. #[test] #[cfg(feature = "async")] fn a_refined_end_stays_reached_after_its_drops_expire() { let (mut producer, _subscriber, _pool) = reached_then_expired(); producer.finish_at(Position { group: 2, frame: 1 }).unwrap(); let mut late = producer.subscribe(patient()); drain_groups(&mut late); assert!(matches!( late.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Ok(None)) )); } /// A reached end settles an abort after its dropped gap is forgotten: readers end cleanly. #[test] #[cfg(feature = "async")] fn an_abort_after_a_reached_end_drops_expire_ends_readers_cleanly() { let (producer, mut subscriber, _pool) = reached_then_expired(); producer.abort(Error::Cancel).unwrap(); assert!(matches!( subscriber.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Ok(None)) )); } /// A reader that judged a dropped run before the pool forgot it, and has not read past it, /// asks a fill for it once it is unknown again. #[test] #[cfg(feature = "async")] fn a_reader_asks_a_fill_for_a_forgotten_run_it_judged() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let producer = track_producer_pooled("test", pool.clone()); producer.received(); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(patient()); write_group(&producer, 0); producer.drop_groups(1..2, StreamError::Old); write_group(&producer, 2); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { panic!("group 0 is handed out"); }; assert_eq!(first.start, 0); assert!( dynamic.poll_requested_groups(&waiter).is_pending(), "a dropped run asks no fill" ); elapse(&producer, Duration::from_secs(40)); pool.sweep(); assert!(producer.state.read().drops.is_empty(), "the run expired"); let _ = subscriber.poll_recv_group(&waiter); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&waiter) else { panic!("the forgotten run is asked of a fill"); }; assert_eq!(fill.start().group..fill.end(), 1..2); } /// A source refusing a fill in disjoint groups pays for the runs it records, as a drop does. #[test] #[cfg(feature = "async")] fn a_fragmented_fill_refusal_stays_within_the_pool() { const RUNS: u64 = 1000; const START: u64 = 2 * RUNS; let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); let (mut producer, pool) = pooled_producer(capacity); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..)); producer.start_at(START).unwrap(); write_group(&producer, START); assert_eq!(drain_groups(&mut subscriber), [START]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_groups(&kio::Waiter::noop()) else { panic!("the miss below the start is asked"); }; // Nothing arrives once the feed ends, so a refusal is a verdict on the track. producer.finish_shared().unwrap(); for run in 0..RUNS { fill.drop_groups(2 * run..2 * run + 1); } let held = producer.state.read().drops.len() as u64; assert!(held > 0, "the refusals are recorded"); assert!(held < RUNS, "the lowest runs were shed"); assert!( pool.used() <= capacity + 2 * (RUN_OVERHEAD + BORN_OVERHEAD) + 1024, "{}", pool.used() ); } /// A flood a reader reported, fragmented into runs alternating with upstream skips, /// bills its record too. Once the pool shrinks under it and its track has shed what it /// can, the reader folds the runs, stale or not, and the groups it read into its floor and /// stays within the pool: every report is final. #[test] fn a_reported_flood_folds_or_ends_the_reader() { const RUNS: u64 = 1000; let budget = Duration::from_secs(1); for stale in [false, true] { let (producer, pool) = pooled_producer(1 << 30); producer.received(); let waiter = kio::Waiter::noop(); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(budget)) .with_range(0..), ); let hole = match stale { true => Hole::stale(budget), false => Hole::drop(StreamError::Cancel), }; let mut reported = RangeSet::new(); let flood = |run: u64| { producer.give_up(2 * run..2 * run + 1, hole.clone()); producer.give_up(2 * run + 1..2 * run + 2, Hole::Skip); }; for run in 0..RUNS { flood(run); } while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { assert!(!reported.overlaps(&run), "stale={stale}: {run:?} reported twice"); reported.insert(run); } assert_eq!( reported.iter().map(|run| run.end - run.start).sum::(), 2 * RUNS, "stale={stale}" ); assert!(subscriber.poll_recv_group(&waiter).is_pending(), "stale={stale}"); let used = pool.used(); assert!( used >= 2 * RUNS * super::super::record::RUN_OVERHEAD, "stale={stale}: the record is billed" ); // The pool shrinks, and groups the reader reads above the flood accrue its debt: the // track sheds its own drop set first, then the reader pays. let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); pool.resize(capacity); let mut read = None; for key in 2 * RUNS..2 * RUNS + 64 { let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; 16 * 1024]) .unwrap(); group.finish().unwrap(); match subscriber.poll_recv_group(&waiter) { Poll::Ready(Ok(Some(group))) => assert_eq!(group.start, key, "stale={stale}"), res => { read = Some(res.map(|res| res.map(|_| ()))); break; } } } assert!(read.is_none(), "stale={stale}: the reader runs on"); while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { assert!( !reported.overlaps(&run), "stale={stale}: {run:?}, folded, was reported again" ); } assert!( pool.used() <= capacity + 2 * (16 * 1024 + cache::ENTRY_OVERHEAD), "stale={stale}: the pool holds {} past {capacity}", pool.used() ); } } /// A drop-only flood of disjoint runs bills the pool and sheds its lowest runs to stay /// within it, with no cached group to evict; a shed run is unknown again, so a wire cursor /// reports only the runs the track still holds. #[test] fn a_drop_flood_stays_within_the_pool() { const RUNS: u64 = 1000; let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); let (producer, pool) = pooled_producer(capacity); // A peer's drops, which a received copy holds. producer.received(); let waiter = kio::Waiter::noop(); let mut subscriber = producer.subscribe(replay().with_range(0..)); let mut reported = Vec::new(); for run in 0..RUNS { producer.drop_groups(2 * run..2 * run + 1, StreamError::Old); assert!( pool.used() <= capacity + 2 * (RUN_OVERHEAD + BORN_OVERHEAD), "run {run} took the pool to {}", pool.used() ); // Midway, the cursor takes in every run held then, and reports the lowest; the // rest are shed before it reports them. if run == RUNS / 2 { let Poll::Ready((first, _)) = subscriber.poll_drop(&waiter) else { panic!("a run is owed"); }; reported.push(first); } } let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); // Each disjoint run is an entry in the drop set and one in its expiry index. assert_eq!(pool.used(), held.len() as u64 * (RUN_OVERHEAD + BORN_OVERHEAD)); let lowest = held.first().expect("the newest runs stay").start; assert!(lowest > 0, "the lowest runs were shed first"); assert!( held.windows(2).all(|pair| pair[1].start == pair[0].start + 2), "only a prefix was shed" ); assert_eq!(held.last().map(|run| run.start), Some(2 * (RUNS - 1))); assert!( !producer.state.read().arrivals.has_gaps(), "the shed runs are accounted for without fragmenting the arrivals" ); assert!( producer.state.read().given_up(0).is_none(), "a shed run is unknown again" ); while let Poll::Ready((run, _)) = subscriber.poll_drop(&waiter) { reported.push(run); } // Billing the reader's record shed the track's lowest runs before it reported them. let held: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(reported[1..], held[..]); } /// A peer fragmenting its arrivals under a grace that never folds a gap bills each run to /// the pool, which folds the oldest gaps early to stay within it. #[test] fn an_arrival_flood_stays_within_the_pool() { const GROUPS: u64 = 1000; let capacity = 64 * ledger::RUN_OVERHEAD; let (producer, pool) = pooled_producer(capacity); producer.received(); // A reader that waits an hour for a late group, so no gap folds by its grace. let _subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(3600)))); for group in 0..GROUPS { write_group(&producer, 2 * group); } let state = producer.state.read(); let runs = state.arrivals.len() as u64; assert!(runs > 1, "the youngest gaps are still waited for"); assert!(runs < GROUPS, "the oldest gaps were folded"); assert_eq!(state.arrivals_bill.size(), (runs - 1) * ledger::RUN_OVERHEAD); assert!( pool.used() <= capacity + 2 * cache::ENTRY_OVERHEAD, "the arrivals took the pool to {}", pool.used() ); assert!(state.arrivals.covers(0..3), "a folded gap is no longer waited for"); } /// A local skip kept above the track's edge is dated once the edge passes it, so the next /// expiry scan past its window forgets it with no other drop to rearm it. #[test] fn a_kept_local_skip_expires_once_the_edge_passes_it() { let producer = track_producer("test", Info::timed()); let pool = producer.broadcast.pool.clone(); producer.skip_groups(0..10); elapse(&producer, cache::DEFAULT_EXPIRY * 2); pool.sweep(); assert!(producer.state.read().skipped(0), "a skip above the edge stays"); // The write's own expiry scan finds the skip, with nothing else to rearm it. let _group = producer.create_group(group::Info { start: 10 }).unwrap(); let state = producer.state.read(); assert!(!state.skipped(0), "the edge passed the skip"); assert_eq!(state.drops_bill.size(), 0, "its bill went with it"); } /// Skips are kept and reported as runs: a huge `skip_groups` or append gap costs one /// entry and one report, whatever its length. #[test] #[cfg(feature = "async")] fn a_huge_skip_costs_one_run() { const FAR: u64 = 1 << 61; let waiter = kio::Waiter::noop(); // Presented just after group 0, since its key in milliseconds overflows. let write_far = |producer: &Producer| { let mut group = producer.create_group(group::Info { start: FAR }).unwrap(); group .write_frame(Timestamp::from_millis(1).unwrap(), &b"x"[..]) .unwrap(); group.finish().unwrap(); }; let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 0); producer.skip_groups(1..FAR); assert_eq!(producer.state.read().drops.iter().count(), 1); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..FAR, Hole::Skip))); write_far(&producer); assert_eq!(drain_groups(&mut subscriber), [0, FAR]); assert!(subscriber.poll_drop(&waiter).is_pending()); let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 0); write_far(&producer); producer.append_next().unwrap().finish().unwrap(); assert_eq!(producer.state.read().drops.iter().count(), 1); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..FAR, Hole::Skip))); assert_eq!(drain_groups(&mut subscriber), [0, FAR, FAR + 1]); } /// A dropped key may be created again, as a group stream wins over its drop, such as /// one whose group was aborted: only a skip is final. #[test] #[cfg(feature = "async")] fn an_aborted_key_is_created_again() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); let group = producer.create_group(group::Info { start: 0 }).unwrap(); group.abort(Error::Stream(StreamError::Cancel)).unwrap(); write_group(&producer, 0); producer.append_next().unwrap().finish().unwrap(); assert!(producer.state.read().drops.is_empty()); let mut read = drain_groups(&mut subscriber); read.dedup(); assert_eq!(read.last(), Some(&1)); } /// `skip_groups` leaves a cached group alone, and replaces every drop, final or stale: the /// skip is final, so a new upstream subscription never supersedes it. #[test] #[cfg(feature = "async")] fn skipping_replaces_every_drop_but_leaves_a_cached_group() { let producer = track_producer("test", Info::timed()); write_group(&producer, 1); producer.give_up(3..4, Hole::drop(StreamError::Cancel)); producer.give_up(4..5, Hole::stale(Duration::from_secs(1))); producer.skip_groups(0..5); producer.supersede_stale(); let drops: Vec<_> = producer .state .read() .drops .iter() .map(|(run, hole)| (run.clone(), hole.clone())) .collect(); assert_eq!(drops, [(0..1, Hole::Skip), (2..5, Hole::Skip)]); } /// A skip after a drop already reported goes out again, replacing it, while a reader that /// moved past the drop is not handed anything again; one already reported as a skip is /// not reported twice, and a later drop leaves it skipped. #[test] #[cfg(feature = "async")] fn a_skip_is_reported_over_a_reported_drop() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); let waiter = kio::Waiter::noop(); write_group(&producer, 0); producer.give_up(1..3, Hole::drop(StreamError::Cancel)); producer.give_up(4..5, Hole::drop(StreamError::NotFound)); write_group(&producer, 5); assert_eq!( subscriber.poll_drop(&waiter), Poll::Ready((1..3, Hole::drop(StreamError::Cancel))) ); assert_eq!( subscriber.poll_drop(&waiter), Poll::Ready((4..5, Hole::drop(StreamError::NotFound))) ); assert!(subscriber.poll_drop(&waiter).is_pending()); assert_eq!(drain_groups(&mut subscriber), [0, 5]); // Group 0 arrived, so the skip leaves it; both drops and the gap between become one skip. producer.skip_groups(0..5); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..5, Hole::Skip))); assert!(subscriber.poll_drop(&waiter).is_pending()); assert!(drain_groups(&mut subscriber).is_empty(), "nothing is delivered again"); producer.give_up(2..3, Hole::drop(StreamError::Cancel)); assert!(subscriber.poll_drop(&waiter).is_pending(), "the skip stands"); assert!(producer.state.read().skipped(2)); } /// Appending and a first group leave no gap, so they skip nothing; nor is anything below /// a track's first group skipped. #[test] #[cfg(feature = "async")] fn appending_and_a_first_group_skip_nothing() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 3); producer.append_next().unwrap().finish().unwrap(); assert!(producer.state.read().drops.is_empty()); assert!(drained_drops(&mut subscriber).is_empty()); } /// A received copy's groups land in any order, so a gap above its newest is still owed. #[test] #[cfg(feature = "async")] fn a_received_gap_is_not_skipped() { let producer = track_producer("test", Info::timed()); producer.received(); let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 2); write_group(&producer, 5); assert!(producer.state.read().drops.is_empty()); assert!(drained_drops(&mut subscriber).is_empty()); write_group(&producer, 3); assert_eq!(drain_groups(&mut subscriber), [2, 5, 3]); } /// A received stale run a new upstream subscription supersedes is unknown again, and a group /// that subscription delivers wins over its drop. #[test] #[cfg(feature = "async")] fn superseded_stale_runs_are_unknown_again() { let producer = track_producer("test", Info::timed()); let budget = Duration::from_millis(500); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([Range::groups(0..)]), ); let waiter = kio::Waiter::noop(); producer.give_up(1..2, Hole::stale(budget)); producer.supersede_stale(); assert!( subscriber.poll_drop(&waiter).is_pending(), "no verdict under the new subscription" ); assert!(producer.state.read().drops.is_empty()); write_group(&producer, 1); assert_eq!(drain_groups(&mut subscriber), [1]); } /// A superseded stale run leaves nothing a cursor still holds to report, even the part /// of it queued behind an earlier report. #[test] #[cfg(feature = "async")] fn superseding_withdraws_a_partly_reported_stale_run() { let producer = track_producer("test", Info::timed()); let budget = Duration::from_millis(500); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_ranges([Range::groups(0..)]), ); let waiter = kio::Waiter::noop(); producer.drop_groups(4..5, StreamError::NotFound); assert_eq!( subscriber.poll_drop(&waiter), Poll::Ready((4..5, Hole::drop(StreamError::NotFound))) ); producer.give_up(3..7, Hole::stale(budget)); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((3..4, Hole::stale(budget)))); producer.supersede_stale(); assert!(subscriber.poll_drop(&waiter).is_pending(), "5 and 6 are unknown again"); } /// A report describes a miss, not a delivery: a late group is still handed out once. #[test] #[cfg(feature = "async")] fn a_reported_drop_still_allows_a_late_group() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); let waiter = kio::Waiter::noop(); producer.drop_groups(2..4, StreamError::NotFound); assert_eq!( subscriber.poll_drop(&waiter), Poll::Ready((2..4, Hole::drop(StreamError::NotFound))) ); assert!(subscriber.poll_drop(&waiter).is_pending()); write_group(&producer, 3); assert_eq!(drain_groups(&mut subscriber), [3]); assert!(subscriber.poll_drop(&waiter).is_pending()); assert!(drain_groups(&mut subscriber).is_empty()); } /// A drop ends a group still arriving with the drop's error at once, for the reader /// already holding it, before any stream reset or track end. A group that finished here /// stays whole. #[test] #[cfg(feature = "async")] fn a_drop_ends_a_held_group_with_its_error() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); let mut open = producer.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(open.key_time(), &b"a"[..]).unwrap(); let mut finished = producer.create_group(group::Info { start: 1 }).unwrap(); finished.write_frame(finished.key_time(), &b"b"[..]).unwrap(); finished.finish().unwrap(); let mut held = subscriber.assert_group(); assert_eq!( &held.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[..], b"a" ); assert!( held.read_frame().now_or_never().is_none(), "the group is still arriving" ); producer.drop_groups(0..2, StreamError::App(5)); assert!(matches!( held.read_frame().now_or_never(), Some(Err(Error::Stream(StreamError::App(5)))) )); let mut whole = subscriber.assert_group(); assert_eq!(whole.start, 1); assert_eq!( &whole.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[..], b"b" ); assert!(matches!(whole.read_frame().now_or_never(), Some(Ok(None)))); } /// A drop judges the subscription, not a fetch: a fetched head still arriving keeps /// serving the group to its end after the drop ends the live copy. #[kio::test] #[cfg(feature = "async")] async fn a_drop_spares_a_fetched_head_still_arriving() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); live.write_frame(live.key_time(), &b"delta"[..]).unwrap(); let _fetch = consumer.fetch_group(0, None); let fill = dynamic.requested_groups().await.unwrap(); let mut head = fill.accept(0, Info::timed()).unwrap().expect("a fresh group"); head.write_frame(head.key_time(), &b"snapshot"[..]).unwrap(); producer.drop_groups(0..1, StreamError::App(5)); head.write_frame(head.key_time(), &b"delta"[..]).unwrap(); head.finish().unwrap(); let mut group = consumer.peek_group(0).expect("the head still serves group 0"); for payload in [&b"snapshot"[..], b"delta"] { assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], payload); } assert!(matches!(group.read_frame().await, Ok(None))); } /// A group the publisher cuts before a reader takes it is still handed out, so the /// reader sees why it ended rather than a group never sent: in arrival and in key /// order, and on a track that then finishes, before the clean end. #[test] #[cfg(feature = "async")] fn a_cut_group_reaches_its_readers() { let producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); let mut cut = producer.create_group(group::Info { start: 0 }).unwrap(); cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); let _producer = producer.finish().unwrap(); let mut first = arrival.assert_group(); assert_eq!(first.start, 0); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); assert_eq!(arrival.assert_group().start, 1); assert!(matches!(arrival.recv_group().now_or_never(), Some(Ok(None)))); let mut first = ordered .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap(); assert_eq!(first.start, 0); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); assert_eq!(drain_ordered(&mut ordered), [1]); assert!(matches!(ordered.next_group().now_or_never(), Some(Ok(None)))); } /// Cache policy and a cancel are deliberate holes, not cuts: an expired, evicted, or /// cancelled group, and a reset carrying such a code, stay out of the readers' way. #[test] #[cfg(feature = "async")] fn a_policy_abort_is_a_hole() { for err in [ Error::Old, Error::Evicted, Error::Timeout, Error::Cancel, Error::Stream(StreamError::Old), Error::Stream(StreamError::DeliveryTimeout), Error::Stream(StreamError::Cancel), ] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(group.key_time(), &b"a"[..]).unwrap(); group.abort(err.clone()).unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); assert_eq!(subscriber.assert_group().start, 1, "{err:?}"); } } /// A cut stays readable for as long as the group it was: the sweep that reclaims /// spent slots leaves it to a reader still behind, and once it expires like a group, /// a relay still forwards it as a drop until that drop outlives the window too. #[test] #[cfg(feature = "async")] fn a_cut_outlives_the_sweep_until_it_expires() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let producer = track_producer_pooled("test", pool.clone()); let mut cut = producer.create_group(group::Info { start: 0 }).unwrap(); cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); producer .create_group(group::Info { start: 1 }) .unwrap() .finish() .unwrap(); let _live = producer.create_group(group::Info { start: 2 }).unwrap(); elapse(&producer, Duration::from_secs(1)); pool.sweep(); let mut late = producer.subscribe(replay()); let mut first = late.assert_group(); assert_eq!(first.start, 0, "the sweep kept the cut"); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); elapse(&producer, Duration::from_secs(20)); pool.sweep(); assert!(!producer.state.read().lookup.contains_key(&0), "the cut expired"); let mut relay = producer.subscribe(replay()); let waiter = kio::Waiter::noop(); assert!(matches!( relay.poll_drop(&waiter), Poll::Ready((range, Hole::Drop { error: StreamError::App(5), .. })) if range == (0..1) )); // The drop it left ages out a window later like any run, unknown again. elapse(&producer, Duration::from_secs(20)); pool.sweep(); assert!(producer.state.read().given_up(0).is_none()); } /// A cut above a reader's cap is handed out once the cap rises over it. #[test] #[cfg(feature = "async")] fn a_raised_cap_reoffers_a_cut() { let producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(replay()).with_range(..1); let mut ordered = producer.subscribe(replay()).ordered().with_range(..1); producer .create_group(group::Info { start: 0 }) .unwrap() .finish() .unwrap(); let mut cut = producer.create_group(group::Info { start: 1 }).unwrap(); cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().is_none(), "capped"); arrival = arrival.with_range(..); let mut reoffered = arrival.assert_group(); assert_eq!(reoffered.start, 1); assert!(matches!( reoffered.read_frame().now_or_never(), Some(Err(Error::App(5))) )); assert_eq!(drain_ordered(&mut ordered), [0]); ordered = ordered.with_range(..); let mut reoffered = ordered .next_group() .now_or_never() .unwrap() .unwrap() .unwrap() .group() .unwrap(); assert_eq!(reoffered.start, 1); assert!(matches!( reoffered.read_frame().now_or_never(), Some(Err(Error::App(5))) )); } /// A time start is the group containing it, the newest starting at or before it, and a /// time end includes the group containing it. #[test] #[cfg(feature = "async")] fn time_anchors_resolve_against_the_keys() { let producer = track_producer("test", Info::timed()); for key in [0, 1000, 2000, 3000] { write_group(&producer, key); } let time = |start, end| { replay().with_ranges([Range { start: Anchor::Time(start), end: Some(Anchor::Time(end)), }]) }; let mut seek = producer.subscribe(time(1500, 2000)); assert_eq!(drain(&mut seek), vec![1000, 2000]); let mut early = producer.subscribe(time(0, 500)); assert_eq!(drain(&mut early), vec![0]); // Past the newest start: open for the groups that start inside the end. let mut live = producer.subscribe(time(3000, 4500)); assert_eq!(drain(&mut live), vec![3000]); for key in [4000, 5000, 6000] { write_group(&producer, key); } assert_eq!(drain(&mut live), vec![4000]); } /// A group joined mid-group still starts at its key, whatever its first frame held says: /// a time start inside it names it. The range is open, since a bounded read takes whole /// groups only. #[test] #[cfg(feature = "async")] fn a_partial_group_starts_at_its_key() { let producer = track_producer("test", Info::timed()); write_group(&producer, 0); write_group(&producer, 1000); let mut partial = producer.create_group(group::Info { start: 2000 }).unwrap(); partial.start_at(1).unwrap(); partial .write_frame(Timestamp::from_millis(2500).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); partial.finish().unwrap(); let mut seek = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(2200), end: None, }])); assert_eq!(drain(&mut seek), vec![2000]); } /// A time start resolves on keys alone: groups opened before their first frames are read /// from the one containing it. #[test] #[cfg(feature = "async")] fn a_time_start_needs_no_frame() { for order in [Order::Ascending, Order::Descending] { let producer = track_producer("test", Info::timed()); let _groups: Vec<_> = (2..10) .map(|second| producer.create_group(group::Info { start: second * 1000 }).unwrap()) .collect(); let since = [Range { start: Anchor::Time(2500), end: None, }]; let mut subscriber = producer.subscribe(replay().with_ranges(since).with_order(order)); let mut read = drain(&mut subscriber); read.sort_unstable(); assert_eq!( read, (2..10).map(|second| second * 1000).collect::>(), "{order:?}" ); } } /// A time end settles on keys alone: a group opened before its first frame is inside the /// end or past it by its key. #[test] #[cfg(feature = "async")] fn a_time_end_needs_no_frame() { let producer = track_producer("test", Info::timed()); let _groups: Vec<_> = [0, 400, 600] .into_iter() .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let mut subscriber = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(0), end: Some(Anchor::Time(500)), }])); assert_eq!(drain(&mut subscriber), vec![0, 400]); assert_eq!( subscriber.resolved_ranges([0]), vec![Some(Resolved { start: 0, end: Some(501), floored: true })] ); } /// A scale-less track's keys are bare units, so a time anchor reads them as they are. #[test] #[cfg(feature = "async")] fn an_untimed_track_reads_time_anchors_as_keys() { let producer = untimed_producer(); for key in [0, 5, 10, 15] { producer .append_group(Timestamp::unscaled(key)) .unwrap() .finish() .unwrap(); } let mut subscriber = producer.subscribe(replay().with_ranges([Range { start: Anchor::Time(7), end: Some(Anchor::Time(10)), }])); assert_eq!(drain(&mut subscriber), vec![5, 10]); } /// A widened budget re-offers the cached groups the cursor skipped as stale, newest /// first by default, and never a group it already handed out. #[test] #[cfg(feature = "async")] fn a_widened_budget_reoffers_what_it_skipped_as_stale() { let mut producer = track_producer("test", Info::timed()); let narrow = Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500))); let mut subscriber = producer.subscribe(narrow.clone()); append_at(&mut producer, 0); assert_eq!(drain(&mut subscriber), vec![0]); for millis in [1000, 2000, 3000, 6000] { append_at(&mut producer, millis); } // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. assert_eq!(drain(&mut subscriber), vec![3, 4]); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(10)))) .unwrap(); assert_eq!( drain(&mut subscriber), vec![2, 1], "newest first, never 0, 3, or 4 again" ); assert_eq!(drain(&mut subscriber), Vec::::new()); } /// An ascending subscription gets the re-offer lowest first. #[test] #[cfg(feature = "async")] fn an_ascending_widening_reoffers_lowest_first() { let mut producer = track_producer("test", Info::timed()); for millis in [0, 1000, 2000, 5000] { append_at(&mut producer, millis); } let narrow = Subscription::default() .with_delay(Delay::default().with_max(Duration::from_millis(1500))) .with_order(Order::Ascending); let mut subscriber = producer.subscribe(narrow.clone()); assert_eq!(drain(&mut subscriber), vec![2, 3]); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(10)))) .unwrap(); assert_eq!(drain(&mut subscriber), vec![0, 1]); } /// A widening that covers only the newer of two skipped groups hands it out without /// folding the older one into the delivered floor: a further widening still owes it. #[test] #[cfg(feature = "async")] fn a_partial_widening_keeps_the_older_skip_owed() { let mut producer = track_producer("test", Info::timed()); let narrow = Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500))); let mut subscriber = producer.subscribe(narrow.clone()); append_at(&mut producer, 0); assert_eq!(drain(&mut subscriber), vec![0]); for millis in [1000, 2000, 4000] { append_at(&mut producer, millis); } assert_eq!(drain(&mut subscriber), vec![2, 3]); for millis in [5000, 6000, 9000] { append_at(&mut producer, millis); } assert_eq!(drain(&mut subscriber), vec![5, 6]); // Group 4 reaches 6s, 3s behind the 9s edge; group 1 reaches 2s, 7s behind. subscriber .update( narrow .clone() .with_delay(Delay::default().with_max(Duration::from_secs(4))), ) .unwrap(); assert_eq!(drain(&mut subscriber), vec![4]); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(10)))) .unwrap(); assert_eq!(drain(&mut subscriber), vec![1]); } /// Handing out the newest group a widened range owes never folds the older ones it still /// owes into the delivered floor before they are judged: a later budget widening gets the /// ones judged stale. #[test] #[cfg(feature = "async")] fn a_newest_first_range_reoffer_keeps_the_older_owed_groups() { let mut producer = track_producer("test", Info::timed()); for millis in [0, 1000, 2000, 3000, 6000] { append_at(&mut producer, millis); } let narrow = Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500))); let mut subscriber = producer.subscribe(narrow.clone()); subscriber.set_range(3..5); assert_eq!(drain(&mut subscriber), vec![3, 4]); for millis in [20_000, 40_000] { append_at(&mut producer, millis); } subscriber.set_range(0..); // Groups 0 through 2 reach 3s at most, far behind the 40s edge. assert_eq!(drain(&mut subscriber), vec![6, 5]); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(60)))) .unwrap(); assert_eq!(drain(&mut subscriber), vec![2, 1, 0]); } /// How many runs of skipped groups `subscriber` holds. #[cfg(feature = "async")] fn skipped_runs(subscriber: &Subscriber) -> usize { let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() }; cursor.record.skipped.runs() } /// The groups handed out around a skipped one aging past the budget does not fold it in /// with them: a widening still owes it. #[test] #[cfg(feature = "async")] fn a_skipped_group_outlives_the_fold_of_its_neighbours() { let mut producer = track_producer("test", Info::timed()); let narrow = Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500))); let mut subscriber = producer.subscribe(narrow.clone()); append_at(&mut producer, 0); assert_eq!(drain(&mut subscriber), vec![0]); for millis in [1000, 2000, 3000, 6000] { append_at(&mut producer, millis); } assert_eq!(drain(&mut subscriber), vec![3, 4]); // Groups 3 and 4 are past the budget now, which would write off the gap below them. for millis in [20_000, 40_000] { append_at(&mut producer, millis); assert_eq!(drain(&mut subscriber).len(), 1); } assert_eq!( subscriber.delivered_runs(), 2, "folded up to the skipped groups, no further" ); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(60)))) .unwrap(); assert_eq!(drain(&mut subscriber), vec![2, 1]); } /// A group skipped at the track's max age bound is final: no widening passes the bound, so /// the cursor keeps no record of it and its gap folds as the groups above it age out. #[test] #[cfg(feature = "async")] fn a_skip_at_the_max_age_bound_folds_into_the_floor() { let mut producer = track_producer("test", Info::timed().with_max_age(Duration::from_millis(500))); for millis in [0, 1000, 2000] { append_at(&mut producer, millis); } let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); assert_eq!(drain(&mut subscriber), vec![1, 2]); assert_eq!(skipped_runs(&subscriber), 0); append_at(&mut producer, 3000); assert_eq!(drain(&mut subscriber), vec![3]); assert_eq!(subscriber.delivered_runs(), 1, "the gap at group 0 folded"); } /// A skipped group gone from the cache can never be offered again, so the cursor forgets /// it and a widening finds nothing. #[test] #[cfg(feature = "async")] fn an_evicted_skip_is_forgotten() { let mut producer = track_producer("test", Info::timed()); let narrow = Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(1500))); let mut subscriber = producer.subscribe(narrow.clone()); append_at(&mut producer, 0); assert_eq!(drain(&mut subscriber), vec![0]); for millis in [1000, 2000, 3000, 6000] { append_at(&mut producer, millis); } assert_eq!(drain(&mut subscriber), vec![3, 4]); assert_eq!(skipped_runs(&subscriber), 1); evict(&producer, 1); evict(&producer, 2); append_at(&mut producer, 7000); assert_eq!(drain(&mut subscriber), vec![5]); assert_eq!(skipped_runs(&subscriber), 0); subscriber .update(narrow.with_delay(Delay::default().with_max(Duration::from_secs(60)))) .unwrap(); assert_eq!(drain(&mut subscriber), Vec::::new()); } /// Distinct transport starts retain delivery outcomes, but share the same indexed /// bytes even while every earlier delivery is held by an unread consumer. #[kio::test] #[cfg(feature = "async")] async fn distinct_tail_readers_share_canonical_content() { for size in [0, 1, 1024] { for bounded in [false, true] { let producer = track_producer("test", Info::timed()); let pool = producer.broadcast.pool.clone(); let dynamic = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(32).unwrap(); live.write_frame(live.key_time(), vec![32; size]).unwrap(); assert!( live.handle().content().is_none(), "a single copy keeps its fast representation" ); let mut held = Vec::new(); let mut deliveries = vec![live.handle()]; let end = if bounded { 17 } else { 33 }; for start in (1..=8).rev() { let pending = consumer.fetch_range( 0, FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end: bounded.then_some(end), }, ); let fill = dynamic.requested_fill_group().await.unwrap(); let mut tail = fill.accept(Info::timed()).unwrap().expect("a fresh group"); for frame in start..end { tail.write_frame(tail.key_time(), vec![frame as u8; size]).unwrap(); } let tail = tail.finish().unwrap(); deliveries.push(tail.handle()); held.push(pending.await.unwrap()); } let owner = deliveries[0].content().unwrap(); for delivery in &deliveries { assert!(Arc::ptr_eq(&owner, &delivery.content().unwrap())); } let (rows, bytes, shared_charge) = owner.cost(); assert_eq!(rows, if bounded { 17 } else { 32 }); assert_eq!(bytes, rows * size); let total = pool.used(); let delivery_charge: u64 = deliveries.iter().map(group::Handle::delivery_size).sum(); eprintln!( "CANONICAL_TAIL_COST size={size} bounded={bounded} delivery_handles={} canonical_rows={rows} unique_payload={bytes} shared_charge={shared_charge} delivery_charge={delivery_charge} discovery_charge={} total={total}", deliveries.len(), total - shared_charge - delivery_charge ); let mut pointers = BTreeMap::new(); for (copy, reader) in held.iter_mut().enumerate() { for index in (8 - copy) as u64..end { assert_eq!(reader.index(), index); let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), vec![index as u8; size]); if size != 0 && let Some(prior) = pointers.insert(index, frame.payload.as_ptr() as usize) { assert_eq!( prior, frame.payload.as_ptr() as usize, "one physical payload for every matching frame" ); } } if bounded { assert!(reader.read_frame().await.unwrap().is_none()); } } drop((pointers, owner, deliveries)); assert_eq!( pool.used(), total, "identity probes neither own nor conceal payload copies" ); drop(held); live.abort(Error::Cancel).unwrap(); } } } /// Evicting a Slot does not hide a canonical owner from still-held older tails; /// explicitly recreating the live group, however, starts a new incarnation. #[kio::test] #[cfg(feature = "async")] async fn tail_owner_survives_slot_removal_but_not_live_recreation() { let producer = track_producer("test", Info::timed()); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(8).unwrap(); live.write_frame(live.key_time(), b"eight".as_slice()).unwrap(); let mut tails = Vec::new(); let mut held = Vec::new(); for start in [4, 2] { let mut tail = producer .state .write() .ok() .unwrap() .insert_group_request( 0, group::Bounds { start, end: Some(start + 1), }, None, ) .unwrap() .expect("a fresh group"); tail.write_frame(tail.key_time(), vec![start as u8; 4]).unwrap(); let tail = tail.finish().unwrap(); held.push(tail.consume().with_range(start..)); tails.push(tail); } let owner = tails[0].handle().content().unwrap(); producer .state .write() .ok() .unwrap() .uncache(0) .unwrap() .abort(Error::Evicted); assert!( !tails[0].handle().is_aborted(), "the replaced delivery is held outside the Slot" ); let mut refill = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 0, end: Some(1) }, None) .unwrap() .expect("a fresh group"); assert!(Arc::ptr_eq(&owner, &refill.handle().content().unwrap())); refill.write_frame(refill.key_time(), b"zero".as_slice()).unwrap(); let refill = refill.finish().unwrap(); refill.abort(Error::Cancel).unwrap(); let mut recreated = producer.create_group(group::Info { start: 0 }).unwrap(); recreated.start_at(8).unwrap(); recreated .write_frame(recreated.key_time(), b"new eight".as_slice()) .unwrap(); assert!( recreated.handle().content().is_none(), "an explicit new live incarnation starts independently" ); let mut new = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 0, end: Some(1) }, None) .unwrap() .expect("a fresh group"); new.write_frame(new.key_time(), b"new zero".as_slice()).unwrap(); assert!(!Arc::ptr_eq(&owner, &new.handle().content().unwrap())); assert_eq!(held[0].read_frame().await.unwrap().unwrap().payload.as_ref(), &[4; 4]); } /// Activation preserves an already-streaming reader and producer, and every /// delivery's progress, arrival and abort while sharing the origin's admission. #[kio::test] #[cfg(feature = "async")] async fn canonical_activation_preserves_inflight_delivery_ownership() { for abort_first in [false, true] { let budget = frame::Budget::new(1024); let broadcast = Arc::new(broadcast::Info { frames: budget.clone(), ..Default::default() }); let producer = Producer::new(broadcast, "test", Info::timed()); let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); first.start_at(4).unwrap(); let mut first_read = first.consume().with_range(4..); let mut first = first .create_frame(frame::Info { size: 1024, timestamp: Timestamp::ZERO, }) .unwrap(); first.write(vec![4; 512]).unwrap(); let mut streaming = first_read.next_frame().await.unwrap().unwrap(); let early = streaming.read_chunk().await.unwrap().unwrap(); assert_eq!(early.as_ref(), &[4; 512]); let mut second = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 3, end: Some(5) }, None) .unwrap() .expect("a fresh group"); second.write_frame(second.key_time(), b"three".as_slice()).unwrap(); let mut second_read = second.consume().with_range(4..); let mut second = second .create_frame(frame::Info { size: 1024, timestamp: Timestamp::ZERO, }) .unwrap(); assert!( budget.reserve(1).is_none(), "overlap shares the first producer's reservation" ); assert_eq!(first.remaining(), 512); second.write(vec![4; 256]).unwrap(); assert_eq!(first.remaining(), 512, "another delivery credits no progress"); assert!(streaming.read_chunk().now_or_never().is_none()); kio::clock::sim::advance(Duration::from_millis(20)).await; if abort_first { first.abort(Error::Cancel).unwrap(); assert!(budget.reserve(1).is_none(), "the survivor still holds admission"); second.write(vec![4; 768]).unwrap(); let second = second.finish().unwrap(); second.finish().unwrap(); assert!(matches!(streaming.read_chunk().await, Err(Error::Cancel))); let frame = second_read.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), &[4; 1024]); assert_eq!( early.as_ptr(), frame.payload.as_ptr(), "the survivor reuses the published storage" ); } else { second.write(vec![4; 768]).unwrap(); let second = second.finish().unwrap(); second.finish().unwrap(); let later_copy = second_read.read_frame().await.unwrap().unwrap(); assert!( budget.reserve(1).is_none(), "the slower first delivery still owns admission" ); assert_eq!(first.remaining(), 512); assert!(streaming.read_chunk().now_or_never().is_none()); kio::clock::sim::advance(Duration::from_millis(10)).await; first.write(vec![4; 512]).unwrap(); let first = first.finish().unwrap(); let first = first.finish().unwrap(); let original_copy = first.consume().with_range(4..).read_frame().await.unwrap().unwrap(); assert_eq!( original_copy.arrived().duration_since(later_copy.arrived()), Duration::from_millis(10) ); assert_eq!(original_copy.payload.as_ptr(), later_copy.payload.as_ptr()); assert_eq!(early.as_ptr(), later_copy.payload.as_ptr()); assert_eq!(streaming.read_chunk().await.unwrap().unwrap().as_ref(), &[4; 512]); } assert!( budget.reserve(1024).is_some(), "all completed producers return admission" ); } } /// IETF Object IDs can occupy different delivery ordinals. Sharing bytes must /// preserve their typed fields and each delivery's independent terminal record. #[kio::test] #[cfg(feature = "async")] async fn canonical_content_keys_typed_objects_by_id_not_ordinal() { fn wire(first: u64) -> group::Wire { group::Wire { sparse: true, first_object: false, first_id: first, extensions: true, has_end: true, fetchable: true, last: None, } } fn write(group: group::Producer, id: u64, payload: &'static [u8]) -> group::Producer { let mut frame = group .create_object(frame::Header { info: frame::Info { size: payload.len() as u64, timestamp: Timestamp::ZERO, }, fields: Some(Arc::new(super::super::object::Fields { id, properties: super::super::object::Properties { block: Some(super::super::object::Block { // Each Object ID's own property, so sharing can't swap them. bytes: bytes::Bytes::copy_from_slice(&[0x20, id as u8]), immutable: id.is_multiple_of(2), }), }, })), inflated: None, }) .unwrap(); frame.write(payload).unwrap(); frame.finish().unwrap() } let producer = track_producer("test", Info::timed()); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(2).unwrap(); live.set_wire(wire(100)).unwrap(); let live = write(live, 100, b"a"); let head = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 1, end: Some(3) }, None) .unwrap() .expect("a fresh group"); head.set_wire(wire(90)).unwrap(); let head = write(head, 90, b"b"); let head = write(head, 100, b"a"); head.set_end_object(super::super::object::Fields { id: 101, properties: Default::default(), }) .unwrap(); let head = head.finish().unwrap(); let earlier = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 0, end: Some(2) }, None) .unwrap() .expect("a fresh group"); earlier.set_wire(wire(90)).unwrap(); let earlier = write(earlier, 90, b"b"); let earlier = write(earlier, 95, b"c"); let earlier = earlier.finish().unwrap(); assert!(head.consume().end_object().is_some_and(|fields| fields.id == 101)); assert!( earlier.consume().end_object().is_none(), "content sharing copies no terminal metadata" ); let mut prior = head.consume().with_range(1..).compressed(); let mut later = earlier.consume().compressed(); let mut a = arrayvec::ArrayVec::::new(); let mut b = arrayvec::ArrayVec::::new(); assert!(matches!( prior.poll_read_objects(&kio::Waiter::noop(), &mut a), Poll::Ready(Ok(2)) )); assert!(matches!( later.poll_read_objects(&kio::Waiter::noop(), &mut b), Poll::Ready(Ok(2)) )); assert_eq!( a.iter() .map(|object| object.fields.as_ref().unwrap().id) .collect::>(), [90, 100] ); assert_eq!( b.iter() .map(|object| object.fields.as_ref().unwrap().id) .collect::>(), [90, 95] ); assert_eq!( a[0].payload.chunk_at(0).as_ptr(), b[0].payload.chunk_at(0).as_ptr(), "one Object ID shares bytes across different ordinals" ); assert_eq!( b[1].payload.contiguous(0), b"c".as_slice(), "the same ordinal cannot alias a different Object ID" ); for object in a.iter().chain(&b) { let fields = object.fields.as_ref().unwrap(); let block = fields.properties.block.as_ref().unwrap(); assert_eq!(&block.bytes[..], [0x20, fields.id as u8], "object {}", fields.id); assert_eq!(block.immutable, fields.id.is_multiple_of(2), "object {}", fields.id); } assert_eq!(live.handle().content().unwrap().cost().0, 3); } /// Compressed transport refills keep the received slices, share one inflation, /// and continue widening hostile tail requests rather than minting local encoders. #[kio::test] #[cfg(feature = "async")] async fn canonical_compressed_ingress_preserves_raw_plain_and_widening() { fn ingest(group: group::Producer, object: &frame::Object) -> group::Producer { assert!( object.inflated.is_some(), "compressed ingress carries the stored representation" ); let mut frame = group .create_object(frame::Header { info: frame::Info { size: object.payload.len() as u64, timestamp: object.timestamp, }, fields: object.fields.clone(), inflated: object.inflated, }) .unwrap(); let mut at = 0; while at < object.payload.len() { let chunk = object.payload.chunk_at(at); at += chunk.len(); frame.write(bytes::Bytes::copy_from_slice(&chunk)).unwrap(); } frame.finish().unwrap() } let info = Info::timed().with_compression(Compression::Deflate); let reference = track_producer("reference", info.clone()); let mut source = reference.create_group(group::Info { start: 0 }).unwrap(); let expected: Vec<_> = (0..9) .map(|index| bytes::Bytes::from(format!("shared dictionary frame {index}; ").repeat(64))) .collect(); for payload in &expected { source.write_frame(source.key_time(), payload.clone()).unwrap(); } let source = source.finish().unwrap(); let mut stored = arrayvec::ArrayVec::::new(); assert!(matches!( source .consume() .compressed() .poll_read_objects(&kio::Waiter::noop(), &mut stored), Poll::Ready(Ok(9)) )); let producer = track_producer("test", info.clone()); let requests = producer.requests(); let consumer = producer.consume(); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(8).unwrap(); let live = ingest(live, &stored[8]); let mut raw_live = live.consume().with_range(8..).compressed(); let raw_live = raw_live.read_frame().await.unwrap().unwrap(); assert!( matches!(live.consume().with_range(8..).read_frame().await, Err(Error::Lagged)), "canonical storage does not grant a missing compression prefix" ); let pending = consumer.fetch_range( 0, FetchOptions { options: group::Fetch { frame_start: 2, ..Default::default() }, end: Some(4), }, ); let fill = requests.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (0, None)); let mut head = fill.accept(info).unwrap().expect("a fresh group"); for object in &stored[..8] { head = ingest(head, object); } let registry = stats::Registry::new(stats::Config::new()); let meter = registry.publisher_meter(crate::Path::new("demo").unwrap(), &stats::Tier::default()); let mut first = head.consume().with_meter(meter.clone()); assert_eq!(first.read_frame().await.unwrap().unwrap().payload, expected[0]); kio::clock::sim::advance(Duration::from_millis(20)).await; head = ingest(head, &stored[8]); let head = head.finish().unwrap(); pending.await.unwrap(); let mut raw_head = head.consume().with_range(8..).compressed(); let raw_head = raw_head.read_frame().await.unwrap().unwrap(); assert_eq!( raw_live.payload.as_ptr(), raw_head.payload.as_ptr(), "stored compressed bytes are retained once" ); assert_eq!( raw_head.payload, stored[8].payload.contiguous(0), "the relay does not recompress" ); assert_eq!( raw_head.arrived().duration_since(raw_live.arrived()), Duration::from_millis(20) ); let mut second = head.consume().with_meter(meter); for (index, payload) in expected.iter().enumerate() { if index != 0 { assert_eq!(first.read_frame().await.unwrap().unwrap().payload, *payload); } assert_eq!(second.read_frame().await.unwrap().unwrap().payload, *payload); } let inflated: u64 = registry .snapshot() .traffic() .into_iter() .map(|(_, _, traffic)| traffic.inflated) .sum(); assert_eq!( inflated, expected.iter().map(|payload| payload.len() as u64).sum::() ); for start in 1..9 { consumer .fetch_range( 0, FetchOptions { options: group::Fetch { frame_start: start, ..Default::default() }, end: Some(start + 1), }, ) .await .unwrap(); } assert!( requests.poll_requested_fill_group(&kio::Waiter::noop()).is_pending(), "compressed tail starts coalesce through the established widening policy" ); assert_eq!(head.handle().content().unwrap().cost().0, 9); } /// An evicted streaming reader still owns in-flight storage until it is dropped. /// A refill must discover those same bytes without changing that reader's abort. #[kio::test] #[cfg(feature = "async")] async fn canonical_owner_follows_an_evicted_streaming_frame_reader() { let producer = track_producer("test", Info::timed()); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(4).unwrap(); let observing = live.handle(); let mut reader = live.consume().with_range(4..); let mut live = live .create_frame(frame::Info { size: 1024, timestamp: Timestamp::ZERO, }) .unwrap(); live.write(vec![4; 512]).unwrap(); let mut streaming = reader.next_frame().await.unwrap().unwrap(); let first = streaming.read_chunk().await.unwrap().unwrap(); let original = first.as_ptr() as usize; drop(first); let mut head = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 3, end: Some(4) }, None) .unwrap() .expect("a fresh group"); head.write_frame(head.key_time(), b"three".as_slice()).unwrap(); head.finish().unwrap(); let identity = Arc::downgrade(&observing.content().unwrap()); producer .state .write() .ok() .unwrap() .uncache(0) .unwrap() .abort(Error::Evicted); drop(live); assert!( identity.upgrade().is_some(), "the streaming frame still owns canonical storage" ); let refill = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 4, end: Some(5) }, None) .unwrap() .expect("a fresh group"); assert!(Arc::ptr_eq( &identity.upgrade().unwrap(), &refill.handle().content().unwrap() )); let mut refill = refill .create_frame(frame::Info { size: 1024, timestamp: Timestamp::ZERO, }) .unwrap(); refill.write(vec![4; 1024]).unwrap(); let refill = refill.finish().unwrap(); let refill = refill.finish().unwrap(); let complete = refill.consume().with_range(4..).read_frame().await.unwrap().unwrap(); assert_eq!( complete.payload.as_ptr() as usize, original, "refill reuses the reader-held content" ); assert_eq!(complete.payload.as_ref(), &[4; 1024]); assert!(matches!(streaming.read_chunk().await, Err(Error::Evicted))); } /// Existing limits apply to each delivery, not the union of disjoint valid ranges. #[test] fn canonical_content_preserves_individually_valid_disjoint_ranges() { let producer = track_producer("test", Info::timed()); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); const FRAMES: usize = 8192; live.start_at(FRAMES as u64).unwrap(); for _ in 0..FRAMES { live.write_frame(live.key_time(), bytes::Bytes::new()).unwrap(); } let mut head = producer .state .write() .ok() .unwrap() .insert_group_request(0, group::Bounds { start: 0, end: Some(1) }, None) .unwrap() .expect("a fresh group"); head.write_frame(head.key_time(), bytes::Bytes::new()).unwrap(); let head = head.finish().unwrap(); assert_eq!(live.handle().content().unwrap().cost().0, FRAMES + 1); assert_eq!(live.frame_count(), 2 * FRAMES); assert_eq!(head.frame_count(), 1); } /// Frame `index` of the groups below: 16 KiB naming its index. fn numbered(index: u32) -> bytes::Bytes { bytes::Bytes::from(index.to_le_bytes().repeat(4096)) } /// One live group grows through a pool far smaller than it: pool debt sheds its front, /// memory stays near the pool, a reader that keeps up reads every frame, and a late /// reader is `Evicted`. #[test] #[cfg(feature = "async")] fn a_live_group_sheds_its_front_under_pressure() { const POOL: u64 = 4 * 1024 * 1024; let pool = cache::Pool::new(cache::Config::default().with_capacity(POOL)); let producer = track_producer_pooled("test", pool.clone()); let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); let mut reader = live.consume(); let mut late = live.consume(); // 48 MiB, past the 32 MiB tail: pool debt sheds first. for index in 0..3_000 { live.write_frame(live.key_time(), numbered(index)).unwrap(); let frame = reader.read_frame().now_or_never().unwrap(); let frame = frame.unwrap_or_else(|err| panic!("frame {index}: {err}")).unwrap(); assert_eq!(frame.payload, numbered(index)); assert!(pool.used() < 2 * POOL, "frame {index}: {} bytes held", pool.used()); } assert!(matches!(late.read_frame().now_or_never().unwrap(), Err(Error::Evicted))); assert!(!live.is_aborted(), "shedding never ends the group"); } /// A join onto a group whose first frame was shed asks for a new group on a track that /// opts into refresh, rather than reading a group it could only see `Evicted` in. #[test] fn a_join_past_a_shed_first_frame_wants_a_new_group() { let pool = cache::Pool::new(cache::Config::default().with_capacity(256 * 1024)); let producer = Producer::new( Arc::new(broadcast::Info { pool, ..Default::default() }), "test", Info::default().with_refresh(Refresh::default().with_ratio(1_000_000)), ); let mut live = producer.append_next().unwrap(); for index in 0..1_000 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } let _subscriber = producer.subscribe(None); assert_eq!(producer.demand().wanted_start(), Some(1)); } /// A reader that reaches a shed frame of the newest group asks for a new group on a /// track that opts into refresh. #[test] #[cfg(feature = "async")] fn a_reader_at_a_shed_frame_wants_a_new_group() { let pool = cache::Pool::new(cache::Config::default().with_capacity(256 * 1024)); let producer = Producer::new( Arc::new(broadcast::Info { pool, ..Default::default() }), "test", Info::default().with_refresh(Refresh::default().with_ratio(1_000_000)), ); // Joined before the group grew, so the join asks for nothing. let _subscriber = producer.subscribe(None); let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); for index in 0..1_000 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } assert_eq!(producer.demand().wanted_start(), Some(0), "nothing asked yet"); assert!(matches!( reader.read_frame().now_or_never().unwrap(), Err(Error::Evicted) )); assert_eq!(producer.demand().wanted_start(), Some(1), "the reader asked"); let _joined = producer.subscribe(None); assert_eq!( producer.demand().wanted_start(), Some(1), "a join asks for the same group" ); } /// The frame a read takes right now, as its number in [`numbered`]. #[cfg(feature = "async")] fn read_numbered(reader: &mut group::Consumer) -> Option { let frame = reader.read_frame().now_or_never()?; let frame = frame.expect("read failed").expect("the group ended"); Some(u32::from_le_bytes(frame.payload[..4].try_into().unwrap())) } /// A late joiner of a live group that shed its front boards a flight from frame 0, /// served by the track's handler, reads it as the handler writes it, and hands over to /// the live copy once that copy holds its frame within reach of the live reader. The /// flight is then unwanted, so its handler stops. #[test] #[cfg(feature = "async")] fn a_late_joiner_rides_a_flight_and_hands_over() { let producer = track_producer("test", Info::default()); let requests = producer.requests(); let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); let mut late = live.consume(); // 48 MiB, half again the live tail. const FRAMES: u32 = 3_000; for index in 0..FRAMES { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); } // Frame 0 was shed: the late reader asks the handler for a flight from it. assert_eq!(read_numbered(&mut late), None); let fill = match requests.poll_requested_groups(&kio::Waiter::noop()) { Poll::Ready(fill) => fill.unwrap(), Poll::Pending => panic!("a flight is asked"), }; assert_eq!(fill.start(), Position { group: 0, frame: 0 }); let mut flight = fill.accept(0, None).unwrap().expect("a fresh group"); assert_eq!(read_numbered(&mut late), None, "aboard, waiting on the handler"); let unwanted = |flight: &group::Producer| flight.handle().poll_unwanted(&kio::Waiter::noop()).is_ready(); assert!(!unwanted(&flight), "ridden"); // The handler writes as the rider reads, until the rider leaves for the live copy. let mut index = 0; while !unwanted(&flight) { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut late), Some(index)); index += 1; assert!(index < FRAMES, "handed over before the flight caught up"); } // The live writer runs its whole window ahead of the live reader at once: the // rider, as far behind that reader as it was admitted, keeps its frame. let mut written = FRAMES; while live .poll_ready(&kio::Waiter::noop(), &frame_info(&numbered(0))) .is_ready() { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(written)) .unwrap(); written += 1; } assert!(written > FRAMES + 900, "a whole window: {written}"); while index < written { assert_eq!( read_numbered(&mut late), Some(index), "frame {index} from the live copy" ); index += 1; } assert_eq!(read_numbered(&mut late), None, "at the live edge"); } /// A plain reader of a compressed group that fell off boards a flight from frame 0, /// since every later frame builds on the ones before it, and skips the frames before its /// own as the handler writes them. It then hands over to the live copy. #[test] #[cfg(feature = "async")] fn a_compressed_flight_starts_at_frame_0_and_its_rider_skips_ahead() { let producer = track_producer("test", Info::default().with_compression(Compression::Deflate)); let requests = producer.requests(); let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); let mut behind = live.consume(); const FRAMES: u32 = 3_000; for index in 0..FRAMES { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); if index < 10 { assert_eq!(read_numbered(&mut behind), Some(index)); } } // The live reader's inflated copy shed frame 10, which the decoder is past. assert_eq!(read_numbered(&mut behind), None); let fill = asked(&requests).expect("a flight"); assert_eq!(fill.start(), Position { group: 0, frame: 0 }, "from frame 0"); let mut flight = fill.accept(0, None).unwrap().expect("a fresh group"); let unwanted = |flight: &group::Producer| flight.handle().poll_unwanted(&kio::Waiter::noop()).is_ready(); for index in 0..10 { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut behind), None, "frame {index} is skipped"); } let mut index = 10; while !unwanted(&flight) { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut behind), Some(index)); index += 1; assert!(index < FRAMES, "handed over before the flight caught up"); } while index < FRAMES { assert_eq!( read_numbered(&mut behind), Some(index), "frame {index} from the live copy" ); index += 1; } assert_eq!(read_numbered(&mut behind), None, "at the live edge"); } /// The size of `payload` as a frame, for [`group::Producer::poll_ready`], which weighs /// nothing else. #[cfg(feature = "async")] fn frame_info(payload: &bytes::Bytes) -> frame::Info { frame::Info { size: payload.len() as u64, timestamp: Timestamp::unscaled(0), } } /// A live group of `frames` numbered frames past its tail, read along by a live reader, /// on a track with a handler. #[cfg(feature = "async")] fn shed_live_group(frames: u32) -> (Producer, Requests, group::Producer, group::Consumer) { let producer = track_producer("test", Info::default()); let requests = producer.requests(); let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); for index in 0..frames { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); } (producer, requests, live, reader) } /// The next fill asked of `requests`, if any. #[cfg(feature = "async")] fn asked(requests: &Requests) -> Option { match requests.poll_requested_groups(&kio::Waiter::noop()) { Poll::Ready(fill) => Some(fill.unwrap()), Poll::Pending => None, } } /// Joiners at the same shed frame share one flight; one joining later boards it while /// its frame is within reach of the flight's fastest rider, and asks a flight of its /// own once it is not. #[test] #[cfg(feature = "async")] fn joiners_within_reach_share_one_flight() { let (_producer, requests, live, _reader) = shed_live_group(3_000); let (mut first, mut second) = (live.consume(), live.consume()); assert_eq!(read_numbered(&mut first), None); assert_eq!(read_numbered(&mut second), None); let fill = asked(&requests).expect("a flight"); assert!(asked(&requests).is_none(), "the second joiner shares it"); let mut flight = fill.accept(0, None).unwrap().expect("a fresh group"); // 8 MiB in, half the reach. for index in 0..500 { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut first), Some(index)); } let mut near = live.consume(); assert_eq!(read_numbered(&mut near), Some(0), "boards the flight at once"); assert_eq!(read_numbered(&mut second), Some(0)); assert!(asked(&requests).is_none(), "one upstream fill"); // 17 MiB in, past the reach. for index in 500..1_100 { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut first), Some(index)); } let mut far = live.consume(); assert_eq!(read_numbered(&mut far), None, "too far behind the flight's rider"); let own = asked(&requests).expect("a flight of its own"); assert_eq!(own.start(), Position { group: 0, frame: 0 }); } /// A flight sheds its front under pool debt like the copy it replays, so a pool far /// smaller than one tail holds a live group and its flight near the pool. #[test] #[cfg(feature = "async")] fn pool_debt_sheds_a_flight() { const POOL: u64 = 4 * 1024 * 1024; // A read held open elsewhere in the process stalls the log's deferred frees, which // must not stall what the pool counts as freed. let _read = crossbeam_epoch::pin(); let pool = cache::Pool::new(cache::Config::default().with_capacity(POOL)); let producer = Producer::new( Arc::new(broadcast::Info { pool: pool.clone(), ..Default::default() }), "test", Info::default(), ); let requests = producer.requests(); let mut live = producer.append_next().unwrap(); let mut late = live.consume(); for index in 0..1_000 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } assert_eq!(read_numbered(&mut late), None); let mut flight = asked(&requests) .expect("a flight") .accept(0, None) .unwrap() .expect("a fresh group"); assert_eq!(read_numbered(&mut late), None); for index in 0..1_000 { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert_eq!(read_numbered(&mut late), Some(index)); assert!(pool.used() < 2 * POOL, "frame {index}: {} bytes held", pool.used()); } } /// A reader waiting on the flight it asked for still goes stale: its budget gives up on /// the group once a newer one supersedes it, though the handler never answered. #[test] #[cfg(feature = "async")] fn a_reader_boarding_a_flight_goes_stale() { let producer = track_producer("test", Info::timed()); let requests = producer.requests(); let mut sub = producer.subscribe(None); let mut live = producer.append_next().unwrap(); for index in 0..3_000u32 { let at = Timestamp::from_millis(index.into()).unwrap(); live.write_frame(at, numbered(index)).unwrap(); } let mut late = sub.recv_group().now_or_never().unwrap().unwrap().expect("group 0"); assert!(late.read_frame().now_or_never().is_none(), "boarding"); let _flight = asked(&requests).expect("a flight, never answered"); let mut newer = producer.append_next().unwrap(); newer .write_frame(Timestamp::from_millis(100_000).unwrap(), numbered(0)) .unwrap(); assert!(matches!(late.read_frame().now_or_never(), Some(Err(Error::Old)))); } /// A flight asks upstream with its rider's priority and budget, a subscriber's or a /// fetch's, as any other fill does, so a late joiner's replay is not ranked below live /// traffic. It asks oldest first whatever the rider's order, since it holds one group. #[test] #[cfg(feature = "async")] fn a_flight_asks_with_its_riders_priority() { let budget = Duration::from_secs(10); let producer = track_producer("test", Info::default()); let requests = producer.requests(); let mut sub = producer.subscribe( Subscription::default() .with_priority(5) .with_delay(Delay::default().with_max(budget)) .with_order(Order::Descending), ); let mut live = producer.append_next().unwrap(); for index in 0..3_000u32 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } let mut late = sub.recv_group().now_or_never().unwrap().unwrap().expect("group 0"); assert_eq!(read_numbered(&mut late), None, "boarding"); let fill = asked(&requests).expect("a flight"); assert_eq!(fill.start(), Position { group: 0, frame: 0 }); assert_eq!(fill.priority(), 5); assert_eq!(fill.max_delay(), Some(budget)); assert_eq!(fill.order(), Order::Ascending); drop(late); drop(fill); let fetch = group::Fetch::default().with_priority(-3).with_max_delay(budget); let mut fetched = producer .consume() .fetch_group(0, fetch) .now_or_never() .expect("cached") .expect("group 0"); assert_eq!(read_numbered(&mut fetched), None, "boarding"); let fill = asked(&requests).expect("a flight"); assert_eq!(fill.priority(), -3); assert_eq!(fill.max_delay(), Some(budget)); } /// A rider that boarded a flight already asked under a tighter budget is not failed by /// that budget's `Old`: it asks a flight of its own with its budget. #[test] #[cfg(feature = "async")] fn a_looser_rider_asks_again_after_old() { let producer = track_producer("test", Info::default()); let requests = producer.requests(); let subscribe = |budget: u64| { let delay = Delay::default().with_max(Duration::from_secs(budget)); producer.subscribe(Subscription::default().with_delay(delay)) }; let (mut tight, mut loose) = (subscribe(1), subscribe(10)); let mut live = producer.append_next().unwrap(); for index in 0..3_000u32 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } let recv = |sub: &mut Subscriber| sub.recv_group().now_or_never().unwrap().unwrap().expect("group 0"); let (mut tight, mut loose) = (recv(&mut tight), recv(&mut loose)); assert_eq!(read_numbered(&mut tight), None, "boarding"); let req = requests.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(req.max_delay(), Some(Duration::from_secs(1))); assert_eq!(read_numbered(&mut loose), None, "boards the same flight"); assert!(requests.requested_fill_group().now_or_never().is_none(), "one flight"); req.reject(Error::Old); assert!(matches!(tight.read_frame().now_or_never(), Some(Err(Error::Evicted)))); assert_eq!(read_numbered(&mut loose), None, "asks again"); let req = requests.requested_fill_group().now_or_never().unwrap().unwrap(); assert_eq!(req.max_delay(), Some(Duration::from_secs(10))); let mut flight = req.accept(None).unwrap().expect("a fresh group"); flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(0)) .unwrap(); assert_eq!(read_numbered(&mut loose), Some(0)); } /// Pool debt sheds a flight even once the cached copy of its group was evicted whole. #[test] #[cfg(feature = "async")] fn pool_debt_sheds_a_flight_without_its_cached_group() { const POOL: u64 = 4 * 1024 * 1024; // A read held open elsewhere in the process stalls the log's deferred frees, which // must not stall what the pool counts as freed. let _read = crossbeam_epoch::pin(); let pool = cache::Pool::new(cache::Config::default().with_capacity(POOL)); let producer = Producer::new( Arc::new(broadcast::Info { pool: pool.clone(), ..Default::default() }), "test", Info::default(), ); let requests = producer.requests(); let mut old = producer.append_next().unwrap(); let mut late = old.consume(); for index in 0..1_000 { old.write_frame(crate::Timestamp::unscaled(old.start), numbered(index)) .unwrap(); } assert_eq!(read_numbered(&mut late), None); let mut flight = asked(&requests) .expect("a flight") .accept(0, None) .unwrap() .expect("a fresh group"); assert_eq!(read_numbered(&mut late), None); // A newer group leaves group 0 cold, and its cached copy goes whole. let mut newer = producer.append_next().unwrap(); for index in 0..1_000 { newer .write_frame(crate::Timestamp::unscaled(newer.start), numbered(index)) .unwrap(); } assert!(!producer.state.read().lookup.contains_key(&0), "group 0 evicted"); for index in 0..1_000 { flight .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) .unwrap(); assert!(pool.used() < 2 * POOL, "frame {index}: {} bytes held", pool.used()); } // Ending every copy frees what they held, though the read still holds their slots. flight.abort(Error::Cancel).unwrap(); drop((late, old, newer)); assert_eq!(pool.used(), 0); } /// A handler that refuses a flight leaves the reader where a track without one does: a /// track that opts into refresh asks for a new group, and the read is `Evicted`. #[test] #[cfg(feature = "async")] fn a_refused_flight_falls_back_to_refresh() { let producer = track_producer( "test", Info::default().with_refresh(Refresh::default().with_ratio(1_000_000)), ); let requests = producer.requests(); // Joined before the group grew, so the join asks for nothing. let _subscriber = producer.subscribe(None); let mut live = producer.append_next().unwrap(); let mut late = live.consume(); for index in 0..3_000 { live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) .unwrap(); } assert_eq!(read_numbered(&mut late), None, "asks a flight"); assert_eq!(producer.demand().wanted_start(), Some(0), "nothing refreshed yet"); asked(&requests).expect("a flight").refuse_groups(0..1, Error::NotFound); assert!(matches!(late.read_frame().now_or_never().unwrap(), Err(Error::Evicted))); assert_eq!( producer.demand().wanted_start(), Some(1), "the reader asked for a new group" ); } /// A rider that falls off its flight's tail never holds the flight's writer back, and /// boards a flight of its own from the frame it fell off at. #[test] #[cfg(feature = "async")] fn a_slow_rider_falls_off_and_flies_on_its_own() { let (_producer, requests, live, _reader) = shed_live_group(3_000); let (mut fast, mut slow) = (live.consume(), live.consume()); assert_eq!(read_numbered(&mut fast), None); assert_eq!(read_numbered(&mut slow), None); let mut flight = asked(&requests) .unwrap() .accept(0, None) .unwrap() .expect("a fresh group"); assert_eq!(read_numbered(&mut fast), None); assert_eq!(read_numbered(&mut slow), None); assert_eq!(read_numbered(&mut slow), None); for index in 0..2_500 { let payload = numbered(index); assert!( flight .poll_ready(&kio::Waiter::noop(), &frame_info(&payload)) .is_ready(), "frame {index}: paced by the fast rider only" ); flight .write_frame(crate::Timestamp::unscaled(flight.start), payload) .unwrap(); assert_eq!(read_numbered(&mut fast), Some(index)); } assert_eq!(read_numbered(&mut slow), None, "fell off"); let own = asked(&requests).expect("a flight of its own"); assert_eq!(own.start(), Position { group: 0, frame: 0 }); } /// What a read returns right now: `Some(Some(key))` for a group, `Some(None)` for /// its end, `None` while it parks. fn read_now(subscriber: &mut Subscriber, ordered: bool) -> Option> { match subscriber.poll_group(ordered, &kio::Waiter::noop()) { Poll::Ready(res) => Some(res.expect("would have errored").map(|group| group.start)), Poll::Pending => None, } } /// Exhaustion stays observable through repeated reads and unchanged or narrower ranges; /// widening reopens the cursor without replaying groups already handed out. fn bounded_read_reopens(track: &Consumer, source: &mut Producer) { for ordered in [false, true] { for parks in [false, true] { let subscription = replay() .with_order(Order::Ascending) .with_ranges([Range::groups(1..2), Range::groups(3..4)]); let mut subscribing = track.subscribe(subscription.clone()); let Poll::Ready(Ok(mut reader)) = kio::Task::poll(&mut *subscribing, &kio::Waiter::noop()) else { panic!("the track info is available"); }; if parks { reader = reader.parked(); } let end = if parks { None } else { Some(None) }; assert_eq!(read_now(&mut reader, ordered), Some(Some(1))); assert_eq!(read_now(&mut reader, ordered), Some(Some(3))); for _ in 0..3 { assert_eq!(read_now(&mut reader, ordered), end, "ordered={ordered}, parks={parks}"); } source.set_idle(); for _ in 0..2 { assert_eq!(read_now(&mut reader, ordered), end, "the source is idle"); } source.set_live(Some(Position::group(5)), None).unwrap(); reader.update(subscription.clone()).unwrap(); assert_eq!(read_now(&mut reader, ordered), end); reader.update(subscription.clone().with_range(3..4)).unwrap(); assert_eq!(read_now(&mut reader, ordered), end); reader.update(subscription.clone().with_range(0..6)).unwrap(); let newly_covered = if ordered { &[4, 5][..] } else { &[0, 2, 4, 5][..] }; for key in newly_covered { assert_eq!(read_now(&mut reader, ordered), Some(Some(*key))); } for _ in 0..3 { assert_eq!(read_now(&mut reader, ordered), end); } reader.update(subscription.with_range(0..)).unwrap(); assert_eq!( read_now(&mut reader, ordered), None, "the open range still owes new groups" ); } } } #[test] fn a_bounded_read_stays_ended_until_widened() { let mut producer = track_producer("test", Info::timed()); for key in 0..6 { write_group(&producer, key); } bounded_read_reopens(&producer.consume(), &mut producer); } #[test] fn a_routed_bounded_read_stays_ended_until_widened() { let routes = super::super::resume::Producer::new(); let logical = Request::new(Arc::new(broadcast::Info::default()), "test") .routes(routes.consume()) .accept(Info::timed()); let mut producer = track_producer("test", Info::timed()); for key in 0..6 { write_group(&producer, key); } routes.serve(producer.consume()); bounded_read_reopens(&logical.consume(), &mut producer); } /// An open range, or a bounded one still owed a group, parks; a local cap alone bounds /// nothing, since its reader may raise it again. #[test] fn a_read_with_more_to_come_parks() { let producer = track_producer("test", Info::timed()); producer.create_group(0u64.into()).unwrap().finish().unwrap(); let mut open = producer.subscribe(replay().with_range(0..)); assert_eq!(read_now(&mut open, false), Some(Some(0))); assert_eq!(read_now(&mut open, false), None); let mut capped = producer.subscribe(replay().with_range(0..)).with_range(..1); assert_eq!(read_now(&mut capped, false), Some(Some(0))); assert_eq!(read_now(&mut capped, false), None); let mut owed = producer.subscribe(replay().with_range(0..2)); assert_eq!(read_now(&mut owed, false), Some(Some(0))); assert_eq!(read_now(&mut owed, false), None, "group 1 may still arrive"); producer.create_group(1u64.into()).unwrap().finish().unwrap(); assert_eq!(read_now(&mut owed, false), Some(Some(1))); assert_eq!(read_now(&mut owed, false), Some(None)); } /// A dropped group, or one skipped as stale, accounts for its place in a bounded window /// as a group handed out does. #[test] fn a_bounded_read_ends_past_dropped_and_stale_groups() { let producer = track_producer("test", Info::timed()); let mut dropped = producer.subscribe(replay().with_range(0..2)); producer.create_group(0u64.into()).unwrap().finish().unwrap(); assert_eq!(read_now(&mut dropped, false), Some(Some(0))); assert_eq!(read_now(&mut dropped, false), None); producer.drop_groups(1..2, StreamError::NotFound); for _ in 0..3 { assert_eq!(read_now(&mut dropped, false), Some(None)); } let producer = track_producer("test", Info::timed()); // Group 1 runs on past where group 0 could still present. for (key, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(*at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } group.finish().unwrap(); } let mut stale = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(Duration::from_millis(10))) .with_order(Order::Ascending) .with_range(0..2), ); assert_eq!(read_now(&mut stale, false), Some(Some(1)), "group 0 is stale"); for _ in 0..3 { assert_eq!(read_now(&mut stale, false), Some(None)); } } /// A wire publisher's read parks past its window's end, since its peer would read the /// end of the subscription as the track's. #[test] fn a_parked_bounded_read_waits_past_its_end() { let producer = track_producer("test", Info::timed()); producer.create_group(0u64.into()).unwrap().finish().unwrap(); let mut reader = producer.subscribe(replay().with_range(0..1)).parked(); assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), None); } /// A bounded read waits on the fill its miss asked, and ends once it took the answer. #[kio::test] #[cfg(feature = "async")] async fn a_bounded_read_ends_after_its_fill() { let (producer, _pool) = pooled_producer(1 << 40); // A received copy, whose missing groups may still arrive. producer.received(); let dynamic = producer.requests(); producer.create_group(0u64.into()).unwrap().finish().unwrap(); producer.create_group(2u64.into()).unwrap().finish().unwrap(); let producer = producer.finish().unwrap(); // From group 0, which a time start names without a lookup, through the hole at 1. let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(0..2)); assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), None, "the fill is out"); let fill = dynamic .requested_fill_group() .now_or_never() .expect("the miss asks a fill") .unwrap(); assert_eq!(fill.key(), 1); let mut group = fill.accept(Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"filled")) .unwrap(); group.finish().unwrap(); assert_eq!(read_now(&mut reader, false), Some(Some(1))); assert_eq!(read_now(&mut reader, false), Some(None)); } /// A hole the live feed may still deliver is waited for over the arrival grace before /// it is asked as a miss. With no budget and no publisher max age nothing bounds that /// grace, so the wait is [`ledger::GRACE`]: a bounded read of a hole below where a live /// feed started, with no start declared, ends rather than parking for good. #[kio::test] #[cfg(feature = "async")] async fn a_hole_below_an_undeclared_start_is_asked_after_a_bounded_grace() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); producer.create_group(10u64.into()).unwrap().finish().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) .with_range(0..1), ); assert_eq!(read_now(&mut reader, false), None, "group 0 may still arrive"); assert!(dynamic.requested_fill_group().now_or_never().is_none()); elapse(&producer, ledger::GRACE); assert_eq!(read_now(&mut reader, false), None, "the fill is out"); let fill = dynamic .requested_fill_group() .now_or_never() .expect("the hole is asked once the grace passed") .unwrap(); assert_eq!(fill.key(), 0); fill.reject(Error::NotFound); assert_eq!(read_now(&mut reader, false), Some(None), "the refusal is final"); // With no fill handler, the hole is given up once the grace passed. let (producer, _pool) = pooled_producer(1 << 40); producer.create_group(10u64.into()).unwrap().finish().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) .with_range(0..1), ); assert_eq!(read_now(&mut reader, false), None); elapse(&producer, ledger::GRACE); assert_eq!(read_now(&mut reader, false), Some(None)); } /// A hole below a declared start is asked at once, and a refusal there is held while the /// live feed may still deliver the group late. With nothing bounding the arrival grace the /// hold lasts [`ledger::GRACE`], on a received copy as on a local track, so the read ends. #[kio::test] #[cfg(feature = "async")] async fn a_refusal_below_the_start_is_held_for_a_bounded_grace() { for received in [false, true] { let (mut producer, _pool) = pooled_producer(1 << 40); if received { producer.received(); } let dynamic = producer.requests(); producer.start_at(10).unwrap(); producer.create_group(10u64.into()).unwrap().finish().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) .with_range(0..1), ); assert_eq!(read_now(&mut reader, false), None, "received: {received}"); let fill = dynamic .requested_fill_group() .now_or_never() .expect("a hole below the start is asked at once") .unwrap(); assert_eq!(fill.key(), 0, "received: {received}"); fill.reject(Error::NotFound); assert_eq!( read_now(&mut reader, false), None, "received: {received}: may still arrive" ); elapse(&producer, ledger::GRACE); assert_eq!(read_now(&mut reader, false), Some(None), "received: {received}"); } } /// A copy its session holds no subscription for asks a bounded read's holes as misses /// at once, though it is idle and its feed produced nothing, and the read ends once the /// fill answered or refused each, since nothing else will deliver them. #[kio::test] #[cfg(feature = "async")] async fn an_unfed_copy_asks_a_bounded_read_s_holes_at_once() { let (mut producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); producer.set_idle(); let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(0..2)); assert_eq!(read_now(&mut reader, false), None); assert!( dynamic.requested_groups().now_or_never().is_none(), "an idle copy a feed may still answer asks nothing" ); producer.set_unfed(true); assert_eq!(read_now(&mut reader, false), None, "the fill is out"); let fill = dynamic .requested_groups() .now_or_never() .expect("the holes are asked at once") .unwrap(); assert_eq!((fill.start(), fill.end()), (Position::group(0), 2)); let mut group = fill.accept(0, Info::timed()).unwrap().expect("a fresh group"); group .write_frame(group.key_time(), bytes::Bytes::from_static(b"filled")) .unwrap(); group.finish().unwrap(); fill.refuse_groups(1..2, Error::Unsupported); assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), Some(None), "the refusal is final"); } /// On a copy no subscription feeds, a bounded read takes what the cache holds in its /// window, idle or not, while an open read still waits for a feed to answer. #[test] fn an_unfed_copy_serves_a_bounded_read_from_its_cache() { let producer = idle_copy(); producer.set_unfed(true); let mut bounded = producer.subscribe(replay().with_order(Order::Ascending).with_range(1..3)); assert_eq!(read_now(&mut bounded, false), Some(Some(1))); assert_eq!(read_now(&mut bounded, false), Some(Some(2))); assert_eq!(read_now(&mut bounded, false), Some(None)); let mut open = producer.subscribe(replay().with_range(1..)); assert_eq!(read_now(&mut open, false), None, "the cache is not the live edge"); } /// A long bounded read that writes a group off as stale between each it hands out /// remembers a write-off only while its group is cached, since only then can a wider /// budget offer it again, and still ends. #[test] fn a_bounded_read_retires_its_write_offs_as_it_advances() { // Small enough that each round evicts the groups before it. let (producer, _pool) = pooled_producer(1); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(Duration::from_millis(10))) .with_order(Order::Ascending) .with_range(0..16), ); for round in 0..8u64 { // Group 2k+1 runs on past where group 2k could still present, so 2k is stale. let at = 2_000 * round; for (key, frames) in [(2 * round, vec![at]), (2 * round + 1, vec![at + 1_000, at + 1_100])] { let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } group.finish().unwrap(); } assert_eq!(read_now(&mut reader, false), Some(Some(2 * round + 1)), "round {round}"); assert!( reader.written_off_runs() <= 1, "round {round}: {}", reader.written_off_runs() ); } assert_eq!(read_now(&mut reader, false), Some(None)); } /// A read that skipped a group as stale while its window was open, then narrows to a /// bounded window holding that group, still ends, whether the group is cached or evicted. #[test] fn a_read_narrowed_past_a_stale_skip_ends() { for evict in [false, true] { let (producer, _pool) = pooled_producer(if evict { 1 } else { 1 << 40 }); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(Duration::from_millis(10))) .with_order(Order::Ascending) .with_range(0..), ); for (key, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(*at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } group.finish().unwrap(); } assert_eq!(read_now(&mut reader, false), Some(Some(1)), "evict: {evict}"); reader .update( Subscription::default() .with_delay(crate::track::Delay::default().with_max(Duration::from_millis(10))) .with_order(Order::Ascending) .with_range(0..2), ) .unwrap(); assert_eq!(read_now(&mut reader, false), Some(None), "evict: {evict}"); } } /// A hole below the start that no fill handler can take is held as refused while the feed /// may still deliver it, and the hold ending wakes the parked reader. #[kio::test] #[cfg(feature = "async")] async fn an_unfillable_hole_below_the_start_wakes_its_reader() { let (mut producer, _pool) = pooled_producer(1 << 40); producer.start_at(10).unwrap(); producer.create_group(10u64.into()).unwrap().finish().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) .with_range(0..1), ); let (flag, waiter) = Flag::waiter(); assert!(reader.poll_group(false, &waiter).is_pending()); flag.take(); elapse(&producer, ledger::GRACE); assert!(flag.take(), "the hold's end woke the reader"); assert_eq!(read_now(&mut reader, false), Some(None)); } }