// Copyright 2026 Josh Waldrep // SPDX-License-Identifier: Apache-2.0 package deferred import ( "bufio" "encoding/json" "errors" "fmt" "os" "path/filepath" "sync" "time" "github.com/luckyPipewrench/pipelock/internal/config" "github.com/luckyPipewrench/pipelock/internal/directorysync" ) const ( StateHeld = "deferred_held" StateResolving = "resolving" StateResolvedAllow = "resolved_allow" StateResolvedBlock = "resolved_block" StateResolvedStepUp = "resolved_step_up" // StateAdmissionRejected records an action that was NEVER successfully // held (duplicate defer_id, capacity, cascade-limit). It is an audit-only // marker and must never be replayed as a hold-lifecycle transition: for a // duplicate defer_id it shares the id of a live hold, so treating it as a // resolution would delete that live hold from restart recovery. StateAdmissionRejected = "admission_rejected" SourceContext = "context" SourceTimeout = "timeout" SourceCancel = "cancel" SourceRestartRecovery = "restart_recovery" SourceKillSwitch = "kill_switch" SourceCapacity = "capacity" SourceCascade = "cascade" SourceCascadeLimit = "cascade_limit" SourceDuplicateDeferID = "duplicate_defer_id" SourcePolicyReload = "policy_reload" SourceApproval = "approval" SourceAuthority = "authority" SourceOperator = "operator" SourceToolInventory = "tool_inventory" LinkageSessionPendingAncestor = "session_pending_ancestor" DefaultTimeoutSeconds = 2 DefaultMaxPending = 64 DefaultMaxPendingSession = 8 DefaultMaxPendingBytes = 1024 * 1024 DefaultMaxCascadeDepth = DefaultMaxPendingSession ) // SourceUpstreamContract records a deferred release that the live upstream // contract gate denied at the irreversible send boundary. const SourceUpstreamContract = "upstream_contract" // ReasonReceiptNotWritten is the resolution reason for an allow that was closed // because its required receipt could not be written. const ReasonReceiptNotWritten = "required receipt could not be written" // ReasonReleaseNotJournaled is the resolution reason for an allow that was // closed because the journal could not record the release as final after the // allow receipt was written. The call is not sent: restart recovery would close // the still-pending release to block, so sending it would contradict that. const ReasonReleaseNotJournaled = "release could not be journaled" // Config controls held-action bounds and timers. type Config struct { Enabled bool Timeout time.Duration MaxPending int MaxPendingPerSession int MaxPendingBytes int MaxCascadeDepth int JournalPath string JournalWriteGuard func(func() error) error Warningf func(format string, args ...any) } // ResolutionPolicy is persisted into the defer receipt. type ResolutionPolicy struct { Timeout time.Duration `json:"timeout"` MaxPending int `json:"max_pending"` MaxPendingPerSession int `json:"max_pending_per_session"` MaxPendingBytes int `json:"max_pending_bytes"` MaxCascadeDepth int `json:"max_cascade_depth"` } func (p ResolutionPolicy) String() string { data, err := json.Marshal(p) if err != nil { return "" } return string(data) } type ReceiptCascade struct { ParentDeferID string `json:"parent_defer_id,omitempty"` CascadeDepth int `json:"cascade_depth"` Linkage string `json:"linkage,omitempty"` } type ReceiptPolicy struct { Bounds ResolutionPolicy `json:"bounds"` AllowOn config.DeferAllowOn `json:"allow_on,omitempty"` StepUpOn config.DeferStepUpOn `json:"step_up_on,omitempty"` Cascade *ReceiptCascade `json:"cascade,omitempty"` } type ReceiptPolicyOptions struct { Bounds ResolutionPolicy Rule config.DeferResolutionPolicy Cascade *ReceiptCascade } func ReceiptPolicyString(bounds ResolutionPolicy, rule config.DeferResolutionPolicy) string { return ReceiptPolicyStringFor(ReceiptPolicyOptions{Bounds: bounds, Rule: rule}) } func ReceiptPolicyStringFor(opts ReceiptPolicyOptions) string { data, err := json.Marshal(ReceiptPolicy{ Bounds: opts.Bounds, AllowOn: opts.Rule.AllowOn, StepUpOn: opts.Rule.StepUpOn, Cascade: opts.Cascade, }) if err != nil { return "" } return string(data) } // AuthoritySnapshot preserves the original identity through async resolution. type AuthoritySnapshot struct { Principal string Actor string SessionID string SessionIDOriginal string } // HeldAction is the immutable action payload stored while awaiting resolution. type HeldAction struct { DeferID string ActionID string ShardIndex int ShardSelected bool Target string Reason string Surface string Method string SizeBytes int Policy ResolutionPolicy RulePolicy config.DeferResolutionPolicy Authority AuthoritySnapshot ParentDeferID string CascadeDepth int Linkage string Deadline time.Time Payload []byte ArgDigest string Resolve func(Resolution) BeforeAllow func() (release func(), ok bool) // Prepare, when set, runs after BeforeAllow and before the terminal // journal entry. It may downgrade the resolution (for example when a // release-boundary check cancels an allow) so the journal records the // outcome that actually happens. The returned finish func runs, via // defer, after Resolve returns or panics. Prepare func(Resolution) (Resolution, func()) // AfterJournal, when set, normally runs for an allow, after the journal has // recorded the release as pending and before it records the allow as final. // It is where evidence that must not exist for a call that is never sent // (the allow resolution receipt) belongs: Prepare runs ahead of the journal // write and cannot know it will succeed. Because the journal still shows // the release as pending while this runs, a process that dies here leaves a // hold that restart recovery closes to block with its own receipt. A // non-nil error closes the allow. Whenever a release closes to block, it // is called again with the block resolution before the terminal journal // entry, so that block's receipt is written first. Prepare's // finish func is still pending, so the release claim and sink lock cover // this call. AfterJournal func(Resolution) error timer *time.Timer state string createdAt time.Time } // Resolution is delivered exactly once for a held action. type Resolution struct { DeferID string ParentActionID string ShardIndex int ShardSelected bool FinalDecision string ResolutionSource string Authority AuthoritySnapshot ParentDeferID string CascadeDepth int Linkage string Policy ResolutionPolicy Target string Method string Reason string } // Manager tracks per-action holds. It is safe for concurrent use. type Manager struct { mu sync.Mutex journalMu sync.Mutex cfg Config warningf func(format string, args ...any) holds map[string]*HeldAction sessionHolds map[string][]string totalBytes int pendingJournal map[string]journalEntry // fileSync replaces File.Sync for journal writes in tests; nil uses it. fileSync func(*os.File) error // dirSync replaces directorysync.Sync in tests; nil uses it. dirSync func(string) error // journalDirSynced records that this manager has synced the journal's // directory since it started. It is set only after a sync succeeds, so a // failed attempt is retried on the next write, and it starts false so a // file created by an earlier process that died before its directory sync // is covered too. Guarded by journalMu. journalDirSynced bool } var ( ErrDisabled = errors.New("defer manager disabled") ErrCapacity = errors.New("defer capacity exceeded") ErrNotFound = errors.New("defer hold not found") ErrCascadeLimit = errors.New("defer cascade depth exceeded") ) type CascadeLimitError struct { Depth int Limit int ParentDeferID string } func (e *CascadeLimitError) Error() string { return fmt.Sprintf("%s: depth %d exceeds limit %d", ErrCascadeLimit, e.Depth, e.Limit) } func (e *CascadeLimitError) Unwrap() error { return ErrCascadeLimit } func HoldFailureSource(err error) string { if errors.Is(err, ErrCascadeLimit) { return SourceCascadeLimit } return SourceCapacity } // NewManager constructs a bounded held-action manager. func NewManager(cfg Config) *Manager { if cfg.Timeout <= 0 { cfg.Timeout = DefaultTimeoutSeconds * time.Second } if cfg.MaxPending <= 0 { cfg.MaxPending = DefaultMaxPending } if cfg.MaxPendingPerSession <= 0 { cfg.MaxPendingPerSession = DefaultMaxPendingSession } if cfg.MaxPendingBytes <= 0 { cfg.MaxPendingBytes = DefaultMaxPendingBytes } if cfg.MaxCascadeDepth <= 0 { cfg.MaxCascadeDepth = DefaultMaxCascadeDepth } return &Manager{ cfg: cfg, warningf: cfg.Warningf, holds: make(map[string]*HeldAction), sessionHolds: make(map[string][]string), pendingJournal: make(map[string]journalEntry), } } func (m *Manager) warnf(format string, args ...any) { if m != nil && m.warningf != nil { m.warningf(format, args...) } } func (m *Manager) Enabled() bool { return m != nil && m.cfg.Enabled } func (m *Manager) Policy() ResolutionPolicy { if m == nil { return ResolutionPolicy{} } return ResolutionPolicy{ Timeout: m.cfg.Timeout, MaxPending: m.cfg.MaxPending, MaxPendingPerSession: m.cfg.MaxPendingPerSession, MaxPendingBytes: m.cfg.MaxPendingBytes, MaxCascadeDepth: m.cfg.MaxCascadeDepth, } } func (m *Manager) JournalPath() string { if m == nil { return "" } return m.cfg.JournalPath } // Hold stores an action and starts its hard timeout. Capacity rejects the new // action; existing held actions are never evicted open. func (m *Manager) Hold(action HeldAction) error { if !m.Enabled() { return ErrDisabled } if action.DeferID == "" { return fmt.Errorf("defer_id is required") } if action.Resolve == nil { return fmt.Errorf("resolve callback is required") } if action.SizeBytes < 0 { action.SizeBytes = 0 } now := time.Now().UTC() action.Policy = m.Policy() action.Deadline = now.Add(m.cfg.Timeout) action.createdAt = now action.state = StateHeld m.mu.Lock() if _, exists := m.holds[action.DeferID]; exists { m.mu.Unlock() m.journalRejectedHold(action, SourceDuplicateDeferID) return fmt.Errorf("defer hold %q already exists", action.DeferID) } if len(m.holds) >= m.cfg.MaxPending || len(m.sessionHolds[action.Authority.SessionID]) >= m.cfg.MaxPendingPerSession || action.SizeBytes > m.cfg.MaxPendingBytes || m.totalBytes > m.cfg.MaxPendingBytes-action.SizeBytes { m.mu.Unlock() m.journalRejectedHold(action, SourceCapacity) return ErrCapacity } action.Linkage = LinkageSessionPendingAncestor action.CascadeDepth = 1 // An empty session ID is not an identity. Linking the shared "" bucket // would chain unrelated flows into one ancestry: one flow could // cascade-block another and the signed receipts would claim a false // parent/child relationship. Session-less holds stay independent roots // (the per-session capacity cap still bounds the "" bucket). if action.Authority.SessionID != "" { if ids := m.sessionHolds[action.Authority.SessionID]; len(ids) > 0 { parent := m.holds[ids[len(ids)-1]] action.ParentDeferID = parent.DeferID action.CascadeDepth = parent.CascadeDepth + 1 } } if action.CascadeDepth > m.cfg.MaxCascadeDepth { m.mu.Unlock() limitErr := &CascadeLimitError{ Depth: action.CascadeDepth, Limit: m.cfg.MaxCascadeDepth, ParentDeferID: action.ParentDeferID, } m.journalRejectedHold(action, SourceCascadeLimit) return limitErr } journalAction := action stored := action held := &stored // Commit the StateHeld journal entry BEFORE publishing the hold, while still // holding m.mu. A concurrent admission rejection for a related id blocks on // m.mu until this returns, so it cannot append its StateAdmissionRejected // entry ahead of this hold's StateHeld and causally invert the audit trail. // On journal failure nothing is published, so no rollback is needed. if err := m.appendJournal(journalEntryFromHeld(journalAction, StateHeld, "")); err != nil { m.mu.Unlock() return fmt.Errorf("journal defer hold: %w", err) } m.holds[action.DeferID] = held m.sessionHolds[action.Authority.SessionID] = append(m.sessionHolds[action.Authority.SessionID], action.DeferID) m.totalBytes += action.SizeBytes m.mu.Unlock() m.mu.Lock() if live := m.holds[action.DeferID]; live != nil && live.state == StateHeld { live.timer = time.AfterFunc(m.cfg.Timeout, func() { _ = m.Resolve(action.DeferID, "block", SourceTimeout) }) } m.mu.Unlock() return nil } func (m *Manager) journalRejectedHold(action HeldAction, source string) { // Duplicate and capacity rejections happen before Hold derives lineage, so // the action still carries caller-supplied ParentDeferID/CascadeDepth/ // Linkage. Never let a caller inject false ancestry into the audit journal; // only cascade-limit rejections have computed lineage worth preserving. if source != SourceCascadeLimit { action.ParentDeferID = "" action.CascadeDepth = 0 action.Linkage = "" } if err := m.appendJournal(journalEntryFromHeld(action, StateAdmissionRejected, source)); err != nil { m.warnf("pipelock: warning event=deferred_journal_write_failed audit_gap=true source=%s defer_id=%s parent_defer_id=%s cascade_depth=%d: %v\n", source, action.DeferID, action.ParentDeferID, action.CascadeDepth, err) } } // Resolve atomically transitions a held action and invokes its callback once. func (m *Manager) Resolve(deferID, finalDecision, source string) error { _, err := m.resolveApplied(deferID, finalDecision, source) return err } // resolveApplied is Resolve that also reports the terminal decision actually // applied. The requested decision is only a request: the release-time // kill-switch precheck, a Prepare hook (upstream contract gate, send claim) and // a journal failure can each close an allow, and callers that surface the // outcome to an operator must report that one, not the request. func (m *Manager) resolveApplied(deferID, finalDecision, source string) (string, error) { if m == nil { return "", ErrDisabled } if finalDecision == "" { finalDecision = "block" } if source == "" { source = SourceContext } m.mu.Lock() held := m.holds[deferID] if held == nil || held.state != StateHeld { m.mu.Unlock() return "", ErrNotFound } held.state = StateResolving delete(m.holds, deferID) m.removeSessionHoldLocked(held.Authority.SessionID, deferID) m.totalBytes -= held.SizeBytes if held.timer != nil { held.timer.Stop() } m.mu.Unlock() if finalDecision == config.ActionAllow && held.BeforeAllow != nil { release, ok := held.BeforeAllow() if !ok || release == nil { finalDecision = config.ActionBlock source = SourceKillSwitch } else { defer release() } } res := Resolution{ DeferID: held.DeferID, ParentActionID: held.ActionID, ShardIndex: held.ShardIndex, ShardSelected: held.ShardSelected, FinalDecision: finalDecision, ResolutionSource: source, Authority: held.Authority, ParentDeferID: held.ParentDeferID, CascadeDepth: held.CascadeDepth, Linkage: held.Linkage, Policy: held.Policy, Target: held.Target, Method: held.Method, Reason: held.Reason, } if held.Prepare != nil { prepared, finish := held.Prepare(res) if finish != nil { defer finish() } // Prepare may only keep or close the decision; it can never open one. if prepared.FinalDecision == config.ActionAllow && res.FinalDecision != config.ActionAllow { prepared.FinalDecision = config.ActionBlock } res = prepared } if res.FinalDecision == config.ActionAllow { res = m.journalRelease(held, res) } else if err := m.appendJournal(journalEntryFromHeld(*held, resolvedState(res.FinalDecision), res.ResolutionSource)); err != nil { res.FinalDecision = config.ActionBlock res.ResolutionSource = SourceCancel _ = m.appendJournal(journalEntryFromHeld(*held, resolvedState(res.FinalDecision), res.ResolutionSource)) } if res.FinalDecision != config.ActionAllow && res.ResolutionSource != SourceCascade { m.cascadeBlockDescendants([]string{held.DeferID}) } held.Resolve(res) return res.FinalDecision, nil } // journalRelease records an allow in three durable steps so that no point in // it can leave a hold without a resolution receipt: // // 1. a release-pending entry, which keeps the hold pending for recovery; // 2. AfterJournal, which writes the allow resolution receipt; // 3. the terminal resolved_allow entry. // // The call is sent only after step 3, from Resolve. A process that dies after // step 1 leaves a pending hold that restart recovery closes to block with its // own receipt, and the payload was never journaled, so it cannot be re-sent. // Any step that fails closes the allow to block. The block's own receipt is // written through AfterJournal before its terminal entry, so a process that // dies in between leaves a pending hold, never a closed hold with no block // receipt. If that corrective receipt fails, no terminal entry is written at // all and the hold stays pending, so restart recovery closes it with its own // receipt. A failed allow write can still have reached the file (a write whose // sync reports failure), and a terminal entry without its receipt would leave // a closed hold whose evidence nothing will revisit. A recorder that keeps // failing therefore surfaces at the next start, when recovery cannot write its // receipts, instead of as a hold closed with missing or contradicting evidence. func (m *Manager) journalRelease(held *HeldAction, res Resolution) Resolution { closeAllow := func(reason string) { res.FinalDecision = config.ActionBlock res.ResolutionSource = SourceCancel res.Reason = reason // Resolve writes the final receipt, but only after the journal. Write // it here first so the terminal entry never precedes its receipt; // Resolve then reuses it. if held.AfterJournal != nil { if err := held.AfterJournal(res); err != nil { m.warnf("pipelock: warning event=deferred_resolution_receipt_failed audit_gap=true source=%s defer_id=%s: %v\n", res.ResolutionSource, held.DeferID, err) return } } if err := m.appendJournal(journalEntryFromHeld(*held, StateResolvedBlock, res.ResolutionSource)); err != nil { m.warnf("pipelock: warning event=deferred_journal_write_failed audit_gap=true source=%s defer_id=%s: %v\n", res.ResolutionSource, held.DeferID, err) } } if err := m.appendJournal(releasePendingEntry(*held, res.ResolutionSource)); err != nil { closeAllow(res.Reason) return res } if held.AfterJournal != nil { if err := held.AfterJournal(res); err != nil { closeAllow(ReasonReceiptNotWritten) return res } } if err := m.appendJournal(journalEntryFromHeld(*held, StateResolvedAllow, res.ResolutionSource)); err != nil { closeAllow(ReasonReleaseNotJournaled) } return res } // ResolveAll resolves every currently held action with the same final decision. func (m *Manager) ResolveAll(finalDecision, source string) { if m == nil { return } m.mu.Lock() ids := make([]string, 0, len(m.holds)) for id := range m.holds { ids = append(ids, id) } m.mu.Unlock() for _, id := range ids { _ = m.Resolve(id, finalDecision, source) } } func (m *Manager) RecordRestartRecovery(held HeldAction) error { if m == nil { return ErrDisabled } return m.appendJournal(journalEntryFromHeld(held, StateResolvedBlock, SourceRestartRecovery)) } func RecordRestartRecoveryJournal(path string, held HeldAction) error { if path == "" { return nil } m := &Manager{cfg: Config{JournalPath: path}} return m.appendJournal(journalEntryFromHeld(held, StateResolvedBlock, SourceRestartRecovery)) } func (m *Manager) Snapshot() []HeldAction { if m == nil { return nil } m.mu.Lock() defer m.mu.Unlock() out := make([]HeldAction, 0, len(m.holds)) for _, held := range m.holds { cp := *held cp.timer = nil cp.Resolve = nil cp.Payload = append([]byte(nil), held.Payload...) out = append(out, cp) } return out } // HeldCount reports how many actions are currently held, without copying them. func (m *Manager) HeldCount() int { if m == nil { return 0 } m.mu.Lock() defer m.mu.Unlock() return len(m.holds) } func (m *Manager) Held(deferID string) (HeldAction, bool) { held, err := m.snapshotOne(deferID) if err != nil { return HeldAction{}, false } return held, true } func (m *Manager) ResolveToolInventory(sessionID, finalDecision string) { if m == nil { return } for _, held := range m.Snapshot() { if held.Authority.SessionID == sessionID && held.RulePolicy.AllowOn.ToolInventoryBaseline { _ = m.Resolve(held.DeferID, finalDecision, SourceToolInventory) } } } // ResolveApproval resolves a hold from an explicit approval decision. A // misconfigured positive approval cannot open the action; it resolves closed. // The resolution is labeled SourceApproval (the interactive HITL path); the // operator admin surface uses ResolveApprovalResult with SourceOperator. func (m *Manager) ResolveApproval(deferID, finalDecision string) error { _, err := m.ResolveApprovalResult(deferID, finalDecision, SourceApproval) return err } // ResolveApprovalResult resolves a hold from an explicit approval decision and // reports the terminal decision that was actually applied. A positive approval // that the held rule does not permit resolves CLOSED (block), and the returned // decision reflects that outcome - so an operator surface can report the honest // result instead of assuming "approved". source labels the resolution // provenance in the receipt/journal (SourceApproval for the interactive HITL // path, SourceOperator for the admin API). // // The returned error mirrors Resolve: ErrNotFound when the hold is unknown or // was already resolved (by timeout, cascade, or another resolver) between the // snapshot and the resolve. On any non-nil error nothing was resolved and the // returned decision is empty. func (m *Manager) ResolveApprovalResult(deferID, finalDecision, source string) (string, error) { held, err := m.snapshotOne(deferID) if err != nil { return "", err } decision := approvalDecision(held.RulePolicy, finalDecision) // Report what was applied, not what was asked: a kill switch that activated // before release, or a release gate that refused, closes an approved hold. return m.resolveApplied(deferID, decision, source) } // approvalDecision maps an approval input onto the terminal decision, enforcing // the held rule's affirmative policy: a positive "allow" only opens the hold // when the rule permits approval, and a step-up only escalates when the rule // permits it; everything else - including a not-permitted allow - resolves // closed with block. Single source of truth for the approval gate, shared by // ResolveApproval and ResolveApprovalResult. func approvalDecision(rp config.DeferResolutionPolicy, finalDecision string) string { switch finalDecision { case config.ActionAllow: if rp.AllowOn.Approval { return config.ActionAllow } case config.ActionAsk, config.ActionStepUp: if rp.StepUpOn.ApprovalRequestsHuman { return config.ActionAsk } } return config.ActionBlock } type ReloadEvaluator func(HeldAction) (string, error) // ResolvePolicyReload re-evaluates held actions against a caller-supplied fresh // policy view. Only affirmative policy_permits may allow; block/error resolves // closed; defer remains held until another resolver or the hard deadline. func (m *Manager) ResolvePolicyReload(evaluate ReloadEvaluator) { if m == nil || evaluate == nil { return } for _, held := range m.Snapshot() { decision, err := evaluate(held) if err != nil { _ = m.Resolve(held.DeferID, config.ActionBlock, SourcePolicyReload) continue } switch decision { case config.ActionAllow: if held.RulePolicy.AllowOn.PolicyPermits { _ = m.Resolve(held.DeferID, config.ActionAllow, SourcePolicyReload) } case config.ActionBlock: _ = m.Resolve(held.DeferID, config.ActionBlock, SourcePolicyReload) case config.ActionDefer: continue default: _ = m.Resolve(held.DeferID, config.ActionBlock, SourcePolicyReload) } } } func (m *Manager) snapshotOne(deferID string) (HeldAction, error) { if m == nil { return HeldAction{}, ErrDisabled } m.mu.Lock() defer m.mu.Unlock() held := m.holds[deferID] if held == nil || held.state != StateHeld { return HeldAction{}, ErrNotFound } cp := *held cp.timer = nil cp.Resolve = nil cp.Payload = append([]byte(nil), held.Payload...) return cp, nil } func (m *Manager) removeSessionHoldLocked(sessionID, deferID string) { ids := m.sessionHolds[sessionID] for i, id := range ids { if id == deferID { ids = append(ids[:i], ids[i+1:]...) break } } if len(ids) == 0 { delete(m.sessionHolds, sessionID) return } m.sessionHolds[sessionID] = ids } func (m *Manager) cascadeBlockDescendants(parentIDs []string) { frontier := append([]string(nil), parentIDs...) for len(frontier) > 0 { m.mu.Lock() children := make([]string, 0) for id, held := range m.holds { for _, parentID := range frontier { if held.ParentDeferID == parentID { children = append(children, id) break } } } m.mu.Unlock() if len(children) == 0 { return } for _, id := range children { _ = m.Resolve(id, config.ActionBlock, SourceCascade) } frontier = children } } func resolvedState(finalDecision string) string { switch finalDecision { case "allow": return StateResolvedAllow case "ask": return StateResolvedStepUp default: return StateResolvedBlock } } type journalEntry struct { DeferID string `json:"defer_id"` ActionID string `json:"action_id"` ShardIndex int `json:"shard_index,omitempty"` ShardSelected bool `json:"shard_selected,omitempty"` State string `json:"state"` // ReleasePending marks a deferred_held entry written when an allow began // releasing the hold. It is a flag on the held state rather than a state // of its own so that a binary that predates it, whose journal reader // rejects unknown states, still reads the hold as pending and recovers it // to block. Only deferred_held entries may carry it. ReleasePending bool `json:"release_pending,omitempty"` Source string `json:"source,omitempty"` Target string `json:"target,omitempty"` Surface string `json:"surface,omitempty"` Method string `json:"method,omitempty"` Reason string `json:"reason,omitempty"` Authority AuthoritySnapshot `json:"authority"` Policy ResolutionPolicy `json:"policy"` RulePolicy config.DeferResolutionPolicy `json:"rule_policy"` ParentDeferID string `json:"parent_defer_id,omitempty"` CascadeDepth int `json:"cascade_depth,omitempty"` Linkage string `json:"linkage,omitempty"` Deadline time.Time `json:"deadline,omitempty"` Timestamp time.Time `json:"timestamp"` SizeBytes int `json:"size_bytes,omitempty"` } func journalEntryFromHeld(held HeldAction, state, source string) journalEntry { return journalEntry{ DeferID: held.DeferID, ActionID: held.ActionID, ShardIndex: held.ShardIndex, ShardSelected: held.ShardSelected, State: state, Source: source, Target: held.Target, Surface: held.Surface, Method: held.Method, Reason: held.Reason, Authority: held.Authority, Policy: held.Policy, RulePolicy: held.RulePolicy, ParentDeferID: held.ParentDeferID, CascadeDepth: held.CascadeDepth, Linkage: held.Linkage, Deadline: held.Deadline, Timestamp: time.Now().UTC(), SizeBytes: held.SizeBytes, } } // releasePendingEntry is the journal entry for a hold whose allow has begun // releasing but is not yet final. See journalEntry.ReleasePending. func releasePendingEntry(held HeldAction, source string) journalEntry { entry := journalEntryFromHeld(held, StateHeld, source) entry.ReleasePending = true return entry } // appendJournal writes one entry and returns only after it is on stable // storage. Recovery trusts the journal's ordering against the receipt chain, // whose own writes for these resolutions are fsync-confirmed, so an entry // still in the page cache could be lost while a later receipt survives. func (m *Manager) appendJournal(entry journalEntry) error { if m == nil || m.cfg.JournalPath == "" { return nil } m.journalMu.Lock() defer m.journalMu.Unlock() write := func() error { path := filepath.Clean(m.cfg.JournalPath) dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o750); err != nil { return err } f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return err } defer func() { _ = f.Close() }() data, err := json.Marshal(entry) if err != nil { return err } if _, err := f.Write(append(data, '\n')); err != nil { return err } if err := m.syncFile(f); err != nil { return fmt.Errorf("sync defer journal: %w", err) } if !m.journalDirSynced { if err := m.syncDir(dir); err != nil { return fmt.Errorf("sync defer journal directory: %w", err) } m.journalDirSynced = true } return nil } if m.cfg.JournalWriteGuard != nil { return m.cfg.JournalWriteGuard(write) } return write() } // syncDir makes the journal's directory entry durable. func (m *Manager) syncDir(dir string) error { if m.dirSync != nil { return m.dirSync(dir) } return directorysync.Sync(dir) } // syncFile makes a journal write durable. func (m *Manager) syncFile(f *os.File) error { if m.fileSync != nil { return m.fileSync(f) } return f.Sync() } // PendingJournal returns held actions from a prior process that lack a terminal // journal entry. Callers should emit restart_recovery block receipts for each. func PendingJournal(path string) ([]HeldAction, error) { if path == "" { return nil, nil } f, err := os.Open(filepath.Clean(path)) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, nil } return nil, err } defer func() { _ = f.Close() }() pending := map[string]journalEntry{} terminal := map[string]struct{}{} scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) for scanner.Scan() { var entry journalEntry if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { return nil, fmt.Errorf("parse defer journal: %w", err) } if entry.ReleasePending && entry.State != StateHeld { return nil, fmt.Errorf("defer journal integrity: release_pending on state %q for defer_id %q", entry.State, entry.DeferID) } switch entry.State { case StateHeld: if _, seen := terminal[entry.DeferID]; seen { return nil, fmt.Errorf("defer journal integrity: held entry after terminal state for defer_id %q", entry.DeferID) } if _, seen := pending[entry.DeferID]; entry.ReleasePending && !seen { return nil, fmt.Errorf("defer journal integrity: release_pending without a prior hold for defer_id %q", entry.DeferID) } // A release-pending entry stays pending: its allow never reached // the journal as final, so the call was not sent and recovery // closes it to block. pending[entry.DeferID] = entry case StateResolvedAllow, StateResolvedBlock, StateResolvedStepUp: delete(pending, entry.DeferID) terminal[entry.DeferID] = struct{}{} case StateAdmissionRejected: // Audit-only marker for an action that was never held. It must not // touch hold lifecycle: for a duplicate defer_id it shares a live // hold's id, so resolving/terminalizing it here would drop that // live hold from recovery. default: return nil, fmt.Errorf("defer journal integrity: unknown state %q for defer_id %q", entry.State, entry.DeferID) } } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("scan defer journal: %w", err) } out := make([]HeldAction, 0, len(pending)) for _, entry := range pending { out = append(out, HeldAction{ DeferID: entry.DeferID, ActionID: entry.ActionID, ShardIndex: entry.ShardIndex, ShardSelected: entry.ShardSelected, Target: entry.Target, Reason: entry.Reason, Surface: entry.Surface, Method: entry.Method, SizeBytes: entry.SizeBytes, Policy: entry.Policy, RulePolicy: entry.RulePolicy, Authority: entry.Authority, ParentDeferID: entry.ParentDeferID, CascadeDepth: entry.CascadeDepth, Linkage: entry.Linkage, Deadline: entry.Deadline, }) } return out, nil }