[ { "id": "cosmos-mainnet-gaia-v27.2.0-upgrade-2026-04-15", "date": "2026-04-15T14:10:00Z", "incident_type": "upgrade", "chain": "Cosmos", "network": "mainnet", "node_type": ["validator"], "client": "gaia v27.2.0", "note": "Cosmos Hub successfully upgraded to Gaia v27.2.0 on mainnet following Proposal #1028. Cumulo infrastructure performed the upgrade across all validator and node systems at the scheduled block height 30675400 without any incidents. Post-upgrade validation confirms full synchronization and normal participation in consensus.", "url": "https://github.com/cosmos/gaia/releases/tag/v27.2.0" }, { "id": "genlayer-bradbury-telemetry-monitoring-deployment-2026-04-14", "date": "2026-04-14T00:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "GenLayer", "network": "testnet (bradbury)", "node_type": ["validator"], "client": "genlayer v0.5.8", "note": "Cumulo deployed and configured centralized telemetry monitoring for its GenLayer Bradbury validator using Alloy with Prometheus remote_write and Loki integration to Grafana Cloud. Initial authentication issues (401 invalid token) were identified and resolved by correcting environment variables and credentials. Metrics and logs ingestion are now fully operational, and the validator is correctly reporting as active in the GenLayer monitoring dashboard.", "url": "https://docs.genlayer.com/validators" }, { "id": "dymension-mainnet-governance-proposal-39-2026-04-15", "date": "2026-04-15T00:00:00Z", "incident_type": "governance_proposal", "status": "active", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #39 on Dymension mainnet: Update USDT send rate limit from 15% to 100%. The proposal updates the IBC rate limit on channel-3 for USDT (IBC denom B72B5B3F7AD44...) increasing max_percent_recv from 15% to 100% over a 24-hour window. Currently passing with 99.98% YES votes. Cumulo voted YES. Voting ends on 2026-04-19 13:59:24 UTC.", "url": "https://dym.fyi/proposal/39", "vote": "https://dym.fyi/tx/EFFA6DB76DC42E82C04031B786C6972778862C8D0A892316590B35071DC8ED92" }, { "id": "celestia-app-v8.0.2-mocha-node-v0.30.2-mocha-upgrade-2026-04-14", "date": "2026-04-14T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus", "bridge"], "client": ["celestia-app v8.0.2-mocha", "celestia-node v0.30.2-mocha"], "note": "Cumulo upgraded its Celestia Mocha testnet nodes to celestia-app v8.0.2-mocha and celestia-node v0.30.2-mocha. The celestia-app release fixes a data race condition in the signing context GetSigners method, adds Grafana latency and failure metrics dashboards, updates the v3 multiplexer binary to version 3.12.0, and updates Ubuntu OS version requirements documentation. The celestia-node patch improves P2P stability by removing broken Mocha bootstrapper nodes that were causing peer discovery issues, and adjusts the log level in shrex-client to reduce diagnostic noise. Upgrade applied across validator, RPC, consensus, and bridge nodes. All services verified and fully operational post-upgrade.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v8.0.2-mocha" }, { "id": "starknet-mainnet-testnet-update-juno-v0.15.22-2026-04-13", "date": "2026-04-13T00:00:00Z", "incident_type": "update", "status": "completed", "chain": "Starknet", "network": "mainnet/testnet", "node_type": ["validator", "rpc", "backup"], "client": "Juno v0.15.22", "note": "Upgrade to Juno v0.15.22 applied across all Starknet nodes (mainnet and testnet). The service was stopped via systemd, the new binary was downloaded from the official NethermindEth GitHub release, and all validator, RPC, and backup nodes were restarted and verified. All services confirmed fully synchronized and operational post-upgrade.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.22" }, { "id": "celestia-mocha-db-migration-pebbledb-2026-04-12", "date": "2026-04-12T00:00:00Z", "incident_type": "infrastructure_migration", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v8", "note": "Cumulo migrated its Celestia Mocha testnet node database from LevelDB (goleveldb) to PebbleDB as part of the celestia-app v8 rollout. PebbleDB offers lower disk I/O, faster block synchronization, and reduced RPC response times compared to goleveldb, making it the recommended backend going forward. The node was resynced from a PebbleDB-format snapshot and config.toml was updated to set db_backend = \"pebbledb\". The node uses Horcrux for remote signing. Post-migration the node is fully operational and synchronized.", "url": "https://github.com/celestiaorg/celestia-app/tree/main/tools/migrate-db" }, { "id": "quicksilver-mainnet-governance-proposal-64-2026-04-10", "date": "2026-04-10T00:00:00Z", "incident_type": "governance_proposal", "status": "in_progress", "chain": "Quicksilver", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #64 on Quicksilver mainnet to upgrade to v1.10.2. Cumulo voted YES on this proposal. The upgrade targets block height 17775700 (~2026-04-13 16:00 UTC) and includes offboarding Omniflix and Stargaze zones, along with a fix for HandleFailedUndelegate IBC acknowledgement blocking ICA connections.", "url": "https://github.com/quicksilver-zone/quicksilver/releases/tag/v1.10.2", "vote": "https://explorer.quicksilver.zone/transactions/D9F5A41DFCCC1E13B880C7260117A69E0A0D44C380BC99D3CC58782BE2262E28" }, { "id": "starknet-mainnet-governance-proposal-v0142-2026-04-10", "date": "2026-04-10T00:00:00Z", "incident_type": "governance_proposal", "status": "in_progress", "chain": "Starknet", "network": "mainnet", "node_type": ["validator"], "note": "Cumulo voted YES. Onchain vote for the Starknet v0.14.2 protocol upgrade is live and closes on April 16 at 16:00 UTC. The upgrade includes four key changes: In-Protocol Proof Verification (SNIP-36) enabling S-Two proof verification; Revised Storage Costs (SNIP-37) aligning pricing with actual resource usage; StarkGate Token Upgrade (SNIP-13) with indexed ERC-20 events and USDC becoming USDC.e; and Decentralization Prep (SNIP-33) for future decentralized validation. This upgrade includes breaking changes affecting block hash calculations, storage pricing, and ERC-20 event formats. Voting ends 2026-04-16 16:00 UTC.", "url": "https://governance.starknet.io/#/sn:0x009fedaf0d7a480d21a27683b0965c0f8ded35b3f1cac39827a25a06a8a682a4/proposal/11", "vote": "https://voyager.online/tx/0x1ca8c9a48441be4b6526d4962818cf429c167388f03884d6cc77d926caf2ee0" }, { "id": "cosmos-mainnet-governance-proposal-1029-2026-04-09", "date": "2026-04-09T10:02:00Z", "incident_type": "governance_proposal", "status": "in_progress", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #1029 opened on Cosmos Hub mainnet: Acquisition and Merger of Osmosis into the Cosmos Hub. The Cosmos Hub and broader ecosystem are entering a new phase, where economic gravity and sustainability require deeper integration. Voting ends on 2026-04-16 10:02 UTC.", "url": "https://cumulo.pro/services/cosmos/governance#proposal-1029", "vote": "https://cumulo.pro/services/cosmos/search?q=6BA582ECF4D47558721769FC5D65F81F5E66F13C0F1E1139F02B1D568A032296" }, { "id": "celestia-node-v0.30.1-mocha-upgrade-2026-04-08", "date": "2026-04-08T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["bridge"], "client": "celestia-node v0.30.1-mocha", "note": "Cumulo upgraded its Celestia Mocha testnet node to celestia-node v0.30.1-mocha. This release adds full support for celestia-app v8 and is required once the v8 activation height takes place. Key improvements include TLS support for JSON-RPC (users are encouraged to enable TLS to secure endpoint connections) and OpenTelemetry export for all Prometheus metrics including libp2p, removing the need for a separate Prometheus instance.", "url": "https://github.com/celestiaorg/celestia-node/releases/tag/v0.30.1-mocha" }, { "id": "cosmos-mainnet-governance-proposal-1028-2026-04-07", "date": "2026-04-07T18:50:00Z", "incident_type": "governance_proposal", "status": "ended", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #1028 (Gaia v27.2.0 Upgrade) on Cosmos mainnet. Cumulo voted YES. Voting period ended on 2026-04-14 18:54 UTC.", "url": "https://cumulo.pro/services/cosmos/governance#proposal-1028" }, { "id": "cosmos-gaia-v27.2.0-rc0-upgrade-2026-04-07", "date": "2026-04-07T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "provider testnet", "node_type": ["validator", "rpc", "consensus"], "client": "gaia v27.2.0-rc0", "note": "Cosmos provider testnet successfully upgraded to Gaia v27.2.0-rc0 at block height 16776100 following governance Proposal 299. Cumulo infrastructure performed the upgrade, including binary update and validator restart at the designated height. The upgrade name included the full version as required (v27.2.0). Post-upgrade checks confirm validators are operational, synchronized, and actively participating in consensus without issues.", "url": "https://github.com/cosmos/testnets/tree/master/provider#v2720-rc0-upgrade" }, { "id": "celestia-app-v8.0.1-mocha-upgrade-2026-04-07", "date": "2026-04-07T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v8.0.1-mocha", "note": "Celestia Mocha testnet upgrade to celestia-app v8.0.1-mocha successfully applied on Cumulo infrastructure. The upgrade includes critical changes such as the new minimum OS requirement (Ubuntu 24.04+), and a redesigned forwarding address derivation mechanism binding token identity (destDomain, destRecipient, tokenId) to unique forwarding addresses. Cumulo validators have upgraded and signaled readiness for upgrade (signal 8), actively participating in the coordination process. Post-upgrade validation confirms nodes are operational and aligned with the new protocol requirements.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v8.0.1-mocha" }, { "id": "starknet-testnet-validator-migration-juno-v0.15.20-2026-04-05", "date": "2026-04-05T00:00:00Z", "incident_type": "infrastructure_migration", "status": "completed", "chain": "Starknet", "network": "testnet", "node_type": ["validator", "attestation"], "client": "juno v0.15.20", "note": "Cumulo migrated its Starknet testnet validator node to a new server infrastructure. The validator services were successfully restored, including the attestation system. The Juno node (v0.15.20) is fully synchronized and operational, with the attestor actively running and successfully validating epochs. Post-migration checks confirm stable performance and correct participation in the network.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.20" }, { "id": "genlayer-testnet-bradbury-upgrade-v0.5.8-2026-04-03", "date": "2026-04-03T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (bradbury)", "node_type": ["validator"], "client": "genlayer v0.5.8", "note": "Cumulo upgraded its GenLayer validator node on the Bradbury testnet to version v0.5.8. This release introduces a gen_syncing RPC method for sync progress monitoring, bounded sync wait with configurable timeout (default 20s), and per-IP rate limiting on RPC endpoints. Bug fixes include a Docker AVX-512 fix moving WASM precompilation to container startup, sync resilience improvements with recursive dependency resolution, sync pipeline hiccup retry, prime recovery after chain reorganizations, RPC block range fix, standardized EIP-1474 error codes, and transaction lookup retry for RPC indexing lag.", "url": "https://docs.genlayer.com/validators/changelog#v058" }, { "id": "genlayer-asimov-testnet-v0.5.8-upgrade-2026-04-03", "date": "2026-04-03T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.8", "note": "Cumulo upgraded its GenLayer validator node on the Asimov testnet to version v0.5.8. This release introduces a new gen_syncing RPC endpoint for improved sync monitoring, bounded sync wait for faster RPC responses, and per-IP rate limiting for enhanced endpoint protection. Additional improvements include enhanced sync resilience, retry mechanisms for transient RPC failures, validator recovery after reorgs, and standardized EIP-1474 error codes. The upgrade was applied successfully and the node resumed normal operation.", "url": "https://docs.genlayer.com/validators/changelog#v058" }, { "id": "story-aeneid-testnet-v1.6.1-upgrade-2026-04-01", "date": "2026-04-01T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Story", "network": "Aeneid Testnet", "node_type": ["validator", "rpc","backup"], "client": "storyd v1.6.1", "note": "Cumulo successfully upgraded its Story Aeneid Testnet validator and RPC nodes to v1.6.1 at the scheduled upgrade height 16,332,000. This mandatory upgrade introduces Distributed Key Generation (DKG) and Confidential Data Rails (CDR), enabling on-chain confidential data management. The upgrade was executed using Cosmovisor with the correct upgrade name (v1.6.0), ensuring automatic binary swap at halt height. Nodes resumed normal operation after the upgrade without issues.", "url": "https://github.com/piplabs/story/releases/tag/v1.6.1" }, { "id": "cosmoshub-mainnet-gaia-v27-1-0-upgrade-2026-04-01", "date": "2026-04-01T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "gaia v27.1.0", "note": "Cumulo upgraded its Cosmos Hub mainnet validator and RPC nodes to Gaia v27.1.0 following the passing of governance proposal #1026. This state-breaking upgrade bumps wasmd from v0.60.2 to v0.60.5, wasmvm from v2.2.4 to v2.3.2, and Golang from v1.24 to v1.25. The binary was downloaded from the official release, deployed, and the service restarted via systemd. Both nodes resumed normal operation after the upgrade.", "url": "https://github.com/cosmos/gaia/releases/tag/v27.1.0" }, { "id": "story-aeneid-testnet-validator-backup-deployment-2026-03-31", "date": "2026-03-31T14:30:00Z", "incident_type": "deployment", "status": "completed", "chain": "Story", "network": "Aeneid Testnet", "node_type": ["validator", "rpc"], "client": "geth v1.2.1 / storyd v1.5.2", "note": "Deployment of a new Story Aeneid Testnet backup validator and RPC node operated by Cumulo. This backup node enhances network reliability and validator redundancy. Infrastructure setup follows Cumulo’s standard practices, including secure key management, systemd-based services, monitoring, and operational documentation to ensure resilient, transparent, and maintainable operations.", "url": "https://github.com/Cumulo-pro/cumulo-story-aeneid-infra/" }, { "id": "xrplevm-mainnet-governance-proposal-27-2026-03-25", "date": "2026-03-25T18:06:00Z", "incident_type": "governance_proposal", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #27 opened on XRPL EVM mainnet to remove the Informal Systems and Grove validators from the validator set. Voting period ends on 2026-04-08 17:05:46 UTC.", "url": "https://governance.xrplevm.org/xrplevm/proposals/27" }, { "id": "xrplevm-mainnet-governance-proposal-28-2026-03-25", "date": "2026-03-25T18:37:00Z", "incident_type": "governance_proposal", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #28 opened on XRPL EVM mainnet to remove the Informal Systems and Grove validators from the validator set. Voting period ends on 2026-04-08 17:37:15 UTC.", "url": "https://governance.xrplevm.org/xrplevm/proposals/28" }, { "id": "xrplevm-testnet-governance-proposal-41-2026-03-25", "date": "2026-03-25T18:13:00Z", "incident_type": "governance_proposal", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["validator"], "note": "Governance Proposal #41 opened on XRPL EVM testnet to remove the Grove, Cosmostation, MOAI, xToadz, Informal Systems, and Nexus validators from the testnet validator set. Voting period ends on 2026-04-08 17:12:23 UTC.", "url": "https://governance.testnet.xrplevm.org/xrplevm/proposals/41" }, { "id": "xrplevm-testnet-governance-proposal-42-2026-03-25", "date": "2026-03-25T18:38:00Z", "incident_type": "governance_proposal", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["validator"], "note": "Governance Proposal #42 opened on XRPL EVM testnet to remove the Grove, Cosmostation, MOAI, xToadz, Informal Systems, and Nexus validators from the testnet validator set. Voting period ends on 2026-04-08 17:38:05 UTC.", "url": "https://governance.testnet.xrplevm.org/xrplevm/proposals/42" }, { "id": "seda-testnet-v1.0.8-rc.1-upgrade-2026-03-25", "date": "2026-03-25T20:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "SEDA", "network": "testnet", "node_type": ["validator", "rpc"], "client": "seda-chain v1.0.8-rc.1", "note": "Cumulo successfully upgraded its SEDA testnet validator and RPC nodes to v1.0.8-rc.1 (upgrade name: v1.0.8) at block height 8,001,175. This consensus-breaking upgrade updates the Cosmos SDK from v0.50 to v0.53, IBC from v8 to v10, and Tally VM from v2.6.0 to v2.6.1. The binary was built from source, installed via Cosmovisor under the correct upgrade name, and both nodes resumed normal operation after the halt.", "url": "https://github.com/sedaprotocol/seda-chain/releases/tag/v1.0.8-rc.1" }, { "id": "starknet-mainnet-juno-v0.15.20-upgrade-2026-03-24", "date": "2026-03-24T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Starknet", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "juno v0.15.20", "note": "Cumulo upgraded its Starknet mainnet Juno node to v0.15.20. The binary was downloaded from the official NethermindEth release, deployed to /usr/local/bin, and the service restarted via systemd. Normal node operation verified after restart.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.20" }, { "id": "cosmoshub-mainnet-governance-proposal-1027-vote-yes-2026-03-24", "date": "2026-03-24T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator"], "note": "Cumulo voted YES on Cosmos Hub mainnet governance proposal #1027 to recover IBC Client 07-tendermint-620 connecting Cosmos Hub to the EVMOS network. The recovery replaces the state of the expired client with the state of a newly created substitute client 07-tendermint-1470. The client expired due to a recent Ethermint vulnerability that caused the EVMOS network to be down for several days, locking user ATOM tokens in the IBC channel.", "url": "https://cosmos.explorers.guru/proposal/1027" }, { "id": "cosmoshub-mainnet-governance-proposal-1026-vote-yes-2026-03-24", "date": "2026-03-24T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator"], "client": "gaia v27.1.0", "note": "Cumulo voted YES on Cosmos Hub mainnet governance proposal #1026 to approve the upgrade to Gaia v27.1.0. This state-breaking upgrade bumps wasmd from v0.60.2 to v0.60.5, wasmvm from v2.2.4 to v2.3.2, and Golang from v1.24 to v1.25. If the proposal passes (7-day voting period), validators will be required to update the Cosmos Hub binary at the specified upgrade height.", "url": "https://cosmos.explorers.guru/proposal/1026" }, { "id": "genlayer-node-v0.5.7-upgrade-asimov-testnet-2026-03-23", "date": "2026-03-23T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.7", "note": "Cumulo upgraded its GenLayer validator node on the Asimov testnet to version v0.5.7. This release introduces LLM Greyboxing, enabling deterministic model ordering with configurable fallback chains (e.g., DeepSeek → Qwen → Claude) to improve consensus convergence. Bug fixes include idle activator rotation, view call cleanup, sync pipeline auto-restart after transient RPC failures, cleaner shutdown logs, and a FindAcceptanceBlock range limit to prevent query failures on nodes with large block gaps.", "url": "https://docs.genlayer.com/validators/changelog#v057" }, { "id": "axone-testnet-v14-0-0-smart-contract-readiness-upgrade-2026-03-22", "date": "2026-03-22T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Axone", "network": "testnet", "node_type": ["validator", "rpc"], "client": "axone v14.0.0", "note": "Cumulo applied the Axone testnet upgrade to v14.0.0 (Smart Contract Readiness Upgrade). This mandatory governance-passed upgrade aligns the network with the latest Cosmos SDK, addresses critical bugs, and introduces vesting logic. It serves as a prerequisite for enabling smart contract deployment and transitioning to Axone's application layer.", "url": "https://axone.discourse.group/t/proposal-testnet-cosmos-sdk-upgrade-smart-contract-readiness/59" }, { "id": "warden-mainnet-v1.0.0-upgrade-execution-2026-03-19", "date": "2026-03-19T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Warden", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "wardend v1.0.0", "note": "Cumulo upgraded its Warden mainnet validator infrastructure to wardend v1.0.0 at the scheduled network halt on block height 5418000. The chain halted and resumed once validators completed the upgrade. Cumulo prepared the binaries in advance, performed the required restart at the target height, and verified node synchronization and proposal activity after restart.", "url": "https://github.com/warden-protocol/wardenprotocol/releases/tag/v1.0.0" }, { "id": "cosmoshub-provider-testnet-gaia-v27-1-0-rc0-upgrade-2026-03-17", "date": "2026-03-17T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "testnet (provider)", "node_type": ["validator"], "client": "gaia v27.1.0-rc0", "note": "Cumulo participated in the Cosmos Hub provider testnet upgrade to Gaia v27.1.0-rc0 (upgrade name: v27.1.0) at block height 16,463,900 (~14:33 CET). To qualify for participation tracking, Cumulo signed within the first five blocks after the upgrade. Note: fruit validators remained offline for 10 minutes post-upgrade height to give other validators a chance to come online before the chain resumed.", "url": "https://github.com/cosmos/testnets/tree/master/provider#v2710-rc0-upgrade" }, { "id": "seda-testnet-governance-proposal-9-vote-yes-2026-03-17", "date": "2026-03-17T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "SEDA", "network": "testnet", "node_type": ["validator"], "client": "seda-chain v1.0.8-rc.1", "note": "Cumulo voted YES on SEDA Testnet governance proposal #9 to approve the upgrade to v1.0.8-rc.1. The upgrade, scheduled at block height 8,001,175 (~March 24th, 2026 at 15:00 CET), updates the Cosmos SDK from v0.50 to v0.53, IBC from v8 to v10, and Tally VM from v2.6.0 to v2.6.1 (bug fix). As a consensus-breaking upgrade, Cumulo will prepare the new binary ahead of the upgrade height and monitor the transition to mainnet in the coming weeks.", "url": "https://testnet.keplr.app/chains/seda-(testnet)/proposals/9" }, { "id": "genlayer-node-v0.5.6-upgrade-bradbury-testnet-2026-03-16", "date": "2026-03-16T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (bradbury)", "node_type": ["validator"], "client": "genlayer v0.5.6", "note": "Cumulo upgraded its GenLayer validator node on the Bradbury testnet to version v0.5.6 following the release announcement. This update introduces priority-based admission control for transaction processing, improving transaction ordering and throughput under load. Cumulo applied the upgrade as part of its ongoing maintenance of GenLayer validator infrastructure and verified normal node operation after restart.", "url": "https://docs.genlayer.com/validators/changelog#v056" }, { "id": "celestia-app-v7.0.3-mocha-security-upgrade-2026-03-16", "date": "2026-03-16T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v7.0.3-mocha", "note": "Cumulo upgraded its Celestia Mocha validator infrastructure to celestia-app v7.0.3-mocha following the release announcement. This patch release includes a security fix mitigating a potential OOM denial-of-service vector, along with stability improvements addressing gRPC port conflict panics and nil pointer dereferences in FinalizeBlock. Cumulo applied the upgrade to maintain secure and stable validator, RPC, and consensus operations on Mocha.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v7.0.3-mocha" }, { "id": "genlayer-node-v0.5.6-upgrade-asimov-testnet-2026-03-16", "date": "2026-03-16T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.6", "note": "Cumulo upgraded its GenLayer validator node on the Asimov testnet to version v0.5.6 following the release announcement. This update introduces priority-based admission control for transaction processing, improving ordering and throughput under load. Cumulo applied the upgrade as part of its routine maintenance of GenLayer validator infrastructure and verified normal node operation after restart.", "url": "https://docs.genlayer.com/validators/changelog#v056" }, { "id": "genlayer-bradbury-phase1-validator-deployment-2026-03-14", "date": "2026-03-14T18:55:00Z", "incident_type": "infrastructure_deployment", "status": "completed", "chain": "GenLayer", "network": "testnet (bradbury phase1)", "node_type": ["validator"], "client": "genlayer v0.5.5", "note": "Cumulo deployed and initialized a new validator node on GenLayer Bradbury Phase 1 testnet. The deployment included fresh validator creation for Bradbury using the existing owner wallet, a new operator wallet, GenVM setup, WebDriver configuration, Heurist LLM provider integration, and full validator synchronization. After completing setup and health checks, the node reached normal validating status with health endpoint reporting validating=up and zksync-connectivity=up.", "url": "https://docs.genlayer.com/validators/setup-guide#testnet-bradbury" }, { "id": "genlayer-node-v0.5.5-upgrade-execution-2026-03-14", "date": "2026-03-14T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.5", "note": "Cumulo upgraded its GenLayer validator node to version v0.5.5 following the release announcement. This upgrade introduces enhanced telemetry and observability improvements, including network labels in health endpoints and metrics, as well as improved reliability for transaction processing and finalization queues. The release also resolves several issues related to cache miss recovery, stuck nonces, shutdown stability, and GenVM startup on macOS. Cumulo applied the new version and verified normal validator operation after restart.", "url": "https://docs.genlayer.com/validators/changelog#v055" }, { "id": "story-geth-v1.2.1-mainnet-aeneid-security-upgrade-2026-03-14", "date": "2026-03-14T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Story", "network": "mainnet / testnet (Aeneid)", "node_type": ["validator", "rpc", "backup"], "client": "story-geth v1.2.1", "note": "Cumulo upgraded its Story execution-layer infrastructure to Story-Geth v1.2.1 on both Mainnet and Aeneid Testnet. This required release merges upstream security fixes addressing CVE-2026-26314, CVE-2026-26315, and two additional p2p vulnerabilities. The upgrade also adds DNS resolution support for bootnodes, allowing configuration with either domain names or IP addresses. Cumulo applied the update as part of its ongoing security and reliability maintenance for Story validator and RPC operations.", "url": "https://github.com/piplabs/story-geth/releases/tag/v1.2.1" }, { "id": "celestia-node-v0.29.3-mocha-upgrade-execution-2026-03-12", "date": "2026-03-12T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["bridge", "consensus"], "client": "celestia-node v0.29.3-mocha", "note": "Cumulo upgraded its Celestia Mocha node infrastructure to celestia-node v0.29.3-mocha. This release includes fixes for light node startup failures caused by syncer tail height overshooting the pruning window, a bridge DASer gossip race fix, core endpoint verification, go-header improvements with parallel header fetching, a new delegation rewards API, and new cel-shed tools. Cumulo applied the upgrade to maintain compatibility and improve bridge and consensus node stability on Mocha.", "url": "https://github.com/celestiaorg/celestia-node/releases/tag/v0.29.3-mocha" }, { "id": "cosmoshub-gaia-v27-0-0-mainnet-upgrade-2026-03-11", "date": "2026-03-11T15:04:54Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "gaia v27.0.0", "note": "Cumulo upgraded its Cosmos Hub mainnet validator and RPC infrastructure to Gaia v27.0.0 following the network upgrade process. This release introduces protocol-level changes including disabling the creation of new consumer chains as part of the phased removal of Interchain Security, and adds gas surcharges and recipient limits for multi-send transactions. The upgrade was applied successfully and the Cumulo nodes resumed normal operation after restart.", "url": "https://github.com/cosmos/gaia/releases/tag/v27.0.0" }, { "id": "genlayer-testnet-node-v0.5.2-upgrade-execution-2026-03-11", "date": "2026-03-11T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.2", "note": "Cumulo upgraded its GenLayer validator node to version v0.5.2 following the release announcement. The upgrade improves validator reliability with automatic recovery for missing transaction data, fixes timeout cache invalidation issues, and introduces several consensus stability improvements such as duplicate proposal deduplication and stale block rejection. Additional fixes enhance sync reliability and logging. Cumulo applied the new binary and verified normal validator operation after restart.", "url": "https://docs.genlayer.com/validators/changelog#v052" }, { "id": "genlayer-testnet-node-v0.5.1-upgrade-execution-2026-03-06", "date": "2026-03-06T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (asimov)", "node_type": ["validator"], "client": "genlayer v0.5.1", "note": "Cumulo upgraded its GenLayer validator node to version v0.5.1 following the release announcement. The upgrade introduces improvements to transaction nonce resilience, hanging transaction detection, and finalization logic, along with support for Morpheus as an OpenAI-compatible LLM provider for validators. Cumulo applied the upgrade to maintain compatibility with the network and verified validator operation after restart.", "url": "https://docs.genlayer.com/validators/changelog#v051" }, { "id": "xrplevm-testnet-proposal-40-cumulo-vote-yes-2026-03-05", "date": "2026-03-05T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["validator"], "vote": "YES", "proposal_id": "40", "note": "Cumulo validator voted YES on XRPL EVM Testnet governance Proposal #40. The proposal updates feemarket parameters by reducing base_fee to 100 GWei to enable the deployment of the Arachnid Deterministic Deployment Proxy, allowing deterministic contract deployments across environments.", "url": "https://governance.testnet.xrplevm.org/xrplevm/proposals/40" }, { "id": "warden-testnet-barra-v1.0.0-upgrade-execution-2026-03-05", "date": "2026-03-05T11:15:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Warden", "network": "testnet (barra)", "node_type": ["validator","rpc"], "client": "wardend v1.0.0", "note": "Cumulo upgraded its Warden Barra Testnet validator to wardend v1.0.0 at the scheduled network halt (block height 4,619,000). The upgrade followed the governance proposal and required validators to restart their nodes with the new binary once the chain halted. Cumulo performed the upgrade at the target height and verified node health and synchronization after restart.", "url": "https://github.com/warden-protocol/wardenprotocol/releases/tag/v1.0.0" }, { "id": "cosmoshub-mainnet-proposal-1025-cumulo-vote-yes-2026-03-03", "date": "2026-03-03T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator"], "client": "gaiad", "note": "Cumulo voted YES on Cosmos Hub Governance Proposal #1025, which schedules the v27.0.0 upgrade. The proposal introduces protocol changes including disabling the creation of new consumer chains as part of the planned removal of Interchain Security from the Hub, along with additional transaction validation and fee mechanism updates. As an active validator, Cumulo participated in governance and supported the upgrade proposal.", "url": "https://cosmos.explorers.guru/proposal/1025" }, { "id": "warden-testnet-proposal-6-cumulo-vote-yes-2026-03-05", "date": "2026-03-05T11:15:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Warden", "network": "testnet (barra)", "node_type": ["validator"], "client": "wardend", "note": "Cumulo voted YES on Warden Barra Testnet Governance Proposal #6, which schedules the upgrade to v1.0.0 at block height 4,619,000. As an active validator, Cumulo participated in the governance process supporting the upgrade to the v1.0.0 release.", "url": "https://testnet.itrocket.net/warden/gov/6" }, { "id": "xrplevm-mainnet-upgrade-proposal-26-v10-0-0-execution-2026-03-02", "date": "2026-03-02T13:30:00Z", "incident_type": "upgrade", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator", "rpc"], "client": ["exrpd v10.0.2", "CometBFT", "Cosmos EVM"], "note": "Cumulo executed the XRPL EVM Mainnet v10.0.0 upgrade (node release v10.0.2) as scheduled by Governance Proposal #26 at block height 4,749,700, deploying the critical cosmos/evm fix. As part of the upgrade, Cumulo applied the mandatory configuration change in app.toml by setting [evm] evm-chain-id = \"1440000\" prior to the upgrade height to ensure correct post-upgrade EVM execution and consensus participation. Validator and public RPC services were restarted and verified healthy after the upgrade.", "url": "https://governance.xrplevm.org/xrplevm/proposals/26" }, { "id": "xrplevm-mainnet-rpc-evm-websocket-cors-deployment-2026-02-27", "date": "2026-02-27T18:30:00Z", "incident_type": "infrastructure_update", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["rpc", "json-rpc", "websocket"], "client": ["exrpd v10.x", "CometBFT", "Cosmos EVM"], "note": "XRPL EVM Mainnet public infrastructure upgrade. Cumulo enabled CORS headers on the RPC endpoint and deployed secure WebSocket support for both CometBFT (Cosmos layer) and EVM JSON-RPC (eth_subscribe). The deployment required reverse proxy adjustments for HTTP upgrade handling, TLS termination, and WebSocket forwarding to the execution-layer RPC service. This enhancement enables real-time block subscriptions, governance tracking, and EVM event streaming for dApps and monitoring systems.", "url": "https://rpc.xrpl.cumulo.org.es" }, { "id": "xrplevm-testnet-rpc-evm-websocket-cors-deployment-2026-02-27", "date": "2026-02-27T18:15:00Z", "incident_type": "infrastructure_update", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["rpc", "json-rpc", "websocket"], "client": ["exrpd v10.x", "CometBFT", "Cosmos EVM"], "note": "XRPL EVM Testnet infrastructure enhancement. Public RPC endpoints were updated to support cross-origin resource sharing (CORS) and secure WebSocket connectivity for both CometBFT subscriptions and EVM JSON-RPC (wss). NGINX reverse proxy rules were adapted to properly forward Upgrade and Connection headers, enabling persistent bidirectional streams. This deployment improves compatibility with browser-based dashboards, governance monitors, and EVM event subscribers.", "url": "https://rpc.xrpl.cumulo.com.es" }, { "id": "celestia-mainnet-rpc-websocket-cors-enhancement-2026-02-28", "date": "2026-02-28T13:00:00Z", "incident_type": "infrastructure_update", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["rpc", "websocket"], "client": "celestia-app v0.38.x (CometBFT)", "note": "Infrastructure enhancement on Celestia Mainnet RPC endpoints. Cumulo enabled CORS support on the public RPC endpoint and activated secure WebSocket (wss://) access for CometBFT subscriptions (/websocket). This update allows browser-based clients, indexers, dashboards, and monitoring systems to consume real-time block and event data without cross-origin restrictions. Reverse proxy rules were updated to support HTTP/1.1 upgrade headers and WebSocket forwarding.", "url": "https://celestia.cumulo.org.es" }, { "id": "celestia-mocha-rpc-websocket-cors-enhancement-2026-02-28", "date": "2026-02-28T13:05:00Z", "incident_type": "infrastructure_update", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["rpc", "websocket"], "client": "celestia-app v0.38.x-mocha", "note": "Celestia Mocha testnet RPC infrastructure updated to support CORS and secure WebSocket (wss://) connectivity. Public RPC endpoints now allow cross-origin requests and real-time CometBFT event subscriptions. NGINX reverse proxy configuration was adjusted to forward Upgrade headers and maintain persistent WebSocket connections. This enhancement improves compatibility with browser dashboards, explorers, and developer tooling.", "url": "https://rpc.xrpl.cumulo.com.es" }, { "id": "xrplevm-mainnet-proposal-26-cumulo-vote-yes-2026-02-26", "date": "2026-02-26T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "exrpd", "note": "Cumulo voted YES on XRPL EVM Mainnet Governance Proposal #26, which schedules the v10.0.0 upgrade (node release v10.0.2) including a critical cosmos/evm fix at block height 4,749,700. The vote was cast within the expedited governance window to support network stability and ensure timely deployment of the execution-layer fix.", "url": "https://governance.xrplevm.org/xrplevm/proposals/26" }, { "id": "xrplevm-mainnet-v9-v10-critical-upgrades-cumulo-execution-2026-02-26", "date": "2026-02-26T13:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator", "rpc"], "client": ["exrpd v9.0.0", "exrpd v10.0.0"], "note": "Cumulo participated in and executed the expedited mainnet upgrades to v9.0.0 and v10.0.0 following critical Cosmos/EVM fixes. Votes were cast within the required governance windows and binaries were upgraded immediately after proposal execution to avoid any network halt risk. Both upgrades were applied under compressed timelines, requiring coordinated validator action and operational readiness. Cumulo upgraded its validator and RPC infrastructure within the designated upgrade blocks and verified node health post-restart.", "url": "https://github.com/xrplevm/node/releases" }, { "id": "xrplevm-mainnet-pruned-snapshot-deployment-2026-02-25", "date": "2026-02-25T00:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["snapshot"], "client": "exrpd", "note": "Cumulo deployed a new PRUNED snapshot endpoint for XRPL EVM Mainnet. The snapshot is generated from a pruned mainnet node and published as .tar.zst with a stable latest symlink plus index.json metadata (last update, latest ledger, size). This deployment provides a fast recovery path for validators and RPC operators without requiring a full archive sync.", "url": "https://cumulo.pro/services/xrplevm_mainnet/snapshot" }, { "id": "xrplevm-mainnet-proposal-25-cumulo-vote-yes-2026-02-25", "date": "2026-02-25T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "exrpd", "note": "Cumulo voted YES on XRPL EVM Mainnet Governance Proposal #25 (v9.0.0 Migration from Evmos to Cosmos EVM v0.4.2). This vote supports the transition from an unmaintained Evmos dependency to a fully supported Cosmos EVM fork, improving long-term sustainability, EVM compatibility, and interoperability within the Cosmos ecosystem. Vote cast as part of Cumulo’s validator governance responsibilities.", "url": "https://governance.xrplevm.org/xrplevm/proposals/25" }, { "id": "cosmos-provider-testnet-gaia-v27-0-0-rc0-upgrade-2026-02-24", "date": "2026-02-24T14:30:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "testnet", "node_type": ["validator"], "client": "gaia v27.0.0-rc0", "note": "Governance-gated upgrade of the Cosmos Provider testnet (ICS testnet) to Gaia v27.0.0-rc0. Scheduled for February 24, 2026 at 14:30 UTC, targeting block height 16150500. Upgrade name includes the full version 'v27.0.0'. Validators must follow governance proposal #297 and execute the upgrade at the specified height. This upgrade is part of the regular Testnet Tuesday cycle.", "url": "https://explorer.polypore.xyz/provider/gov/297" }, { "id": "celestia-node-v0.29.1-mocha-consensus-upgrade-2026-02-23", "date": "2026-02-23T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["consensus", "bridge"], "client": "celestia-node v0.29.1-mocha", "note": "Upgrade of Celestia Mocha consensus/bridge node to celestia-node v0.29.1-mocha. This release adds support for celestia-app v7 and is required after the app-v7 migration. A configuration update was required (celestia bridge config-update --p2p.network mocha). Key improvements include Shrex server enablement on bridge nodes, improved fresh start from non-archival consensus nodes, removal of deprecated full node option, header inclusion in subscription responses, resource leak fixes in store accessors, Go 1.26 update, and shrex fix for empty EDS sample handling. Upgrade completed successfully across Cumulo Mocha infrastructure.", "url": "https://github.com/celestiaorg/celestia-node/releases/tag/v0.29.1-mocha" }, { "id": "xrplevm-testnet-upgrade-v10-0-0-execution-2026-02-23", "date": "2026-02-23T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["validator", "rpc"], "client": "exrpd v10.0.0", "note": "Execution of XRPL EVM Testnet upgrade to v10.0.0 following approval of Governance Proposal #39. Migration from Cosmos EVM v0.4.2 to v0.6.0 (fully supported fork). Binary upgraded manually (node_10.0.0_Linux_amd64.tar.gz), service restarted via systemd, and node version verified post-restart. Upgrade completed successfully on February 23, 2026.", "url": "https://github.com/xrplevm/node/releases/tag/v10.0.0" }, { "id": "xrplevm-testnet-proposal-39-upgrade-v10-0-0-2026-02-XX", "date": "2026-02-19T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["validator"], "note": "Governance Proposal #39 LIVE – XRPL EVM Testnet v10.0.0 expedited upgrade. Proposal to migrate Cosmos EVM from v0.4.2 to v0.6.0 (fully supported fork). Validators are required to review and vote. Upgrade instructions will follow if the proposal passes. Voting ends February 20, 2026 at 19:53 (GMT+1).", "url": "https://governance.testnet.xrplevm.org/xrplevm/proposals/39" }, { "id": "celestia-app-v7.0.2-mocha-consensus-upgrade-2026-02-19", "date": "2026-02-19T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "bridge", "consensus"], "client": ["celestia-app v7.0.2-mocha", "celestia-node v0.29.1-mocha"], "note": "Consensus-breaking upgrade on Mocha testnet to celestia-app v7.0.2-mocha and celestia-node v0.29.1-mocha. This release enforces new validator commission bounds (min 20%, max 60%), deprecates Horcrux for v7, reduces min-retain-blocks from 14 days to 3000 blocks, and introduces state machine changes. Validators must signal the upgrade (signal 7) and once 5/6+ voting power has signaled, a 2-day delay is enforced before execution height. Operators upgrading from v6 must carefully review release notes due to breaking changes.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v7.0.2-mocha" }, { "id": "warden-mainnet-emergency-upgrade-v1.0.0-rc1-4476100-2026-02-XX", "date": "2026-02-16T13:15:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Warden", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "wardend v1.0.0-rc1-1-gabb089391", "note": "Emergency halt and upgrade on Warden Mainnet following a failed prior upgrade attempt related to the Path Security patch. Validators were instructed to halt at height 4473500 during the initial rollout, but a subsequent emergency fix required a new coordinated halt at height 4476100 (~13:15 UTC). Nodes were restarted using wardend start --halt-height 4476100 and upgraded to the provided binary-only release. As this was part of a private security handling process, no public source repository was made available at the time of deployment.", "url": "https://explorer.wardenprotocol.org/block/countdown/4476100" }, { "id": "concordium-mainnet-node-10-0-5-protocol-v10-2026-02-16", "date": "2026-02-16T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Concordium", "network": "mainnet", "node_type": ["validator", "node"], "client": "Concordium Node 10.0.5", "note": "Concordium Node 10.0.5 released for Mainnet, introducing support for Protocol Version 10. This major upgrade enables Sponsored Transactions functionality. All mainnet node runners must upgrade to v10.0.5 before the protocol activation scheduled for March 10, 2026. Nodes running older versions will automatically shut down at protocol transition.", "url": "https://docs.concordium.com/en/mainnet/docs/release-notes/release-notes-lp.html#mainnet" }, { "id": "genlayer-testnet-phase-5-matterlabs-migration-2026-02-14", "date": "2026-02-14T16:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet", "node_type": ["validator", "rpc"], "client": "genlayer v0.5.0", "note": "GenLayer Testnet Phase 5 rollout — final phase before Bradbury. Migration to new RaaS provider (Matter Labs / ZKsync Atlas). Testnet history wiped and new RPC endpoints introduced. Validators must rejoin staking via genlayer-cli, update RPC/WS endpoints, update consensus addresses (Main + Data), update validator wallet, clean storage and restart node before Epoch 1 (Monday 17:00 CET).", "url": "https://docs.genlayer.com/validators/changelog#v050" }, { "id": "warden-mainnet-governance-proposal-4-set-fixed-3-percent-inflation-2026-02-13", "date": "2026-02-13T08:46:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Warden", "network": "mainnet", "node_type": ["validator"], "client": "wardend v0.7.x", "note": "Governance Proposal #4 on Warden Mainnet to set a fixed 3% annual inflation rate. The proposal updates mint parameters by setting inflation_min and inflation_max to 0.03, and inflation_rate_change to 0, effectively disabling dynamic inflation adjustments. The goal_bonded parameter remains at 0.67. Voting period currently active.", "url": "https://mainnet.itrocket.net/warden/gov/4" }, { "id": "dymension-mainnet-governance-proposal-38-enroll-kas-solana-router-2026-02-11", "date": "2026-02-11T17:22:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "client": "dymension v4.x", "note": "Governance Proposal #38 on Dymension Hub Mainnet to enroll the Solana KAS warp route router for the KAS token. This proposal enables KAS transfers from Dymension Hub to Solana via remote router enrollment (Msg Enroll Remote Router). Voting period active until February 12, 2026 at 17:22 UTC.", "url": "https://dymension.explorers.guru/proposal/38" }, { "id": "xrplevm-governance-proposal-23-add-atlas-staking-validator-2026-02-18", "date": "2026-02-18T20:47:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "xrplevm-node", "note": "Governance Proposal #23 on XRPL EVM Mainnet to add Atlas Staking as a validator. The proposal is currently in voting period and requires participation from validators and governance stakeholders before the deadline (2026-02-24 19:46:54 UTC). This entry records the governance tracking of validator set changes within XRPL EVM.", "url": "https://governance.xrplevm.org/xrplevm/proposals/23" }, { "id": "xrplevm-governance-proposal-22-replace-cryptocrew-validator-2026-02-16", "date": "2026-02-16T16:22:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "xrplevm-node", "note": "Governance Proposal #22 on XRPL EVM Mainnet to replace the CryptoCrew validator. The proposal evaluates the replacement of CryptoCrew Validators, an Austrian-based Web3 infrastructure provider operating across 40+ interchain networks with enterprise-grade security, IBC relaying services, and long-standing protocol contributions. The proposal is in voting period and requires validator and governance participation before the deadline.", "url": "https://governance.xrplevm.org/xrplevm/proposals/22" }, { "id": "celestia-mainnet-horcrux-signer-relocation-eu-2026-02-09", "date": "2026-02-09T17:00:00Z", "incident_type": "horcrux_maintenance", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["signer"], "client": "Horcrux v3.3.3-celestia", "note": "Horcrux signer cluster maintenance: Cumulo is replacing signer-1 (previously located in Canada) with a new signer hosted in France to reduce network latency and improve signing reliability. The goal is to concentrate at least 2/3 signers and 1 chain node closer geographically to the validator sentry nodes, minimizing timeout risk and reducing chances of desync-related signing instability that can negatively impact uptime metrics. Migration is performed without re-sharding keys by copying the signer shard/ecies material and state files, updating p2pAddr for shardID 1 across all signers, and regenerating Raft state (raft directory removed on all signers) to form a clean cluster with the new endpoint.", "url": "https://github.com/strangelove-ventures/horcrux" }, { "id": "cosmoshub-testnet-validator-creation-2026-02-05", "date": "2026-02-05T12:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "Cosmos Hub", "network": "testnet", "node_type": ["validator"], "client": "gaia", "note": "Creation of a new Cosmos Hub testnet validator (CreateValidator transaction). The validator was successfully registered on-chain and joined the active validator set as part of Cumulo’s Cosmos infrastructure expansion strategy. Monitoring, key management, and operational security policies were applied according to Cumulo’s standard validator deployment procedures.", "url": "https://github.com/Cumulo-pro/cumulo-cosmoshub-infra/" }, { "id": "celestia-app-v6.4.10-mainnet-upgrade-2026-02-03", "date": "2026-02-03T15:30:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.10", "note": "Upgrade of Celestia Mainnet to celestia-app v6.4.10. This release fixes a pruning-related bug affecting node stability. Validators and node operators were recommended to upgrade as soon as possible to avoid potential long-term issues related to pruning behavior. No governance proposal was required for this upgrade.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.10" }, { "id": "starknet-juno-v0.15.18-mainnet-testnet-upgrade-2026-01-27", "date": "2026-01-27T10:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Starknet", "network": "mainnet, testnet", "node_type": ["validator", "rpc"], "client": "juno v0.15.18", "note": "Upgrade of Juno to v0.15.18 on Starknet Mainnet and Testnet. This release updates the underlying Juno database version to unlock future performance improvements and upcoming upgrades. It includes one small migration and one medium-sized migration; validators were advised to upgrade immediately after their attestation window. Estimated migration time is ~1h30m on minimum hardware. The release focuses on polishing RPC v0.10, improving metric reporting, reducing memory allocations, and enhancing overall performance. It also updates the blockifier to v0.16.0-rc.3 with important bug fixes.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.18" }, { "id": "xrplevm-mainnet-gov-proposal-21-2026-01-30", "date": "2026-01-30T12:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "xrplevm", "note": "Governance Proposal #21 on XRPL EVM mainnet to add Flowdesk as a validator. The proposal is currently in the voting period, with validators and governance participants required to cast their votes before the deadline. Cumulo is actively participating in the governance process as part of its ongoing commitment to the security, decentralization, and long-term sustainability of the XRPL EVM validator set.", "url": "https://governance.xrplevm.org/xrplevm/proposals/21" }, { "id": "genlayer-testnet-upgrade-v0.4.5-2026-01-30", "date": "2026-01-30T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet", "node_type": ["validator", "rpc"], "client": "genlayer v0.4.5", "note": "GenLayer testnet node upgrade to v0.4.5. This release improves consensus reliability by decoupling epoch advancement and finalization, preventing blocking issues and increasing validator stability. It also enhances state consistency through a new sync waiting mechanism ensuring GenVM executes contracts with the latest state, and introduces a canPrime check to avoid validator priming when epochs are out of sync. Additional bug fixes include improved GenVM executor process handling and clearer error messages for sync timeouts. Validators are strongly encouraged to upgrade.", "url": "https://docs.genlayer.com/validators/changelog#v045" }, { "id": "fuel-seq-mainnet-1.7-upgrade-2026-01-29", "date": "2026-01-29T16:30:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Fuel", "network": "mainnet", "node_type": ["sequencer", "rpc", "sidecar"], "client": "fuel-sequencer seq-mainnet-1.7", "note": "Scheduled upgrade of the Fuel mainnet sequencer to version seq-mainnet-1.7. This maintenance patch was applied at block height 6572500, with an estimated upgrade time of 17:30 CET (16:30 UTC). The upgrade was performed from previous versions seq-mainnet-1.5 or seq-mainnet-1.6. Validators and node operators were required to register the upgrade name 'maintenance-patch-20260129' when using Cosmovisor. Cumulo successfully upgraded its Fuel mainnet infrastructure, including sequencer and sidecar components, and verified correct operation post-upgrade.", "url": "https://github.com/FuelLabs/fuel-sequencer-deployments/releases/tag/seq-mainnet-1.7" }, { "id": "celestia-app-v6.4.10-mocha-upgrade-2026-01-29", "date": "2026-01-29T15:30:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.10-mocha", "note": "Scheduled upgrade to celestia-app v6.4.10-mocha on Mocha testnet. This release fixes an important pruning bug affecting node storage behavior. Validators and node operators were strongly recommended to upgrade as soon as possible to avoid long-term disk usage issues and state inconsistencies. The maintenance window was scheduled between 15:30 and 15:35 UTC on January 29, 2026. Cumulo upgraded all Mocha infrastructure accordingly and verified correct pruning behavior post-upgrade.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.10-mocha" }, { "id": "fuel-mainnet-gov-15-seq-mainnet-1.6-2026-01-28", "date": "2026-01-28T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Fuel", "network": "mainnet", "node_type": ["sequencer", "validator"], "client": "fuel-sequencer seq-mainnet-1.6", "note": "An amended expedited governance proposal (#15) was submitted to upgrade Fuel Mainnet to seq-mainnet-1.6. Due to the expedited nature of the proposal, validators are required to vote as soon as possible. If approved, the network upgrade is scheduled to take place on Thursday, January 29, 2026 at approximately 15:30 CET. Node operators must ensure readiness for the upgrade and follow the official upgrade instructions provided by the Fuel team.", "url": "https://fuel-seq.simplystaking.xyz/fuel-mainnet/gov/15" }, { "id": "story-aeneid-v1.5.2-horace-upgrade-2026-01-27", "date": "2026-01-27T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Story", "network": "testnet (Aeneid)", "node_type": ["validator", "rpc", "backup"], "client": "Story v1.5.2 (Horace)", "note": "Story v1.5.2 (Horace) binaries were released on January 27, 2026 in preparation for the scheduled Aeneid Testnet upgrade. This release recalibrates token emissions and adjusts the locked staking multiplier as outlined in SIP-00009. Node operators must upgrade to v1.5.2 at least 100 blocks before upgrade height 13,780,500 to remain compatible. Nodes running older versions will halt after the upgrade height, estimated around February 6, 2026 at ~01:00 UTC.", "url": "https://github.com/piplabs/story/releases/tag/v1.5.2" }, { "id": "story-mainnet-v1.5.2-horace-upgrade-2026-01-27", "date": "2026-01-27T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Story", "network": "mainnet", "node_type": ["validator", "rpc", "backup"], "client": "Story v1.5.2 (Horace)", "note": "Story v1.5.2 (Horace) binaries were released on January 27, 2026 in preparation for the scheduled Mainnet upgrade. This is a major release that recalibrates token emissions and adjusts the locked staking multiplier as defined in SIP-00009. All node operators must upgrade to v1.5.2 at least 100 blocks before upgrade height 13,780,500. Nodes running older versions will become incompatible and halt after this height. The upgrade is currently estimated to occur around February 6, 2026 at ~01:00 UTC.", "url": "https://github.com/piplabs/story/releases/tag/v1.5.2" }, { "id": "starknet-mainnet-juno-v0.15.17-upgrade-2026-01-27", "date": "2026-01-27T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Starknet", "network": "mainnet", "node_type": ["validator", "rpc", "backup"], "client": "Juno v0.15.17", "note": "Upgrade to Juno v0.15.17 on Starknet Mainnet. This release updates the underlying Juno database version to unlock performance improvements and future upgrades. It includes one small and one medium-sized migration; validators are advised to upgrade immediately after their attestation window. Estimated total migration time is ~1h30 on minimum-spec machines (4 cores, 8GB RAM, slow SSD). The release polishes RPC v0.10, improves metrics reporting, reduces memory allocation, and updates the blockifier to v0.16.0-rc.3 with important bug fixes.", "url": "https://github.com/NethermindEth/juno/releases" }, { "id": "xrplevm-mainnet-governance-proposal-20-replace-cryptocrew-validator-2026-01-26", "date": "2026-01-26T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #20 opened on XRPL EVM Mainnet to replace the CryptoCrew validator. The proposal is currently in voting period and concerns changes to the active validator set as part of XRPL EVM governance operations. Voting period remains open until February 9th, 2026 at 14:40 UTC.", "url": "https://governance.xrplevm.org/xrplevm/proposals/20" }, { "id": "warden-mainnet-emergency-evm-security-upgrade-v0.7.6-2026-01-22", "date": "2026-01-22T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Warden", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "wardend v0.7.6", "note": "Emergency Warden mainnet upgrade executed without governance proposal to patch a critical EVM vulnerability. A coordinated chain halt was required once sufficient voting power was confirmed. Validators were instructed to immediately restart their nodes with a halt-height at block 3689200 using `wardend start --halt-height 3689200`. After the chain halted, all operators were required to upgrade to wardend v0.7.6 and restart their nodes using the correct binary before resuming the network.", "url": "https://github.com/warden-protocol/wardenprotocol/releases/tag/v0.7.6" }, { "id": "genlayer-testnet-asimov-upgrade-v0.4.4-2026-01-22", "date": "2026-01-22T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (Asimov)", "node_type": ["validator"], "client": "genlayer v0.4.4", "note": "GenLayer Testnet Asimov upgraded to node version v0.4.4. This release fixes memory leaks in thread handling and consensus processing, refactors the ZkSync client for improved reliability, moves storage writes into GenVM for better performance, and enhances logging through context-aware tracing. Validators were advised to upgrade promptly to improve stability, performance, and observability.", "url": "https://docs.genlayer.com/validators/changelog#v044" }, { "id": "xrplevm-mainnet-governance-proposal-19-vote-change-no-2026-01-21", "date": "2026-01-21T09:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "note": "Following the report of a critical issue discovered in CosmosEVM, Cumulo changed its governance vote to NO on Proposal #19, which proposed migrating XRPL EVM from Evmos to Cosmos EVM v0.4.2. Due to the identified risk, the validator community coordinated an urgent response requesting validators to update their votes within a 4-hour window. The proposal ultimately reached ~66% NO votes, preventing the upgrade from taking place and avoiding the introduction of a potentially unsafe dependency into the XRPL EVM mainnet.", "url": "https://governance.xrplevm.org/xrplevm/proposals/19" }, { "id": "dymension-mainnet-security-patch-v4.0.2-2026-01-21", "date": "2026-01-21T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "dymension v4.0.2", "note": "Critical security patch applied on Dymension mainnet to mitigate a CometBFT vulnerability affecting multiple Cosmos-based chains. Version v4.0.2 was released with special handling instructions: source code changes are not yet public and validators must not build from source. Cumulo upgraded using the official pre-built binaries via a rolling upgrade process, with no coordinated halt or specific block height required, ensuring continued network security and validator participation.", "url": "https://github.com/dymensionxyz/dymension/releases/tag/v4.0.2" }, { "id": "xrplevm-testnet-rpc-archive-upgrade-v9.0.2-2026-01-19", "date": "2026-01-19T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["rpc", "archive"], "client": "XRPL EVM Node v9.0.2", "note": "XRPL EVM testnet RPC and Archive nodes were upgraded to v9.0.2 at the request of the Peersyst team. The upgrade was applied across Cumulo-operated public endpoints to ensure compatibility with the latest protocol changes and maintain stability for developers, indexers, and testnet infrastructure consumers.", "url": "https://github.com/xrplevm/node/releases/download/v9.0.2/node_9.0.2_Linux_amd64.tar.gz" }, { "id": "xrplevm-testnet-archive-rpc-infrastructure-deployment-2026-01-19", "date": "2026-01-19T00:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "XRPL EVM", "network": "testnet", "node_type": ["rpc", "archive"], "client": "XRPL EVM Node", "note": "Cumulo deployed a new full archive RPC infrastructure for XRPL EVM testnet in coordination with the Peersyst team. The deployment expands public archive access for developers, indexers, and ecosystem tooling, providing full historical state and multiple access interfaces (RPC, API, JSON-RPC, and gRPC). This infrastructure upgrade strengthens testnet reliability, data availability, and long-term developer support.", "url": "https://cumulo.pro/services/xrplevm/conect" }, { "id": "celestia-app-v6.4.8-mocha-security-upgrade-2026-01-19", "date": "2026-01-19T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.8-mocha", "note": "Celestia-App v6.4.8-mocha released as a critical security patch for the Mocha testnet. Validators and node operators were instructed to upgrade immediately using precompiled standalone binaries. Cumulo verified the binary integrity using published checksums, applied the standalone security patch, and confirmed correct operation post-upgrade. Source code will be published after the patch is fully deployed across networks.", "url": "https://fra1.digitaloceanspaces.com/celestia-app-v6.4.8-mocha/app-vuln-patch-v6.4.8-standalone_Linux_x86_64.tar.gz" }, { "id": "celestia-mainnet-security-upgrade-v6.4.5-2026-01-16", "date": "2026-01-16T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.5", "note": "Celestia-App v6.4.5 released as a critical security patch for Celestia mainnet. Validators and node operators were requested to upgrade as soon as possible. Cumulo applied the standalone security-patched binary provided by the Celestia team to ensure continued consensus safety and network integrity.", "url": "https://fra1.digitaloceanspaces.com/celestia-app-v6.4.5/app-vuln-patch-standalone_Linux_x86_64.tar.gz" }, { "id": "xrplevm-mainnet-cometbft-security-update-v8.0.2-2026-01-16", "date": "2026-01-16T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator", "rpc", "backup"], "client": "xrplevm-node v8.0.2", "note": "Critical security vulnerability identified in the current XRPL EVM node version related to CometBFT. Validators were instructed to upgrade their nodes to v8.0.2 within a 48-hour window to mitigate potential consensus-level risks. The upgrade does not require coordinated halting but timely action is required to ensure network safety. Cumulo proceeded with the upgrade to maintain validator security and protocol integrity.", "url": "https://github.com/xrplevm/node/releases/tag/v8.0.2" }, { "id": "cosmoshub-gaia-v25.3.1-security-upgrade-2026-01-15", "date": "2026-01-15T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "gaia v25.3.1", "note": "Security upgrade applied following the disclosure of an active vulnerability affecting Gaia. Validators patched the issue prior to public disclosure, and Cosmos released a binary-only version (v25.3.1) for node operators. Cumulo upgraded its Cosmos Hub infrastructure using the official prebuilt binary, as building from source was explicitly discouraged. The source code will be published once cleared by the upstream team. This release is consensus-breaking only with v25.3.0 if the vulnerability is exploited.", "url": "https://github.com/cosmos/gaia/releases/tag/v25.3.1" }, { "id": "warden-testnet-critical-security-upgrade-v0.7.4-2-2026-01-15", "date": "2026-01-15T00:00:00Z", "incident_type": "security_upgrade", "status": "completed", "chain": "Warden", "network": "testnet", "node_type": ["validator", "rpc"], "client": "wardend v0.7.4-2", "note": "Urgent security upgrade for Warden testnet validators. Version v0.7.4-2 includes a critical security fix distributed under a private disclosure process. The update is asynchronous and non-consensus-breaking; operators are required to replace the binary and restart their nodes without any coordinated halt. Cumulo applied the upgrade promptly using the officially distributed binaries and container image, following the instruction to avoid public discussion of vulnerability specifics until the Cosmos team publishes further details.", "url": "https://warden-public-bucket.s3.eu-west-1.amazonaws.com/wardend-v0.7.4-2-linux-amd64" }, { "id": "warden-mainnet-v0.7.4-upgrade-2026-01-15", "date": "2026-01-15T17:50:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Warden", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "wardend v0.7.4", "note": "Urgent Warden mainnet upgrade to v0.7.4 executed via governance proposal #2. This is a fast-paced, consensus-breaking upgrade scheduled at block height 3,427,000 (~ Jan 15th, 2026 at 17:50 UTC). Validators and node operators were instructed to prepare binaries in advance, schedule an immediate restart window, and closely monitor logs and telemetry post-upgrade. Cumulo prepared and executed the upgrade following the official runbook, ensuring timely participation in consensus and network stability.", "url": " https://github.com/warden-protocol/wardenprotocol/blob/main/CHANGELOG.md#v074---2026-01-12" }, { "id": "celestia-app-v6.4.7-mocha-upgrade-2026-01-15", "date": "2026-01-15T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "sentry"], "client": "celestia-app v6.4.7-mocha", "note": "New Celestia-App v6.4.7-mocha released for the Mocha testnet. This release fixes critical issues affecting sentry node setups and includes several bug fixes and stability improvements. All Mocha operators were strongly recommended to upgrade as soon as possible. Cumulo, as an active Mocha validator, was closely involved in this upgrade process, working directly with the Celestia team to test multiple Horcrux-related scenarios, performing several pilot installations, and providing detailed feedback and logs from real-world validator and sentry configurations to help identify, reproduce, and validate fixes prior to and during rollout.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.7-mocha" }, { "id": "story-mainnet-scheduled-maintenance-fusaka-aws-2026-01-14", "date": "2026-01-14T23:30:00Z", "incident_type": "maintenance", "status": "completed", "chain": "Story", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "story-geth v1.2.0", "note": "Scheduled maintenance on Story mainnet starting January 14, 2026 at 23:30 UTC, with an expected downtime of 24–48 hours. During this window, the network will be unavailable for transactions, while explorers remain accessible. The maintenance covers two major initiatives: (1) migration of core infrastructure, including mainnet RPC nodes and potentially foundation-operated validators, to AWS to improve reliability, performance, and geographic distribution; and (2) execution of the Fusaka upgrade via story-geth v1.2.0, making Story one of the first EVM Layer 1 chains to adopt Ethereum’s Fusaka hard fork, including EIP-7702 wallet compatibility. Validators and node operators are required to halt nodes at the start of the maintenance window and must not restart until official confirmation. All nodes must be running story-geth v1.2.0 before the network resumes. Target resume window is between January 15, 23:30 UTC and January 16, 23:30 UTC.", "url": "https://github.com/Cumulo-pro/Story_protocol/blob/main/updates/STORY_MAINNET_MAINTENANCE_RUNBOOK.md" }, { "id": "avail-mainnet-validator-commission-change-2026-01-13", "date": "2026-01-13T00:00:00Z", "incident_type": "parameter_change", "status": "completed", "chain": "Avail", "network": "mainnet", "node_type": ["validator"], "note": "Cumulo updated its validator commission on Avail mainnet from 15% to 20% to sustainably cover long-term infrastructure and operational costs. This adjustment aligns with ongoing discussions among validators and the Avail community regarding staking economics and delegation policies, as outlined in AIP-8 and related forum discussions.", "url": "https://forum.availproject.org/t/aip-8-change-ideal-staking-ratio-to-50-and-update-to-foundation-delegation-commission-policy/1742" }, { "id": "quicksilver-mainnet-v1-10-0-governance-upgrade-2026-01-12", "date": "2026-01-12T17:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Quicksilver", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "quicksilver v1.10.0", "note": "Governance Proposal #62 introduces a scheduled Quicksilver Mainnet upgrade to v1.10.0 at height 16,373,000 (expected around January 12, 2026 ~17:00 UTC). This upgrade enables zone offboarding functionality to safely wind down the OmniFlix liquid staking zone, fixes an issue blocking the IBC queue, and includes multiple bug fixes related to out-of-bounds panics and refund handling. Validators must upgrade before the target height to remain in consensus.", "url": "https://github.com/quicksilver-zone/quicksilver/releases/tag/v1.10.0" }, { "id": "warden-testnet-barra-fast-upgrade-v0-7-4-2026-01-12", "date": "2026-01-12T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Warden", "network": "testnet (Barra)", "node_type": ["validator", "rpc"], "client": "wardenprotocol v0.7.4", "note": "Urgent fast-paced upgrade for Warden Testnet (Barra). Version v0.7.4 is scheduled for immediate rollout on January 12th, 2026, targeting block height 3,265,800. Validators and node operators must prepare v0.7.4 binaries in advance, schedule a restart window for today, and closely monitor logs and telemetry after the upgrade. This emergency release is coordinated via governance and requires rapid validator action to maintain network stability.", "url": "https://github.com/warden-protocol/wardenprotocol/blob/main/CHANGELOG.md#v074---2026-01-12" }, { "id": "quicksilver-mainnet-governance-upgrade-v1-10-0-proposal-62", "date": "2026-01-08T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Quicksilver", "network": "mainnet", "node_type": ["validator"], "client": "quicksilver v1.10.0", "note": "Cumulo is participating in governance voting for Quicksilver Mainnet Proposal #62, which proposes upgrading the network to v1.10.0. This upgrade offboards the OmniFlix zone following its chain sunset, introduces new governance messages for zone offboarding and canceling pending unbondings, and fixes issues in the IBC queue. The upgrade is intended to improve network maintenance, governance control, and IBC reliability.", "url": "https://wallet.keplr.app/chains/quicksilver/proposals/62" }, { "id": "xrplevm-mainnet-governance-v9-migration-2026-01-08", "date": "2026-01-08T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "client": "XRPL EVM v9.0.0", "note": "Cumulo participated in governance voting for XRPL EVM Mainnet proposal #19, approving the v9.0.0 chain upgrade to migrate from the unmaintained Evmos dependency to Cosmos EVM v0.4.2. This upgrade improves long-term sustainability, ensures continued compatibility with Ethereum tooling and EIPs, introduces extended ERC-20 functionality (mint, burn, ownership transfer), and strengthens alignment with the Cosmos SDK ecosystem and future interoperability upgrades.", "url": "https://governance.xrplevm.org/xrplevm/proposals/19" }, { "id": "genlayer-testnet-asimov-upgrade-v0.4.3-2026-01-08", "date": "2026-01-08T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (Asimov)", "node_type": ["validator"], "client": "genlayer v0.4.3", "note": "GenLayer Testnet Asimov upgraded to node version v0.4.3. This release fixes node storage handling for non-accepted transactions (e.g. LeaderTimeout), improves validator priming logic, and corrects defaults in the release/.env.example configuration. Validators were advised to upgrade as soon as possible to ensure correct priming behavior and stable transaction handling.", "url": "https://docs.genlayer.com/validators/changelog#v043" }, { "id": "celestia-app-v6.4.4-mainnet-beta-upgrade-2026-01-08", "date": "2026-01-08T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.4", "note": "Celestia-App v6.4.4 released for Mainnet Beta. This update includes bug fixes and improvements, resolving an important issue related to catching up to the tip of the chain. All node operators are strongly recommended to upgrade as soon as possible to ensure proper synchronization and stable operation.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.4" }, { "id": "cosmoshub-mainnet-validator-rpc-deployment-2026-01-02", "date": "2026-01-02T10:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "Cosmos Hub", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "cosmos-sdk (gaia)", "note": "Deployment of a new Cosmos Hub mainnet validator and RPC infrastructure operated by Cumulo. This deployment strengthens Cumulo’s presence in the Cosmos ecosystem, providing resilient validator operations and reliable RPC access. The setup follows Cumulo’s standard infrastructure practices, including secure key management, systemd-based services, monitoring, and operational documentation to ensure transparency, reliability, and long-term maintainability.", "url": "https://github.com/Cumulo-pro/cumulo-cosmoshub-infra/" }, { "id": "xrplevm-mainnet-governance-add-cumulo-validator-2026-01-01", "date": "2026-01-01T15:11:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "XRPL EVM", "network": "mainnet", "node_type": ["validator"], "note": "Governance Proposal #18 to add Cumulo as an official XRPL EVM mainnet validator has been approved. The proposal recognizes Cumulo as a professional PoS validator and blockchain infrastructure provider operating production-grade infrastructure across multiple ecosystems, including EVM, Cosmos-SDK and modular chains. With this approval, Cumulo is formally accepted as a validator participant in the XRPL EVM mainnet validator set.", "url": "https://governance.xrplevm.org/xrplevm/proposals/18" }, { "id": "story-geth-v1.2.0-yasunari-upgrade-mainnet-testnet-2026-01-01", "date": "2026-01-01T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Story", "network": "mainnet/testnet", "node_type": ["validator", "rpc", "backup"], "client": "story-geth v1.2.0 (Yasunari)", "note": "Mandatory upgrade of the Story execution client (Story-Geth) to v1.2.0 (codename: Yasunari). This release includes the Fusaka upgrade to maintain Ethereum compatibility and adds EIP-7702 wallet compatibility enhancements. The Aeneid testnet execution upgrade is scheduled for 2026-01-08 00:00 UTC, and the Story mainnet execution upgrade is scheduled for 2026-01-15 00:00 UTC. All node operators must upgrade before the respective deadlines. Nodes running older versions after these timestamps will become incompatible and may experience execution client halts.", "url": "https://github.com/piplabs/story-geth/releases/tag/v1.2.0" }, { "id": "genlayer-testnet-node-upgrade-v0.4.2-2025-12-21", "date": "2025-12-29T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet", "node_type": ["validator"], "client": "genlayer-node v0.4.2", "note": "GenLayer testnet upgrade to v0.4.2 released. This version introduces mandatory telemetry and monitoring configuration to enable centralized observability of the testnet. Validators must configure Prometheus and Loki authentication via environment variables (CENTRAL_MONITORING_USERNAME, CENTRAL_LOKI_USERNAME, CENTRAL_MONITORING_PASSWORD, CENTRAL_LOKI_PASSWORD). Tokens are distributed privately via GenLayer Telegram groups and must be identical for monitoring and Loki. Validators are also required to update their profile in the GenLayer points system to ensure automatic version detection. Upgrade and configuration are required for continued participation and monitoring.", "url": "https://docs.genlayer.com/validators/changelog" }, { "id": "genlayer-testnet-node-upgrade-v0.4.1-2025-12-21", "date": "2025-12-21T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet", "node_type": ["validator"], "client": "genlayer-node v0.4.1", "note": "GenLayer testnet node upgrade to v0.4.1 released. This update introduces new account management commands (show/list), improved health checks with explicit warnings for unprimed validators, indefinite retry logic for validator priming (removing the need for manual restarts), enhanced GenVM health checks and timeout handling, fixes for transaction rotation and sync filtering issues, and improved message handling to prevent dropped events. Validators are encouraged to upgrade at their earliest convenience.", "url": "https://docs.genlayer.com/validators/changelog#v041" }, { "id": "celestia-mainnet-governance-proposal-9-recover-ibc-clients-2025-12-21", "date": "2025-12-21T12:11:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator"], "client": "celestia-app", "note": "Governance Proposal #9 on Celestia Mainnet aims to recover existing IBC clients using KYVE-based substitute clients. Voting period runs from Dec 21st 2025, 12:11 UTC to Dec 28th 2025, 12:11 UTC. As of today, turnout is 9.83%, below the required quorum of 33.40%, and the proposal is currently expected to be rejected due to insufficient participation. YES votes account for 100% of cast votes, with no NO, NO WITH VETO, or ABSTAIN votes recorded.", "url": "https://www.mintscan.io/celestia/proposals/9" }, { "id": "celestia-app-v6.4.4-mocha-upgrade-2025-12-19", "date": "2025-12-19T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.4-mocha", "note": "New Celestia-App v6.4.4-mocha released for Mocha testnet. This upgrade includes bug fixes and improvements, resolving a critical issue related to catching up to the tip of the chain. All Mocha-4 nodes are strongly recommended to upgrade as soon as possible to maintain synchronization and network stability.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.4-mocha" }, { "id": "dymension-mainnet-governance-proposal-37-usdc-rate-limit-update", "date": "2025-12-18T00:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "note": "New governance proposal (#37) to update the USDC send rate limit from 15% to 70%. The proposal aims to increase liquidity throughput by relaxing current rate limits applied to USDC transfers. Validators are required to review and vote before the voting period ends on December 19, 2025 at 15:34 UTC.", "url": "https://dym.fyi/proposal/37" }, { "id": "dymension-mainnet-governance-proposal-36-beyond-bridge-authority-2025-12-11", "date": "2025-12-17T18:10:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "client": "hyperlane.warp", "note": "Governance Proposal #36 is live on Dymension Mainnet. The proposal assigns Beyond bridge authority to on-chain governance, enabling additional bridging capabilities via Hyperlane. Validators are expected to review the proposal details and participate in the vote. This change impacts how bridge authority is managed and governed within the Dymension ecosystem.", "url": "https://dym.fyi/proposal/36" }, { "id": "celestia-mocha-horcrux-sentry-sync-bug-log-contribution-2025-12-11", "date": "2025-12-11T10:00:00Z", "incident_type": "investigation", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "sentry", "rpc"], "client": ["celestia-app v6.4.x", "Horcrux"], "note": "Active investigation and log contribution related to a critical Celestia Horcrux issue causing sentry nodes to stop syncing due to repeated consensus and propagation errors (invalid part hash, invalid root hash, invalid proposal signature, and Horcrux refusing to sign votes on nil BlockID). Cumulo observed identical behavior on its own validator and sentry infrastructure (Mocha testnet and Mainnet) and provided detailed log excerpts to upstream developers. The issue has been acknowledged by the Celestia core team, marked as high priority, and is believed to surface under high block fullness conditions. Debug logs are being used to assist root-cause analysis and fix development.", "url": "https://github.com/celestiaorg/celestia-app/issues/6300#issuecomment-3647263981" }, { "id": "story-mainnet-aeneid-v1.4.2-mandatory-security-upgrade-2025-12-11", "date": "2025-12-11T09:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Story", "network": "mainnet / testnet (Aeneid)", "node_type": ["validator", "rpc", "backup"], "client": "story v1.4.2", "note": "Mandatory upgrade to Story v1.4.2 for both Mainnet and Aeneid Testnet. This release fixes a critical security issue affecting the Story consensus client. All node operators must immediately download, install, and run the updated binaries if the private v1.4.2 patch has not already been applied. Failure to upgrade may result in consensus participation issues or security exposure.", "url": "https://github.com/piplabs/story/releases/tag/v1.4.2" }, { "id": "story-mainnet-stlouis-network-degradation-2025-12-11", "date": "2025-12-11T01:00:00Z", "incident_type": "service_disruption", "status": "completed", "chain": "Story", "network": "mainnet", "node_type": ["validator"], "client": "story", "note": "Regional network degradation affecting Cumulo’s Story Protocol validator infrastructure in St. Louis (USA). Impact window 01:00–01:08 UTC (02:00–02:08 CET). Symptoms included a temporary consensus stall and repeated proposal/round timeouts due to delayed or missing gossip/proposal data. Local host checks showed the NIC remained UP with no RX/TX errors or carrier drops, no service restarts, and no firewall interference. The validator recovered automatically once upstream routing stabilized; no slashing and no resync required.", "url": "https://github.com/Cumulo-pro/Cumulo-Front-Chain/blob/main/postmortem/2025-12-11-story-network-incident.md" }, { "id": "dymension-mainnet-governance-proposal-35-adjust-inflation-2025-12-11", "date": "2025-12-11T12:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "client": "governance", "note": "New on-chain governance proposal #35: Adjust Inflation Parameter — Decrease GoalBonded from 50% to 30%. With the current bonded ratio at ~42.58%, the protocol is programmatically increasing inflation to chase an outdated target. The proposal updates /cosmos.mint.v1beta1.MsgUpdateParams. Voting ends on December 15, 2025 at 14:44 UTC. Validators should review and consider the economic implications for staking yields and network security.", "url": "https://dym.fyi/proposal/35" }, { "id": "celestia-app-v6.4.2-mainnet-beta-upgrade-2025-12-11", "date": "2025-12-11T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet (beta)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.2", "note": "Celestia-App v6.4.2 released for Mainnet Beta. This security patch requires immediate upgrade from all operators. The update enhances node security and stability across consensus and RPC components. Cumulo has applied the upgrade and verified validator and RPC health.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.2" }, { "id": "starknet-mainnet-juno-v0.15.15-upgrade-2025-12-11", "date": "2025-12-11T10:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Starknet", "network": "mainnet", "node_type": ["validator", "rpc"], "client": "Juno v0.15.15", "note": "Upgrade to Juno v0.15.15 completed on Starknet mainnet. This update includes performance improvements, bug fixes and enhanced synchronization stability. The process required stopping the systemd service, downloading the new binary, unpacking it, moving it to /usr/local/bin and assigning execution permissions. All Cumulo validator and RPC nodes have been updated and resumed normal operation.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.15" }, { "id": "arcium-devnet-node-upgrade-2025-12-10", "date": "2025-12-10T18:20:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Arcium", "network": "devnet", "node_type": ["validator", "arx-node", "cluster-member"], "client": "arcium-node v0.5.2", "note": "Upgrade of Cumulo’s ARX node from v0.4.x to v0.5.2 on Arcium devnet. All tooling was refreshed using arcup, new BLS keys were generated, and the node was fully re-registered on-chain via init-arx-accs. A new cluster was created (offset 80123456) to comply with the updated cluster requirements introduced in v0.5.x, and the Cumulo node (offset 90441123) was successfully added as an active member. Docker runtime was rebuilt using the updated environment variables for BLS, identity and callback authority. Node activation and pub/sub synchronization were validated through arx-info and arx-active, confirming correct operation with Solana Devnet RPC endpoints. The upgrade resolves the previous BLS .bin compatibility issue and ensures full compliance with the latest ARX protocol.", "url": "https://github.com/Cumulo-pro/cumulo-arcium-infra/blob/main/docs/01_2_upgrade_v0_5_2.md" }, { "id": "celestia-app-v6.4.2-mocha-upgrade-2025-12-10", "date": "2025-12-10T12:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.4.2-mocha", "note": "Celestia-App v6.4.2-mocha released for Mocha testnet. This upgrade is a security patch addressing critical issues, and all node operators were urged to update as soon as possible. The release enhances stability, resolves security vulnerabilities, and is required to maintain proper operation of Mocha nodes. Operators should verify that their validator and RPC nodes are running the patched version.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.4.2-mocha" }, { "id": "genlayer-asimov-testnet-validator-node-upgrade-v0.4.0-2025-12-08", "date": "2025-12-08T15:15:00Z", "incident_type": "upgrade", "status": "completed", "chain": "GenLayer", "network": "testnet (Asimov)", "node_type": ["validator"], "client": "genlayer-node v0.4.0", "note": "Upgrade and full reconfiguration of the GenLayer Asimov testnet validator node to genlayer-node v0.4.0. The existing operator wallet (legacy GenLayer key) was imported and set as operatorAddress, while the validatorWalletAddress was configured according to the staking wizard output. RPC connectivity was migrated to the official genlayer-testnet Caldera endpoints, GenVM was initialized via setup.py with v0.2.7 executors, and the Selenium WebDriver container was validated. Final checks with genlayernode doctor and systemd-based service startup confirm the node is running in validator mode and ready for production monitoring.", "url": "https://github.com/Cumulo-pro/cumulo-genlayer-infra/blob/main/README.md" }, { "id": "arcium-devnet-cluster-creation-2025-12-03", "date": "2025-12-03T10:30:00Z", "incident_type": "deployment", "status": "completed", "chain": "Arcium", "network": "devnet", "node_type": ["cluster"], "client": "arcium-node", "note": "Creation of Cumulo Cluster #1 on Arcium devnet. A new cluster was initialized using arcium init-cluster with a dedicated cluster offset and max-nodes configuration. The cluster definition was successfully published on-chain, Solana RPC confirmed the registration, and the Cumulo ARX node was added as the first active member. Cluster metadata, activation order, and capacity parameters were validated with arcium status. The cluster is now ready to accept additional ARX nodes for trusted execution workloads.", "url": "https://github.com/Cumulo-pro/cumulo-arcium-infra/blob/main/docs/02_cluster_setup.md" }, { "id": "arcium-devnet-node-deployment-2025-12-03", "date": "2025-12-03T10:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "Arcium", "network": "devnet", "node_type": ["validator", "arx-node", "cluster-member"], "client": "arcium-node", "note": "Initial deployment of the Arcium (ARX) node on devnet. The node was initialized with a freshly generated node-keypair.json and successfully registered on-chain using arcium init. Solana RPC connectivity validated, offsets assigned, and node status confirmed via arcium status. The ARX node was then added to Cumulo’s first cluster, with max-nodes defined and the cluster offset recorded. All services are operating normally and producing valid proofs. Configuration steps included: keypair generation, node registration, RPC sync validation, and cluster membership setup.", "url": "https://github.com/Cumulo-pro/cumulo-arcium-infra/blob/main/docs/01_node_setup.md" }, { "id": "dymension-mainnet-governance-proposal-34-tee-fast-withdrawal-2025-12-03", "date": "2025-12-03T12:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "client": "dymension", "note": "Governance Proposal #34 — Introducing TEE-based fast withdrawal for RollApps. Using the new Beyond binary, this proposal enables TEE-backed fast finality so that the Hub can instantly and safely finalize RollApp state updates, allowing users to withdraw without relying on eIBC. Fully additive: existing RollApps can adopt the capability without breaking current flows.", "url": "https://portal.dymension.xyz/dymension/proposal/34" }, { "id": "avail-mainnet-testnet-upgrade-v2.3.4.1-2025-12-01", "date": "2025-12-01T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Avail", "network": "mainnet/testnet", "node_type": ["validator", "rpc", "full"], "client": "avail-node v2.3.4.1", "note": "New Avail node release v2.3.4.1 is available for both mainnet and testnet. The upgrade includes important fixes and improvements. All operators are advised to upgrade as soon as possible to stay aligned with the latest changes. Validator Engagement Team offers support if any issues occur during the update.", "url": "https://github.com/availproject/avail/releases/tag/v2.3.4.1" }, { "id": "celestia-mainnet-upgrade-v6.3.0-node-v0.28.4-2025-11-27", "date": "2025-11-27T15:25:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator", "rpc", "archive", "full", "bridge"], "client": ["celestia-app v6.3.0", "celestia-node v0.28.4"], "note": "Mainnet Beta upgrade to celestia-app v6.3.0 and celestia-node v0.28.4. Scheduled between 15:25–15:55 UTC on November 27, 2025. This release contains major changes required for correct node operation and introduces the foundation for 32MB/6s blocks. All node operators — validators, consensus full nodes, RPC providers, and archive nodes — must upgrade promptly. It is critical to update configs, use an updated KMS, and validate node performance after applying the release.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.3.0" }, { "id": "celestia-mainnet-governance-proposal-8-2025-11-27", "date": "2025-11-27T04:10:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator"], "client": "celestia-app", "note": "Governance Proposal #8 — Increase max square size to 256 and block size to 32 MiB. This proposal updates core DA and block parameters for Celestia mainnet ahead of the transition toward larger square sizes. Validators should review the implications on block production, bandwidth, and hardware before voting.", "url": "https://www.mintscan.io/celestia/proposals/8" }, { "id": "celestia-mocha-governance-proposal-16-2025-11-26", "date": "2025-11-26T16:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator"], "client": "celestia-app", "note": "Governance Proposal #16 — Increase Max Square Size to 512 and Block Size to 128 MiB. Updates blob-related parameters for Mocha testnet. This proposal aligns Mocha with the planned 128MB block architecture. Validators should review and vote accordingly, as this change impacts data availability parameters and block production constraints.", "url": "https://testnet.celestia.explorers.guru/proposal/16" }, { "id": "celestia-app-v6.3.0-mocha-upgrade-2025-11-24", "date": "2025-11-26T15:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "client": "celestia-app v6.3.0-mocha", "note": "New Celestia-App v6.3.0-mocha released for Mocha testnet. This upgrade introduces key bug fixes and enables validators to bump to 128MB/6s blocks. Mocha validators must ensure their hardware is fully compatible with the 128MB/6s requirements to avoid falling out of sync or losing the ability to propose blocks. Recommended to upgrade immediately after the announcement. Refer to v6 hardware and block-size guidelines in the release notes.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.3.0-mocha" }, { "id": "dymension-kaspa-bridge-validator-full-setup-2025-11-25", "date": "2025-11-25T20:10:00Z", "incident_type": "deployment", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["bridge-validator", "hyperlane", "kaspa-node", "indexer"], "client": [ "hyperlane-validator (main-dym branch)", "rusty-kaspad v1.0.1", "kaspa-rest-server", "simply-kaspa-indexer v1.6.0-beta1" ], "url": "https://github.com/dymensionxyz/hyperlane-monorepo/tree/main-dym/dymension", "note": "Completed the full setup of the Cumulo bridge validator for the Dymension ↔ Kaspa connection. The deployment included updating to the latest main-dym Hyperlane monorepo, rebuilding the validator binary, and integrating a fresh validator-config.json from the updated template. Both key pairs (ISM signer + Kaspa escrow) were generated, extracted, and inserted with correct formatting, including the quoted Kaspa escrow secret and 0x-prefixed ISM validator key. Local Kaspa endpoints were configured through kaspaUrlsWrpc=127.0.0.1:17210 and kaspaUrlsRest=http://127.0.0.1:8000. A dummy kaspa.wallet was installed as required, validator DB initialized, environment variables exported, and systemd deployed to run the Hyperlane validator. In parallel, the Kaspa node stack (rusty-kaspad + utxoindex + rest + simply-kaspa-indexer) was fully deployed and verified, with REST API operational via /openapi.json and /info/health. The bridge validator and the complete Kaspa data stack are now fully active, synchronized, and functioning normally." }, { "id": "starknet-juno-v01511-sync-regression-2025-11-25", "date": "2025-11-25T19:45:00Z", "incident_type": "service_disruption", "status": "completed", "chain": "Starknet", "network": "mainnet/testnet", "node_type": ["rpc", "backup"], "client": "Juno v0.15.11", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.11", "note": "Following the upgrade to Juno v0.15.11, both Starknet RPC and backup nodes began experiencing severe sync regression: block ingestion slowed dramatically, recurring 'Restarting sync process (catchUpMode)' events were logged, and L1 WebSocket subscriptions collapsed regularly with 'failed to read frame header: EOF'. This behavior did not exist prior to the upgrade. Investigation revealed that the local Ethereum node (geth + beacon) was under heavy load, causing unstable L1 feeds and triggering gateway throttling. Mitigation steps included: (1) temporarily stopping the local Ethereum node to confirm the bottleneck, which immediately improved block download speed; (2) switching Juno's --eth-node to a remote L1 endpoint hosted on another node of our infrastructure; (3) applying safe Juno performance tuning — increasing db-cache-size to 4096, db-max-handles to 4096, and widening gw-timeouts. After these changes, sync speed normalized, L1 feed stabilized, and catchUpMode loops ceased. All nodes now operate normally and no further impact has been detected." }, { "id": "dymension-mainnet-rpc-hotfix-v4.0.1-2025-11-24", "date": "2025-11-24T12:00:00Z", "incident_type": "update", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["rpc"], "client": "dymension v4.0.1", "note": "Hotfix update v4.0.1 released for Dymension mainnet. Fixes a critical issue affecting eth_gasPrice queries. Required for all operators running public RPC or client-facing nodes. Optional for validator/full nodes. This release is not consensus-breaking and requires no coordination or halt—just a simple binary replacement.", "url": "https://github.com/dymensionxyz/dymension/releases/tag/v4.0.1" }, { "id": "starknet-juno-v0.15.12-upgrade-mainnet-testnet", "date": "2025-11-25T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Starknet", "network": "mainnet/testnet", "node_type": ["validator", "rpc", "backup"], "client": "juno", "note": "Release of Juno v0.15.12 containing multiple bug fixes and improvements, especially important for RPC providers. Mainnet and testnet nodes should update to ensure stability and correct RPC behavior. Update applied across validator, RPC, and backup nodes.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.12" }, { "id": "dymension-kas-bootstrap-module-governance-proposal-2025-11-24", "date": "2025-11-24T18:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "note": "Governance proposal #32 — Bootstrap KAS Module. This proposal initializes the Kaspa–Dymension bridge (KAS) module with its core components, enabling cross-chain operations. Proposal type: /dymensionxyz.dymension.kas.MsgBootstrap. Voting ends on 2025-11-25 at 20:35 (Europe/Madrid). With ❤ by kjnodes.", "url": "https://dym.fyi/proposal/32" }, { "id": "dymension-kaspa-bridge-validator-setup-2025-11-24", "date": "2025-11-24T18:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["bridge-validator", "kms", "hyperlane"], "client": ["hyperlane-monorepo", "kaspa-bridge-validator"], "url": "https://github.com/dymensionxyz/hyperlane-monorepo", "note": "Deployment of the Dymension ↔ Kaspa bridge validator instance. A single validator node is configured with two key pairs to process KAS → DYM bridging. Key generation completed (ISM key + Kaspa escrow key), public keys shared with the Dymension team, and awaiting final validator-config template. Environment prepared for both bare-metal and AWS KMS workflows. Systemd service and Docker/KMS architecture validated. Hardware meets requirements (2 CPU, 2GB RAM, 128GB disk). Metrics exposed on port 9090 for monitoring." }, { "id": "dymension-kaspa-bridge-node-setup-2025-11-14", "date": "2025-11-22T18:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["full node"], "client": ["rusty-kaspad", "kaspa-rest-server", "simply-kaspa-indexer"], "url": "https://docs.kas.fyi/guides/new-to-kaspa/run-a-kaspa-node", "note": "Kaspa node deployed as part of the Dymension ↔ Kaspa bridge infrastructure. Full indexer stack installed: rusty-kaspad (RPC ports 16210/17210), kaspa-rest-server, simply-kaspa-socket-server, simply-kaspa-indexer and PostgreSQL 16 with 10GB shared memory. Managed via Docker Compose with persistent volumes. Repository cloned from supertypo/simply-kaspa-indexer (tag v1.6.0-beta1). Node launched with utxoindex enabled and network testnet-10. Sync validated and logs inspected successfully." }, { "id": "story-mainnet-v1.4.1-terence-hardfork-upgrade", "date": "2025-11-17T02:18:00Z", "incident_type": "upgrade", "chain": "Story", "network": "mainnet", "status": "completed", "node_type": ["validator", "rpc", "backup"], "client": "story", "note": "Mandatory Story Mainnet hardfork (codename Terence) to v1.4.1. Upgrade height is 11,538,000, estimated around December 5, 2025 between 06:00–10:00 UTC. All node operators must upgrade binaries before this height to remain in consensus; staying out of sync for ~1.5 days leads to slashing. Countdown: https://storyscan.io/block/countdown/11538000.", "url": "https://github.com/piplabs/story/releases/tag/v1.4.1" }, { "id": "starknet-juno-v0.15.11-upgrade-mainnet-testnet", "date": "2025-11-18T00:00:00Z", "incident_type": "upgrade", "chain": "Starknet", "network": "mainnet/testnet", "status": "completed", "node_type": ["validator", "rpc", "backup"], "client": "juno", "note": "Release of Juno v0.15.11 with full support for Starknet 0.14.1. Testnet activation scheduled for tomorrow. Previous version v0.15.10 supported SN 0.14.1 except execution RPC calls. Includes a short migration (~1:30) on standard hardware. Validators should upgrade outside their attestation window.", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.11" }, { "id": "dymension-beyond-upgrade-governance-proposal-2025-11-14", "date": "2025-11-14T12:00:00Z", "incident_type": "governance_vote", "status": "completed", "chain": "Dymension", "network": "mainnet", "node_type": ["validator"], "note": "The Beyond upgrade governance proposal is now live. The upgrade is scheduled for block 10192266 (~ Nov 24, 2025, 13:00 UTC). Validators must review and follow the official upgrade instructions. This upgrade includes changes to block times; the team will track which validators have updated both their binaries and fast block configurations. A dedicated Discord thread will be opened for coordination.", "url": "https://dymension.explorers.guru/proposal/31" }, { "id": "empower-mainnet-validator-decommission-2025-11-14", "date": "2025-11-14T09:00:00Z", "incident_type": "decommission", "status": "completed", "chain": "Empower", "network": "mainnet", "node_type": ["validator"], "note": "Cumulo has officially discontinued its validator services on Empower Chain mainnet as of November 14th, 2025. Delegators were advised to restake their assets due to the network's prolonged inactivity and lack of communication.", "url": "https://x.com/Cumulo_pro/status/1986025572724601130" }, { "id": "celestia-app-v6.2.5-mocha-upgrade", "date": "2025-11-13T00:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "client": "celestia-app", "network": "testnet (mocha)", "node_type": ["validator", "rpc", "consensus"], "note": "New Celestia-App v6.2.5-mocha released for Mocha testnet. This update stabilizes mainnet-related logic and fixes indexing bugs introduced in v5. Validators and operators are requested to upgrade as soon as possible.", "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.2.5-mocha" }, { "id": "xrplevm-testnet-archive-node-deployment-2025-11-11", "date": "2025-11-11T12:00:00Z", "incident_type": "deployment", "status": "completed", "chain": "XRPL EVM", "network": "testnet (xrplevm_1449000-1)", "node_type": ["archive", "snapshot"], "client": "XRPL EVM Node", "url": "https://cumulo.pro/services/xrplevm/snapshot", "note": "At the request of the XRPL EVM team, a full archive node has been deployed on the testnet (Chain ID: xrplevm_1449000-1). The node provides complete historical state data to support dApp indexing and testing. A corresponding snapshot of the archive node is now publicly available at the provided URL." }, { "id": "celestia-mainnet-upgrade-matcha-v6-2025-11-08", "date": "2025-11-08T09:00:00Z", "incident_type": "upgrade", "status": "completed", "chain": "Celestia", "network": "mainnet", "node_type": ["validator", "bridge", "rpc"], "client": ["celestia-app v6.2.4", "celestia-node v0.28.2"], "url": "https://github.com/celestiaorg/celestia-app/releases/tag/v6.2.4", "note": "Celestia Matcha upgrade announced — introducing celestia-app v6.2.4 and celestia-node v0.28.2. This is a consensus and configuration breaking release focused on throughput, blob size, inflation reduction (2.5%), and fee parameter changes. Validators must ensure at least 1Gbps bandwidth, update Horcrux (Celestia fork) or tmKMS (v0.15.0), and apply the updated config via `celestia-appd update-config`. Signaling for upgrade to v6 is now active on-chain, with the transition scheduled to execute 7 days after 5/6+ voting power has signaled. The upgrade will move mainnet to the Matcha architecture and enable larger blocks and squares." }, { "id": "dymension-hyperlane-bridge-keys-2025-11-10", "date": "2025-11-10T19:00:00+01:00", "incident_type": "setup", "status": "completed", "chain": "Dymension", "network": "testnet", "node_type": ["validator"], "client": "Hyperlane Monorepo / AWS KMS", "url": "https://github.com/dymensionxyz/hyperlane-monorepo", "note": "Successfully generated and configured bridge validator keys for the Dymension ↔ Kaspa Hyperlane connection. The setup included creation of symmetric (Kaspa escrow) and asymmetric (Dymension signer) KMS keys under AWS region ap-southeast-1, configuration of IAM policies, and validation of outputs. Kaspa Escrow Public Key: 036611f55361dee173e1a5dcffdbcf6ccd142e37ba7a8d8b32caf1bbf8d2465a97, Dymension Public Address: 0x995871E50396de955989415fbF5564AeAd0819F0." }, { "id": "starknet-2025-11-08-update-juno-01510", "date": "2025-11-08T06:00:00+01:00", "incident_type": "update", "status": "completed", "url": "https://github.com/NethermindEth/juno/releases/tag/v0.15.10", "chain": "Starknet", "client": "Juno", "node_type": ["validator", "rpc", "backup"], "network": "mainnet/testnet", "note": "Upgrade to Juno v0.15.10 including important bug fixes and performance improvements. Update applied across validator, RPC, and backup nodes. All services verified and fully synchronized." }, { "id": "story-2025-10-31-update-v1-4-0-aeneid", "date": "2025-11-08T06:00:00+01:00", "incident_type": "update", "status": "completed", "url": "https://github.com/piplabs/story/releases/tag/v1.4.0", "chain": "Story", "client": "Story", "node_type": ["validator", "rpc", "backup"], "network": "testnet (Aeneid)", "note": "Story v1.4.0 (Terence) released for Aeneid Testnet. This major upgrade introduces single binary support, protobuf-based serialization, and new staking module API endpoints. All Aeneid nodes were required to upgrade before height 10886688 (~Nov 8th, 08:45 UTC). Upgrade successfully completed across validator, RPC, and backup nodes with Cosmovisor handling the process." }, { "id": "axone-rpc-cors-preflight-fix-2025-11-07", "date": "2025-11-07T00:00:00Z", "incident_type": "configuration", "status": "completed", "chain": "Axone", "network": "mainnet", "node_type": ["rpc"], "client": "Axone v12.0.0", "url": "https://rpc.axone.cumulo.com.es/", "note": "Enabled proper CORS on preflight (OPTIONS) requests for the Axone mainnet RPC. Previously, no CORS headers were returned on OPTIONS, causing browser-based dApps to be blocked. Now the endpoint serves the expected Access-Control-* headers." }, { "id": "dymension-blumbus-upgrade-v4-0-0-rc05-2025-11-07", "date": "2025-11-07T23:00:00Z", "incident_type": "update", "status": "completed", "chain": "Dymension", "network": "testnet (Blumbus)", "node_type": ["validator", "rpc"], "client": "Dymension v4.0.0-rc05", "url": "https://github.com/dymensionxyz/dymension/releases/tag/v4.0.0-rc05", "note": "Throughout the week, several express upgrades and patches were applied to Blumbus testnet nodes to improve network stability and synchronization. The final upgrade to v4.0.0-rc05 was executed successfully, setting `halt-height = 10301500` (approx. 11h after release). All validator, RPC, and backup nodes resumed operation normally after the halt, completing the transition without downtime." } ]