{ "slug": "agent-footprint-2026", "title": "Linux monitoring agent footprint benchmark (2026): 11 agents, one VPS type, 24 hours", "checked": "2026-11-15", "run": { "provider": "Hetzner Cloud", "location": "Helsinki (HEL1)", "plan": "CX23: 2 vCPU AMD EPYC (shared), 4 GB RAM, 40 GB NVMe", "image": "Ubuntu 26.04 LTS", "kernel": "7.0.0-15/-30/-42/-43-generic (differs by box, see kernels)", "harnessCommit": "d879a73..12c959b", "cycles": [ { "start": 1794086972, "end": 1794699557 }, { "start": 1794179152, "end": 1794699712 }, { "start": 1794362388, "end": 1794699863 } ], "notes": [ "Ubuntu's apt-daily and apt-daily-upgrade timers were masked on every box at provisioning (the pilot's unattended-upgrades restarted dynamically linked agents mid-cycle).", "Network bytes count only the box's physical interfaces; Docker bridge traffic is excluded.", "Processes are counted in the agent's cgroup and its child cgroups.", "Disk written counts data writes. Filesystem metadata is written through the block device's inode, and the kernel charges all writeback of one inode to a single cgroup, whichever owns it, so it is not counted for any row, including the metadata an agent's own file activity causes. In the eight runs whose collector recorded it (Monitorable run 1, Beszel run 3 and the three runs of each Netdata row, measured again on 2026-11-12/13, the Netdata Cloud runs once more on 2026-11-13/14; docs/RUN-2026-slot5.md, docs/RUN-2026-slot6.md) the metadata charged to the unit's cgroup is taken out and the charge as the kernel reported it is published as well (disk_charged_mb_24h). The other 28 runs have no such record and publish the kernel's charge; docs/RUN-2026-slot5.md says why they are taken not to contain that stream.", "Those eight runs replace the runs first measured in their place, under a rule fixed before each retest: every run measured again replaces the run it stands for, whatever it shows. The replaced runs are in data/2026/superseded/. The Netdata Cloud runs of 2026-11-12/13 were replaced in their turn, because the benchmark's operator had caused traffic in them by opening the Netdata Cloud Space (docs/RUN-2026-slot6.md); they are in data/2026-slot5/. A cycle therefore holds runs of different days: each row's `runs` in results.json gives the box and the dates of every run and whether it has the metadata record.", "CPU under load is lower than idle CPU in every row, the control included, and no box recorded steal time; the cause is not established.", "Idle CPU depends on the box. On box-0 and box-6 an agent used 1.3 to 1.5 times the idle CPU time it used on box-3; under load the difference is smaller for three of the four agents measured on all three. Beszel, HetrixTools, Pulse and Glances have two of their three runs on those two boxes, so their idle CPU median is one of those runs (each row's notes give its box-3 figure); every other row was measured on the other seven boxes, except Netdata's run 1. The two boxes' virtual CPUs report 2,495 MHz and the others' 2,445 MHz (read on 2026-11-13, docs/evidence/2026-slot5/), so they are probably on another host type; the cause is not established." ] }, "metrics": [ { "key": "pss_idle_median_mb", "label": "Memory (PSS), idle median", "unit": "MB", "lowerIsBetter": true }, { "key": "pss_idle_p95_mb", "label": "Memory (PSS), idle p95", "unit": "MB", "lowerIsBetter": true }, { "key": "rss_idle_median_mb", "label": "Memory (RSS), idle median", "unit": "MB", "lowerIsBetter": true }, { "key": "memory_current_idle_median_mb", "label": "Memory (cgroup memory.current), idle median", "unit": "MB", "lowerIsBetter": true }, { "key": "memory_peak_mb", "label": "Memory peak (cgroup), whole cycle", "unit": "MB", "lowerIsBetter": true }, { "key": "first_hour_peak_mb", "label": "Memory (PSS) peak, first hour after install", "unit": "MB", "lowerIsBetter": true }, { "key": "cpu_idle_pct", "label": "CPU, idle (% of one vCPU)", "unit": "%", "lowerIsBetter": true }, { "key": "cpu_loaded_pct", "label": "CPU, under load (% of one vCPU)", "unit": "%", "lowerIsBetter": true }, { "key": "disk_written_mb_24h", "label": "Disk written per 24 h", "unit": "MB", "lowerIsBetter": true }, { "key": "disk_charged_mb_24h", "label": "Disk writes charged to the unit's cgroup per 24 h (with any filesystem metadata charged to it)", "unit": "MB", "lowerIsBetter": true }, { "key": "disk_metadata_charged_mb_24h", "label": "Filesystem-metadata writeback charged to the unit's cgroup per 24 h", "unit": "MB", "lowerIsBetter": true }, { "key": "journal_mb_24h", "label": "Log volume handed to journald per 24 h", "unit": "MB", "lowerIsBetter": true }, { "key": "net_out_mb_24h", "label": "Network out per 24 h", "unit": "MB", "lowerIsBetter": true }, { "key": "net_in_mb_24h", "label": "Network in per 24 h", "unit": "MB", "lowerIsBetter": true }, { "key": "net_headline_mb_24h", "label": "Network per 24 h (out for push agents, in+out for polled)", "unit": "MB", "lowerIsBetter": true }, { "key": "install_seconds", "label": "Install wall time", "unit": "s", "lowerIsBetter": true }, { "key": "disk_footprint_mb", "label": "On-disk footprint after the cycle", "unit": "MB", "lowerIsBetter": true }, { "key": "pids_median", "label": "Processes, idle median", "unit": "", "lowerIsBetter": true }, { "key": "threads_median", "label": "Threads, idle median", "unit": "", "lowerIsBetter": true } ], "rows": [ { "key": "hetrixtools", "name": "HetrixTools agent", "version": "2.4.2", "versions": [ "2.4.2", "2.4.2", "2.4.2" ], "licence": "open source, no LICENSE file", "sourceUrl": "https://github.com/hetrixtools/agent", "installUrl": "https://github.com/hetrixtools/agent", "unitNames": [ "hetrixtools_agent.service", "hetrixtools_agent_1794363271-9241.service", "hetrixtools_agent.timer" ], "reportsTo": "sm.hetrixtools.net (wget POST once a minute)", "intervalSeconds": 60, "us": false, "footnotes": [ "This box has no cron, so the vendor installer's own systemd path runs: a minutely timer starting a transient unit per run. Those runs are placed in a persistent slice (one word added to the vendor launcher) so the kernel's accounting for them survives each run. The timer's launcher unit is matched too; a unit that lives for less than the 10-second sampling interval is counted only as far as the samples catch it.", "The CPU figure is 12.6 to 17.4 % of one vCPU in the three runs, and the box's load average agrees: its idle median is 0.22 to 0.30 on this row's boxes against 0.05 to 0.13 on every other box.", "Idle CPU: the published median (17.05 %) is a run on box-0 or box-6, which idle 1.3 to 1.5 times higher than the other boxes for every agent measured on them; this row's run on box-3 shows 12.63 % (see the method notes)." ], "runs": [ { "cycle": 1, "box": "box-0", "start": 1794178759, "end": 1794268759, "metadataRecord": false }, { "cycle": 2, "box": "box-3", "start": 1794270979, "end": 1794360979, "metadataRecord": false }, { "cycle": 3, "box": "box-6", "start": 1794363271, "end": 1794453271, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 8.979456, "cycles": [ 8.979456, 8.845312, 8.987648 ] }, "pss_idle_p95_mb": { "median": 9.156608, "cycles": [ 9.156608, 9.146368, 9.161728 ] }, "rss_idle_median_mb": { "median": 19.202048, "cycles": [ 19.202048, 19.058688, 19.206144 ] }, "memory_current_idle_median_mb": { "median": 3.710976, "cycles": [ 3.72736, 3.637248, 3.710976 ] }, "memory_peak_mb": { "median": 12.45184, "cycles": [ 14.852096, 12.45184, 12.382208 ] }, "first_hour_peak_mb": { "median": 9.225216, "cycles": [ 9.186304, 9.261056, 9.225216 ] }, "cpu_idle_pct": { "median": 17.049758499818818, "cycles": [ 17.363850413093367, 12.625013562024398, 17.049758499818818 ] }, "cpu_loaded_pct": { "median": 12.856514902506962, "cycles": [ 13.039505710306406, 10.899097966573816, 12.856514902506962 ] }, "disk_written_mb_24h": { "median": 5.898952433868825, "cycles": [ 5.898952433868825, 5.898952433868825, 5.898952433868825 ] }, "disk_charged_mb_24h": { "median": 5.898952433868825, "cycles": [ 5.898952433868825, 5.898952433868825, 5.898952433868825 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 4.514347826086956, "cycles": [ 4.514284173913044, 4.514347826086956, 4.514349913043478 ] }, "net_out_mb_24h": { "median": 3.750102302210412, "cycles": [ 3.732158569875589, 3.770809498731731, 3.750102302210412 ] }, "net_in_mb_24h": { "median": 6.413441004952289, "cycles": [ 6.407612474936586, 6.416029143616379, 6.413441004952289 ] }, "net_headline_mb_24h": { "median": 3.750102302210412, "cycles": [ 3.732158569875589, 3.770809498731731, 3.750102302210412 ] }, "install_seconds": { "median": 2.1, "cycles": [ 2.1, 1.8, 2.1 ] }, "disk_footprint_mb": { "median": 0.080777, "cycles": [ 0.080743, 0.080777, 0.085745 ] }, "pids_median": { "median": 4.0, "cycles": [ 4.0, 4.0, 4.0 ] }, "threads_median": { "median": 4.0, "cycles": [ 4.0, 4.0, 4.0 ] } } }, { "key": "beszel", "name": "Beszel agent", "version": "beszel-agent 0.21.1", "versions": [ "beszel-agent 0.21.1", "beszel-agent 0.21.1", "beszel-agent 0.21.1" ], "licence": "MIT", "sourceUrl": "https://beszel.dev/guide/agent-installation", "installUrl": "https://beszel.dev/guide/agent-installation", "unitNames": [ "beszel-agent.service" ], "reportsTo": "Beszel hub on the sink (WebSocket, hub-driven every 60 s)", "intervalSeconds": 60, "us": false, "footnotes": [ "The memory peak (118 MB) is not only an install-time figure. The cgroup's high-water mark is already 117 MB at the first sample, seconds after the install, and its memory then returns to 101 to 118 MB for one 10-second sample every 61 minutes: 24 such samples in each of runs 1 and 3, with a PSS of 116 to 132 MB in them, and none caught in run 2, as can happen with a spike shorter than the sampling interval. Between them the cgroup holds about 8 MB, which is what the idle median shows.", "Idle CPU: the published median (0.17 %) is a run on box-0 or box-6, which idle 1.3 to 1.5 times higher than the other boxes for every agent measured on them; this row's run on box-3 shows 0.12 % (see the method notes).", "Run 3 (box-6) was measured again on 2026-11-12/13, on the same box, with a collector that records which cgroup the kernel charges for the box's filesystem-metadata writeback; it replaces the run of 2026-11-10 (`data/2026/superseded/cycle-3/`). That run was charged 16.6 MB of disk writes in its first seven hours; this one was charged 4 KB in 25 hours, while the box's metadata writeback (150 MB over the run) was charged to other cgroups throughout. `docs/RUN-2026-slot5.md` has the cause and the rule under which the run was replaced." ], "runs": [ { "cycle": 1, "box": "box-0", "start": 1794086972, "end": 1794176972, "metadataRecord": false }, { "cycle": 2, "box": "box-3", "start": 1794179152, "end": 1794269152, "metadataRecord": false }, { "cycle": 3, "box": "box-6", "start": 1794506945, "end": 1794596945, "metadataRecord": true } ], "values": { "pss_idle_median_mb": { "median": 17.85856, "cycles": [ 17.854464, 17.903104, 17.85856 ] }, "pss_idle_p95_mb": { "median": 18.706432, "cycles": [ 18.706432, 18.567168, 18.784256 ] }, "rss_idle_median_mb": { "median": 19.345408, "cycles": [ 19.341312, 19.392512, 19.345408 ] }, "memory_current_idle_median_mb": { "median": 8.21248, "cycles": [ 8.204288, 8.245248, 8.21248 ] }, "memory_peak_mb": { "median": 118.120448, "cycles": [ 118.120448, 117.997568, 118.165504 ] }, "first_hour_peak_mb": { "median": 18.71872, "cycles": [ 18.870272, 18.419712, 18.71872 ] }, "cpu_idle_pct": { "median": 0.16824370696944074, "cycles": [ 0.16824370696944074, 0.12402881628215968, 0.18941628699118251 ] }, "cpu_loaded_pct": { "median": 0.16098409470752087, "cycles": [ 0.16098409470752087, 0.10622250696378831, 0.16623197771587744 ] }, "disk_written_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_metadata_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0 ] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 2.375160942142771, "cycles": [ 2.3761231453074045, 2.3207975069452833, 2.375160942142771 ] }, "net_in_mb_24h": { "median": 1.4193822055803842, "cycles": [ 1.4246419712525666, 1.4059395385916174, 1.4193822055803842 ] }, "net_headline_mb_24h": { "median": 3.794543147723155, "cycles": [ 3.800765116559971, 3.7267370455369004, 3.794543147723155 ] }, "install_seconds": { "median": 3.4, "cycles": [ 5.2, 3.0, 3.4 ] }, "disk_footprint_mb": { "median": 10.789175, "cycles": [ 10.789175, 10.789175, 10.789175 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 9.0, "cycles": [ 9.0, 9.0, 9.0 ] } } }, { "key": "node-exporter", "name": "node_exporter", "version": "v1.12.1", "versions": [ "v1.12.1", "v1.12.1", "v1.12.1" ], "licence": "Apache-2.0", "sourceUrl": "https://prometheus.io/docs/guides/node-exporter/", "installUrl": "https://prometheus.io/docs/guides/node-exporter/", "unitNames": [ "node_exporter.service", "node_exporter.socket" ], "reportsTo": "passive: Prometheus on the sink scrapes /metrics at its default 1 m", "intervalSeconds": 60, "us": false, "footnotes": [ "Network bytes are counted by TCP port (the listening socket is not in the service's cgroup under socket activation, and polled traffic is what this row sends)." ], "runs": [ { "cycle": 1, "box": "box-4", "start": 1794087563, "end": 1794177563, "metadataRecord": false }, { "cycle": 2, "box": "box-7", "start": 1794179793, "end": 1794269793, "metadataRecord": false }, { "cycle": 3, "box": "box-1", "start": 1794362537, "end": 1794452537, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 24.465408, "cycles": [ 24.465408, 25.137152, 24.440832 ] }, "pss_idle_p95_mb": { "median": 24.84224, "cycles": [ 24.84224, 25.489408, 24.82176 ] }, "rss_idle_median_mb": { "median": 24.4736, "cycles": [ 24.4736, 25.145344, 24.449024 ] }, "memory_current_idle_median_mb": { "median": 9.306112, "cycles": [ 9.306112, 9.408512, 9.285632 ] }, "memory_peak_mb": { "median": 10.56768, "cycles": [ 10.56768, 10.4448, 10.592256 ] }, "first_hour_peak_mb": { "median": 23.986176, "cycles": [ 23.719936, 24.334336, 23.986176 ] }, "cpu_idle_pct": { "median": 0.02396352699601401, "cycles": [ 0.02396352699601401, 0.0244227575794178, 0.023285144341104 ] }, "cpu_loaded_pct": { "median": 0.022584373259052923, "cycles": [ 0.020769498607242337, 0.023215264623955432, 0.022584373259052923 ] }, "disk_written_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 18.315465230100255, "cycles": [ 18.315465230100255, 18.341622128276363, 18.311732257519026 ] }, "net_in_mb_24h": { "median": 1.429658577122841, "cycles": [ 1.429658577122841, 1.4336159246285785, 1.4258932528083101 ] }, "net_headline_mb_24h": { "median": 19.745123807223095, "cycles": [ 19.745123807223095, 19.775238052904943, 19.737625510327334 ] }, "install_seconds": { "median": 1.8, "cycles": [ 1.8, 1.7, 1.8 ] }, "disk_footprint_mb": { "median": 23.90084, "cycles": [ 23.90084, 23.90084, 23.90084 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 5.0, "cycles": [ 5.0, 5.0, 5.0 ] } } }, { "key": "monitorable", "name": "Monitorable agent", "version": "monitorable-agent version v1.7.0", "versions": [ "monitorable-agent version v1.7.0", "monitorable-agent version v1.7.0", "monitorable-agent version v1.7.0" ], "licence": "Apache-2.0", "sourceUrl": "https://monitorable.io/docs/agent/install/", "installUrl": "https://monitorable.io/docs/agent/install/", "unitNames": [ "monitorable-agent.service" ], "reportsTo": "Monitorable ingest (OTLP/HTTP every 60 s)", "intervalSeconds": 60, "us": true, "footnotes": [ "Run 1 (box-1) was measured again on 2026-11-12/13, on the same box, with a collector that records which cgroup the kernel charges for the box's filesystem-metadata writeback; it replaces the run of 2026-11-08 (`data/2026/superseded/cycle-1/`). That run was charged 15.8 MB of disk writes in its first eight hours; this one was charged nothing after its install, in 25 hours, while the box's metadata writeback (163 MB over the run) was charged to other cgroups throughout. `docs/RUN-2026-slot5.md` has the cause and the rule under which the run was replaced.", "The pilot run measured v1.5.0, which wrote 115 MB per 24 h to its on-disk export queue. v1.7.0 keeps a batch in memory for up to 15 s and writes it to disk only if it is still undelivered; the re-run was scheduled after that release, so that this row, like every other, is the agent's current release.", "Run 3 (box-7): the 0.3 MB are the agent's export queue. A batch that is not delivered within 15 s is written to disk; that happened seven times between 16:24 and 21:16 UTC on 2026-11-11, while our own ingest endpoint was slow or failing. A run in which every batch is delivered in time writes nothing (runs 1 and 2)." ], "runs": [ { "cycle": 1, "box": "box-1", "start": 1794506165, "end": 1794596165, "metadataRecord": true }, { "cycle": 2, "box": "box-4", "start": 1794271116, "end": 1794361116, "metadataRecord": false }, { "cycle": 3, "box": "box-7", "start": 1794363418, "end": 1794453418, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 45.776896, "cycles": [ 45.981696, 45.477888, 45.776896 ] }, "pss_idle_p95_mb": { "median": 47.095808, "cycles": [ 47.095808, 47.0016, 47.251456 ] }, "rss_idle_median_mb": { "median": 45.785088, "cycles": [ 45.989888, 45.48608, 45.785088 ] }, "memory_current_idle_median_mb": { "median": 14.512128, "cycles": [ 14.604288, 14.4384, 14.512128 ] }, "memory_peak_mb": { "median": 18.198528, "cycles": [ 18.198528, 17.944576, 18.710528 ] }, "first_hour_peak_mb": { "median": 46.821376, "cycles": [ 46.821376, 46.637056, 47.04256 ] }, "cpu_idle_pct": { "median": 0.08514629906993598, "cycles": [ 0.08514629906993598, 0.08289823650199299, 0.08645645005435439 ] }, "cpu_loaded_pct": { "median": 0.06777688022284122, "cycles": [ 0.06779621169916435, 0.058734512534818935, 0.06777688022284122 ] }, "disk_written_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.31204603454523494 ] }, "disk_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.31204603454523494 ] }, "disk_metadata_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0 ] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.009164869565217391 ] }, "net_out_mb_24h": { "median": 8.097008508273946, "cycles": [ 8.083825698755888, 8.097008508273946, 8.334676864355599 ] }, "net_in_mb_24h": { "median": 1.103299509602609, "cycles": [ 1.099926580504892, 1.103299509602609, 1.1298154076579296 ] }, "net_headline_mb_24h": { "median": 8.097008508273946, "cycles": [ 8.083825698755888, 8.097008508273946, 8.334676864355599 ] }, "install_seconds": { "median": 6.7, "cycles": [ 10.8, 6.7, 6.4 ] }, "disk_footprint_mb": { "median": 57.251083, "cycles": [ 57.251083, 57.251083, 57.349387 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 8.0, "cycles": [ 8.0, 9.0, 8.0 ] } } }, { "key": "pulse", "name": "Pulse agent", "version": "v6.6.0", "versions": [ "v6.6.0", "v6.6.0", "v6.6.0" ], "licence": "MIT", "sourceUrl": "https://github.com/rcourtman/Pulse/blob/main/docs/UNIFIED_AGENT.md", "installUrl": "https://github.com/rcourtman/Pulse/blob/main/docs/UNIFIED_AGENT.md", "unitNames": [ "pulse-agent.service" ], "reportsTo": "Pulse server on the sink (HTTP push every 30 s)", "intervalSeconds": 30, "us": false, "footnotes": [ "Auto-update stays at its default (on).", "The agent talks to the Pulse server over the bed's private network (10.0.0.0/24) in plain HTTP: the agent refuses an http:// server URL unless the host is loopback or a private address, and the sink has no TLS hostname. The installer it fetches from the server is checksum-pinned.", "Idle CPU: the published median (0.072 %) is a run on box-0 or box-6, which idle 1.3 to 1.5 times higher than the other boxes for every agent measured on them; this row's run on box-3 shows 0.053 % (see the method notes)." ], "runs": [ { "cycle": 1, "box": "box-6", "start": 1794087884, "end": 1794177884, "metadataRecord": false }, { "cycle": 2, "box": "box-0", "start": 1794270581, "end": 1794360581, "metadataRecord": false }, { "cycle": 3, "box": "box-3", "start": 1794362837, "end": 1794452837, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 49.491968, "cycles": [ 49.491968, 49.147904, 50.294784 ] }, "pss_idle_p95_mb": { "median": 51.0976, "cycles": [ 51.0976, 50.561024, 51.712 ] }, "rss_idle_median_mb": { "median": 49.50016, "cycles": [ 49.50016, 49.156096, 50.302976 ] }, "memory_current_idle_median_mb": { "median": 14.491648, "cycles": [ 14.360576, 14.491648, 14.78656 ] }, "memory_peak_mb": { "median": 17.252352, "cycles": [ 16.973824, 17.252352, 17.911808 ] }, "first_hour_peak_mb": { "median": 51.154944, "cycles": [ 51.154944, 50.491392, 52.011008 ] }, "cpu_idle_pct": { "median": 0.0715691665660104, "cycles": [ 0.07786460200507307, 0.0715691665660104, 0.05327382654910013 ] }, "cpu_loaded_pct": { "median": 0.043996267409470755, "cycles": [ 0.04465961002785515, 0.043996267409470755, 0.04090771587743732 ] }, "disk_written_mb_24h": { "median": 11.793630264524703, "cycles": [ 11.793630264524703, 11.793630264524703, 11.793630264524703 ] }, "disk_charged_mb_24h": { "median": 11.793630264524703, "cycles": [ 11.793630264524703, 11.793630264524703, 11.793630264524703 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 16.19849825341225, "cycles": [ 16.220572571566617, 16.008854480009663, 16.19849825341225 ] }, "net_in_mb_24h": { "median": 10.303374803720256, "cycles": [ 10.303374803720256, 10.29256306317188, 10.383919139992752 ] }, "net_headline_mb_24h": { "median": 16.19849825341225, "cycles": [ 16.220572571566617, 16.008854480009663, 16.19849825341225 ] }, "install_seconds": { "median": 31.3, "cycles": [ 31.4, 31.3, 30.8 ] }, "disk_footprint_mb": { "median": 42.081323, "cycles": [ 42.081323, 42.081323, 42.081323 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 9.0, "cycles": [ 9.0, 9.0, 9.0 ] } } }, { "key": "fivenines", "name": "Fivenines agent", "version": "fivenines-agent 1.23.1", "versions": [ "fivenines-agent 1.23.1", "fivenines-agent 1.23.1", "fivenines-agent 1.23.1" ], "licence": "WTFPL", "sourceUrl": "https://github.com/Five-Nines-io/fivenines_agent#installation", "installUrl": "https://github.com/Five-Nines-io/fivenines_agent#installation", "unitNames": [ "fivenines-agent.service" ], "reportsTo": "Fivenines API (HTTPS push every 60 s)", "intervalSeconds": 60, "us": false, "footnotes": [ "The memory peak (129 MB) is not only an install-time figure. The cgroup's high-water mark is already 125 MB at the first sample, seconds after the install, while it holds 34 MB; later its memory rose again, to as much as 128 MB, in single samples 16 minutes apart: 12 samples between 21.9 and 24.8 hours after the install in run 2 and 14 between 17.9 and 21.6 hours in run 3 (samples above 1.3 times the run's median), none in run 1. Otherwise it holds about 35 MB, which is what the idle median shows." ], "runs": [ { "cycle": 1, "box": "box-8", "start": 1794088170, "end": 1794178170, "metadataRecord": false }, { "cycle": 2, "box": "box-2", "start": 1794270851, "end": 1794360851, "metadataRecord": false }, { "cycle": 3, "box": "box-5", "start": 1794363122, "end": 1794453122, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 49.882112, "cycles": [ 49.881088, 49.882112, 49.922048 ] }, "pss_idle_p95_mb": { "median": 50.110464, "cycles": [ 50.110464, 49.984512, 50.11456 ] }, "rss_idle_median_mb": { "median": 52.89984, "cycles": [ 52.89984, 52.854784, 52.985856 ] }, "memory_current_idle_median_mb": { "median": 35.47136, "cycles": [ 35.442688, 35.618816, 35.47136 ] }, "memory_peak_mb": { "median": 128.708608, "cycles": [ 128.598016, 128.708608, 128.802816 ] }, "first_hour_peak_mb": { "median": 49.726464, "cycles": [ 49.401856, 49.726464, 49.737728 ] }, "cpu_idle_pct": { "median": 0.21632305834037927, "cycles": [ 0.21632305834037927, 0.21434907235173328, 0.2243454535571929 ] }, "cpu_loaded_pct": { "median": 0.20891125348189415, "cycles": [ 0.20891125348189415, 0.1895406406685237, 0.22121364902506963 ] }, "disk_written_mb_24h": { "median": 0.4744809566372751, "cycles": [ 0.4744809566372751, 0.4744809566372751, 0.4744809566372751 ] }, "disk_charged_mb_24h": { "median": 0.4744809566372751, "cycles": [ 0.4744809566372751, 0.4744809566372751, 0.4744809566372751 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.2871015652173913, "cycles": [ 0.287088, 0.2871046956521739, 0.2871015652173913 ] }, "net_out_mb_24h": { "median": 10.037784814591134, "cycles": [ 10.050299717357168, 10.037784814591134, 9.974065466843822 ] }, "net_in_mb_24h": { "median": 4.401272772073922, "cycles": [ 4.412211832346903, 4.388909191931393, 4.401272772073922 ] }, "net_headline_mb_24h": { "median": 10.037784814591134, "cycles": [ 10.050299717357168, 10.037784814591134, 9.974065466843822 ] }, "install_seconds": { "median": 4.1, "cycles": [ 4.5, 4.0, 4.1 ] }, "disk_footprint_mb": { "median": 52.088342, "cycles": [ 52.088342, 52.089538, 52.088342 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 4.0, "cycles": [ 4.0, 4.0, 4.0 ] } } }, { "key": "glances", "name": "Glances", "version": "4.5.9", "versions": [ "4.5.9", "4.5.9", "4.5.9" ], "licence": "LGPL-3.0", "sourceUrl": "https://github.com/nicolargo/glances#installation", "installUrl": "https://github.com/nicolargo/glances#installation", "unitNames": [ "glances.service" ], "reportsTo": "passive: the sink polls /api/4/all at Glances' refresh period", "intervalSeconds": 3, "us": false, "footnotes": [ "Glances ships no systemd unit; the harness adds a two-line one that runs `glances -w`.", "Runs as root (pipx install as root, the documented route); port 61208 is reachable from the sink only.", "Network bytes are counted by TCP port (61208); polled traffic is what this row sends.", "The network figure is the harness's own poller fetching /api/4/all every 3 s (Glances' refresh period); each reply is about 137 KB, which is the 3.9 GB per 24 h. Scale it to your own polling interval.", "Idle CPU: the published median (2.38 %) is a run on box-0 or box-6, which idle 1.3 to 1.5 times higher than the other boxes for every agent measured on them; this row's run on box-3 shows 1.83 % (see the method notes)." ], "runs": [ { "cycle": 1, "box": "box-3", "start": 1794087418, "end": 1794177418, "metadataRecord": false }, { "cycle": 2, "box": "box-6", "start": 1794179635, "end": 1794269635, "metadataRecord": false }, { "cycle": 3, "box": "box-0", "start": 1794362388, "end": 1794452388, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 77.712384, "cycles": [ 77.604864, 77.722624, 77.712384 ] }, "pss_idle_p95_mb": { "median": 78.214144, "cycles": [ 78.116864, 78.214144, 78.24896 ] }, "rss_idle_median_mb": { "median": 87.171072, "cycles": [ 87.138304, 87.293952, 87.171072 ] }, "memory_current_idle_median_mb": { "median": 67.063808, "cycles": [ 67.014656, 67.063808, 67.125248 ] }, "memory_peak_mb": { "median": 72.585216, "cycles": [ 72.585216, 72.798208, 71.917568 ] }, "first_hour_peak_mb": { "median": 76.788736, "cycles": [ 76.773376, 76.788736, 76.819456 ] }, "cpu_idle_pct": { "median": 2.3838728542094456, "cycles": [ 1.8254575637154244, 2.3838728542094456, 2.487292837299191 ] }, "cpu_loaded_pct": { "median": 1.7184529247910862, "cycles": [ 1.7048077158774373, 1.7184529247910862, 1.732654512534819 ] }, "disk_written_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 3903.520435144341, "cycles": [ 3926.3689657591494, 3903.520435144341, 3902.9311775431815 ] }, "net_in_mb_24h": { "median": 26.21566144703467, "cycles": [ 26.21566144703467, 26.196783688851312, 26.546161536417443 ] }, "net_headline_mb_24h": { "median": 3929.7172188331924, "cycles": [ 3952.5846272061845, 3929.7172188331924, 3929.477339079599 ] }, "install_seconds": { "median": 33.4, "cycles": [ 27.0, 34.2, 33.4 ] }, "disk_footprint_mb": { "median": 28.768359, "cycles": [ 28.768359, 28.768359, 28.768188 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 3.0, "cycles": [ 3.0, 3.0, 3.0 ] } } }, { "key": "telegraf", "name": "Telegraf", "version": "Telegraf 1.40.3 (git: HEAD@41b45388)", "versions": [ "Telegraf 1.40.3 (git: HEAD@41b45388)", "Telegraf 1.40.3 (git: HEAD@41b45388)", "Telegraf 1.40.3 (git: HEAD@41b45388)" ], "licence": "MIT", "sourceUrl": "https://docs.influxdata.com/telegraf/v1/install/", "installUrl": "https://docs.influxdata.com/telegraf/v1/install/", "unitNames": [ "telegraf.service" ], "reportsTo": "InfluxDB 1.x on the sink (the stock [[outputs.influxdb]] block, enabled with the sink's URL)", "intervalSeconds": 10, "us": false, "footnotes": [ "Telegraf's stock config enables no output; its [[outputs.influxdb]] block is enabled with the sink's URL (two lines). Nothing else is changed." ], "runs": [ { "cycle": 1, "box": "box-5", "start": 1794087710, "end": 1794177710, "metadataRecord": false }, { "cycle": 2, "box": "box-8", "start": 1794179939, "end": 1794269939, "metadataRecord": false }, { "cycle": 3, "box": "box-2", "start": 1794362680, "end": 1794452680, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 180.178944, "cycles": [ 179.920896, 180.178944, 181.825536 ] }, "pss_idle_p95_mb": { "median": 181.948416, "cycles": [ 181.32992, 181.948416, 183.156736 ] }, "rss_idle_median_mb": { "median": 180.187136, "cycles": [ 179.929088, 180.187136, 181.833728 ] }, "memory_current_idle_median_mb": { "median": 56.762368, "cycles": [ 56.762368, 56.700928, 57.053184 ] }, "memory_peak_mb": { "median": 58.970112, "cycles": [ 58.650624, 58.970112, 59.113472 ] }, "first_hour_peak_mb": { "median": 179.93728, "cycles": [ 179.060736, 179.93728, 180.89984 ] }, "cpu_idle_pct": { "median": 0.21369707814953495, "cycles": [ 0.22116672061843218, 0.2096562507549221, 0.21369707814953495 ] }, "cpu_loaded_pct": { "median": 0.1599616155988858, "cycles": [ 0.1599616155988858, 0.15443295264623955, 0.1661558774373259 ] }, "disk_written_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_charged_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 17.312096050247614, "cycles": [ 17.349892267182025, 17.28458037927286, 17.312096050247614 ] }, "net_in_mb_24h": { "median": 3.6241434376132386, "cycles": [ 3.636741828723276, 3.6241434376132386, 3.613682348109675 ] }, "net_headline_mb_24h": { "median": 17.312096050247614, "cycles": [ 17.349892267182025, 17.28458037927286, 17.312096050247614 ] }, "install_seconds": { "median": 7.9, "cycles": [ 9.1, 7.9, 7.7 ] }, "disk_footprint_mb": { "median": 330.550556, "cycles": [ 330.550556, 330.550556, 330.550556 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 10.0, "cycles": [ 10.0, 10.0, 10.0 ] } } }, { "key": "datadog", "name": "Datadog Agent", "version": "Agent 7.85.3", "versions": [ "Agent 7.85.2", "Agent 7.85.2", "Agent 7.85.3" ], "licence": "Apache-2.0", "sourceUrl": "https://docs.datadoghq.com/agent/basic_agent_usage/ubuntu/", "installUrl": "https://docs.datadoghq.com/agent/basic_agent_usage/ubuntu/", "unitNames": [ "datadog-agent-action-exp.service", "datadog-agent-action.service", "datadog-agent-data-plane-exp.service", "datadog-agent-data-plane.service", "datadog-agent-exp.service", "datadog-agent-installer-exp.service", "datadog-agent-installer.service", "datadog-agent-process-exp.service", "datadog-agent-process.service", "datadog-agent-procmgr-exp.service", "datadog-agent-procmgr.service", "datadog-agent-security-exp.service", "datadog-agent-security.service", "datadog-agent-sysprobe-exp.service", "datadog-agent-sysprobe.service", "datadog-agent-trace-exp.service", "datadog-agent-trace.service", "datadog-agent.service" ], "reportsTo": "Datadog intake (HTTPS)", "intervalSeconds": 15, "us": false, "footnotes": [ "Every datadog-agent* unit is summed (7.84+ runs a process manager).", "The memory peak is the sum of each datadog-agent* unit's own high-water mark over the cycle; those peaks did not happen at the same moment, so the figure overstates the coincident peak.", "Run 3 (box-4) installed Agent 7.85.3; runs 1 and 2 installed 7.85.2. The vendor's documented install takes the release that is current at install time." ], "runs": [ { "cycle": 1, "box": "box-7", "start": 1794088044, "end": 1794178044, "metadataRecord": false }, { "cycle": 2, "box": "box-1", "start": 1794270727, "end": 1794360727, "metadataRecord": false }, { "cycle": 3, "box": "box-4", "start": 1794362986, "end": 1794452986, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 195.680256, "cycles": [ 195.680256, 195.60448, 196.02432 ] }, "pss_idle_p95_mb": { "median": 199.68, "cycles": [ 199.657472, 199.68, 200.79616 ] }, "rss_idle_median_mb": { "median": 205.615104, "cycles": [ 205.615104, 205.275136, 205.889536 ] }, "memory_current_idle_median_mb": { "median": 114.118656, "cycles": [ 113.72544, 114.427904, 114.118656 ] }, "memory_peak_mb": { "median": 198.627328, "cycles": [ 198.508544, 198.627328, 202.24 ] }, "first_hour_peak_mb": { "median": 279.977984, "cycles": [ 279.977984, 275.319808, 281.796608 ] }, "cpu_idle_pct": { "median": 1.588238796956154, "cycles": [ 1.66755283488344, 1.588238796956154, 1.5064854873777025 ] }, "cpu_loaded_pct": { "median": 1.137333704735376, "cycles": [ 1.3429666016713093, 1.137333704735376, 1.0724106963788302 ] }, "disk_written_mb_24h": { "median": 103.22311838627854, "cycles": [ 103.22311838627854, 103.25304060876918, 103.20601997342675 ] }, "disk_charged_mb_24h": { "median": 103.22311838627854, "cycles": [ 103.22311838627854, 103.25304060876918, 103.20601997342675 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 29.295362086956523, "cycles": [ 29.299902260869565, 29.293206260869567, 29.295362086956523 ] }, "net_out_mb_24h": { "median": 89.30747427467085, "cycles": [ 89.24474008938279, 89.31472836816042, 89.30747427467085 ] }, "net_in_mb_24h": { "median": 57.58930896968233, "cycles": [ 57.58930896968233, 57.64651205218021, 57.50647183476266 ] }, "net_headline_mb_24h": { "median": 89.30747427467085, "cycles": [ 89.24474008938279, 89.31472836816042, 89.30747427467085 ] }, "install_seconds": { "median": 17.9, "cycles": [ 18.6, 17.9, 17.0 ] }, "disk_footprint_mb": { "median": 1619.348305, "cycles": [ 1619.348305, 1619.418678, 1619.344397 ] }, "pids_median": { "median": 5.0, "cycles": [ 5.0, 5.0, 5.0 ] }, "threads_median": { "median": 29.0, "cycles": [ 30.0, 29.0, 28.0 ] } } }, { "key": "netdata", "name": "Netdata (self-hosted)", "version": "v2.12.3", "versions": [ "v2.12.3", "v2.12.3", "v2.12.3" ], "licence": "GPL-3.0", "sourceUrl": "https://learn.netdata.cloud/docs/netdata-agent/installation/linux", "installUrl": "https://learn.netdata.cloud/docs/netdata-agent/installation/linux", "unitNames": [ "netdata.service" ], "reportsTo": "nothing (standalone; local dashboard on :19999, not opened)", "intervalSeconds": 1, "us": false, "footnotes": [ "ML anomaly detection stays at its default (on).", "kickstart's default channel is nightly; installed with --stable-channel so the version is a release readers can reproduce.", "Anonymous telemetry and the daily updater timer stay at their defaults. The updater's unit is matched with the agent's; a unit that lives for less than the 10-second sampling interval is counted only as far as the samples catch it.", "The three runs of this row were measured at the same time, on 2026-11-12/13 on box-0, box-3 and box-5, with a collector that records how much of the box's filesystem-metadata writeback the kernel charged to the unit. They replace the runs of 2026-11-07 to 11-11 (`data/2026/superseded/`), which have no such record: their disk figures (254 to 270 MB per 24 h) include an unknown share of that stream. All of this row's figures, not only the disk figure, therefore come from one slot and day. Against the replaced runs' medians, the install time is 26.4 s (was 21.2) and the memory peak 308 MB (294); the other figures moved by less than 2 %. `docs/RUN-2026-slot5.md` has the cause and the rule under which the runs were replaced.", "Disk written counts data writes. Filesystem metadata is written through the block device's inode and is not counted for any row, whichever cgroup the kernel charges it to, and that includes the metadata an agent's own file activity causes. On the six boxes running Netdata the metadata stream was 5.6 to 6.4 MB per hour, on the three other boxes of the same slot 2.8 to 2.9: about 65 to 85 MB per 24 h more, which Netdata's activity appears to cause and which is in no row's figure.", "The disk figure is probably about a tenth too low. The record of the metadata writeback counts whole pages per pass and reads above what the kernel charges: by 11 % on a box where the two could be compared exactly, and by about 13 % (between 6 and 21 %) in the hours of the six Netdata runs. An hour's metadata is never taken out beyond that hour's charge, but this unit writes more than the stream in every hour, so the excess is not caught: about 12 to 15 MB per 24 h in each run at 13 %, 6 to 22 MB over that range.", "Run 1 (box-0): idle CPU is 5.4 % of one vCPU against 3.8 % in runs 2 and 3; under load the three runs agree (2.9, 2.7 and 2.9 %). Idle CPU depends on the box: each of the four rows measured on box-0, box-3 and box-6 shows 1.3 to 1.5 times the idle CPU time on box-0 and box-6 (on box-0 against box-3: HetrixTools 17.4 against 12.6 %, Glances 2.49 against 1.83, Beszel 0.17 against 0.12, Pulse 0.07 against 0.05), and this run's ratio to runs 2 and 3 is 1.42. Why the two boxes differ is not established (`docs/RUN-2026-slot5.md`, \"Deviations\"). The median is not affected.", "Run 1 (box-0): 109.2 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (247.9 MB per 24 h charged in all).", "Run 2 (box-3): 124.0 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (260.2 MB per 24 h charged in all).", "Run 3 (box-5): 102.2 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (242.0 MB per 24 h charged in all)." ], "runs": [ { "cycle": 1, "box": "box-0", "start": 1794506024, "end": 1794596024, "metadataRecord": true }, { "cycle": 2, "box": "box-3", "start": 1794506480, "end": 1794596480, "metadataRecord": true }, { "cycle": 3, "box": "box-5", "start": 1794506796, "end": 1794596796, "metadataRecord": true } ], "values": { "pss_idle_median_mb": { "median": 357.070336, "cycles": [ 355.90656, 360.489984, 357.070336 ] }, "pss_idle_p95_mb": { "median": 367.584256, "cycles": [ 365.449216, 374.315008, 367.584256 ] }, "rss_idle_median_mb": { "median": 439.549952, "cycles": [ 438.648832, 442.85952, 439.549952 ] }, "memory_current_idle_median_mb": { "median": 248.020992, "cycles": [ 247.916544, 252.93824, 248.020992 ] }, "memory_peak_mb": { "median": 307.990528, "cycles": [ 300.187648, 307.990528, 317.41952 ] }, "first_hour_peak_mb": { "median": 311.103488, "cycles": [ 311.957504, 308.81792, 311.103488 ] }, "cpu_idle_pct": { "median": 3.810787392197125, "cycles": [ 5.388088683415872, 3.786408688247373, 3.810787392197125 ] }, "cpu_loaded_pct": { "median": 2.8519834818941505, "cycles": [ 2.8887178551532036, 2.6631734818941504, 2.8519834818941505 ] }, "disk_written_mb_24h": { "median": 138.70659965696342, "cycles": [ 138.70659965696342, 136.22305519024036, 139.7667012537746 ] }, "disk_charged_mb_24h": { "median": 247.87141650924025, "cycles": [ 247.87141650924025, 260.1908229689576, 241.9895624882232 ] }, "disk_metadata_charged_mb_24h": { "median": 109.16481685227684, "cycles": [ 109.16481685227684, 123.96776777871723, 102.22286123444862 ] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_in_mb_24h": { "median": 0.00015027901920521803, "cycles": [ 0.0007346974272255103, 0.00015027901920521803, 6.261625800217417e-05 ] }, "net_headline_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "install_seconds": { "median": 26.4, "cycles": [ 26.4, 23.1, 27.5 ] }, "disk_footprint_mb": { "median": 794.456005, "cycles": [ 795.204442, 794.456005, 791.568411 ] }, "pids_median": { "median": 17.0, "cycles": [ 17.0, 17.0, 17.0 ] }, "threads_median": { "median": 149.0, "cycles": [ 149.0, 149.0, 148.0 ] } } }, { "key": "netdata-cloud", "name": "Netdata (claimed to Netdata Cloud)", "version": "v2.12.3", "versions": [ "v2.12.3", "v2.12.3", "v2.12.3" ], "licence": "GPL-3.0", "sourceUrl": "https://learn.netdata.cloud/docs/netdata-agent/installation/linux", "installUrl": "https://learn.netdata.cloud/docs/netdata-agent/installation/linux", "unitNames": [ "netdata.service" ], "reportsTo": "Netdata Cloud (ACLK, outbound)", "intervalSeconds": 1, "us": false, "footnotes": [ "Same binary as the self-hosted row, claimed to Netdata Cloud. ML anomaly detection stays at its default (on).", "kickstart's default channel is nightly; installed with --stable-channel so the version is a release readers can reproduce.", "Anonymous telemetry and the daily updater timer stay at their defaults. The updater's unit is matched with the agent's; a unit that lives for less than the 10-second sampling interval is counted only as far as the samples catch it.", "The three runs of this row were measured at the same time, on 2026-11-13/14 on box-2, box-4 and box-7, with a collector that records how much of the box's filesystem-metadata writeback the kernel charged to the unit. They are the third measurement of this row. The first runs, of 2026-11-07 to 11-11 (`data/2026/superseded/`), have no such record: their disk figures (231 to 276 MB per 24 h) include an unknown share of that stream. The runs that replaced them, of 2026-11-12/13 (`data/2026-slot5/`), carry traffic that the benchmark's operator caused (see the note on the Netdata Cloud Space below) and were measured again on the same three boxes. The rule fixed before each retest is that every run measured again replaces the run it stands for, whatever it shows, so all of this row's figures, not only the network figure, come from this slot and day. Against the medians of the runs of 2026-11-12/13, which were the published ones from 2026-11-13 until these runs replaced them: network 0.34 MB out and 0.19 in per 24 h (was 0.55 and 0.25), disk written 140.8 MB per 24 h (148.5), install time 25.7 s (30.2), CPU under load 2.9 % (3.2); the other figures moved by 1 % or less, except the kernel's whole charge of disk writes (233.3 MB per 24 h against 223.7) and the metadata taken out of it (90.3 against 75.2), which depend on how long the unit held the metadata stream. Against the first runs' medians: idle CPU 3.9 % (3.7) and CPU under load 2.9 % (2.8); the other figures, apart from the disk figures, are within 1.2 %. `docs/RUN-2026-slot6.md` and `docs/RUN-2026-slot5.md` have the causes and the rule.", "Disk written counts data writes. Filesystem metadata is written through the block device's inode and is not counted for any row, whichever cgroup the kernel charges it to, and that includes the metadata an agent's own file activity causes. On a box running Netdata the metadata stream was 5.6 to 6.4 MB per hour (the six such boxes of the slot of 2026-11-12/13; 5.7 to 5.8 on the three boxes of these runs), on the three boxes without Netdata in the slot of 2026-11-12/13 2.8 to 2.9: about 65 to 85 MB per 24 h more, which Netdata's activity appears to cause and which is in no row's figure.", "The disk figure is probably 10 to 16 MB per 24 h too low in each run. The record of the metadata writeback counts whole pages per pass and reads above what the kernel charges: by 11 % on a box where the two could be compared exactly, and by about 13 % (between 6 and 21 %) in the hours of the six Netdata runs of 2026-11-12/13. These three runs agree with those six where they can be compared: in the samples in which the unit was charged no more than the record shows, the charge is 88 to 90 % of the record. An hour's metadata is never taken out beyond that hour's charge, but this unit writes more than the stream in every hour, so the excess is not caught: 10 to 16 MB per 24 h in each run at 13 %, 5 to 24 MB over that range.", "The Netdata Cloud Space the nodes are claimed to belongs to the benchmark's operator. In the runs of 2026-11-12/13 all three nodes sent more than usual at the same two moments, 20:06:50Z and 20:53:50 to 20:54:50Z on 2026-11-12, which raised this row's network figure from 0.3 to 0.6 MB per 24 h; when those runs were published the cause was not established. It was read on 2026-11-13 from Netdata's own access log on the three boxes: 53, 260 and 52 requests over the Netdata Cloud link, in those minutes and in no other, almost all for chart data, every one from the same signed-in user of the Space, with the role admin. The bursts were the operator looking at the Space's dashboard: a harness incident in this row. The three runs were therefore measured again with nobody opening the Space, and each box's access log was read after the collect and before any rebuild: no request over the Netdata Cloud link on any of the three boxes from the install to the collect (`docs/evidence/2026-slot6/`).", "Run 1 (box-2): one 10-second sample of the idle window, at 23:42:40Z on 2026-11-13, has 21 KB sent and 0.6 KB received, where a usual minute has 0.2 KB sent. It is not a request over the Netdata Cloud link: the access log has none at that time, and the requests of 2026-11-12 came with 2 to 95 KB received per sample. The same sample has a disk write of 2.2 MB and two more threads; six of the nine runs measured for this row show that pair ten minutes after the install (a write of 1.8 to 2.3 MB with 13 to 16 KB sent), so it may be the agent's own housekeeping. The harness's check had logged in to the three boxes over SSH between 23:40:02Z and 23:40:06Z; on this box that was 45 seconds into the idle window, on the two others still in the first hour, and in no other run of this row did that check fall in the idle window. The two other nodes sent nothing of the kind after the first eleven minutes of their runs. The cause is not established. It is why this run's figure is 0.36 MB per 24 h against 0.34 in runs 2 and 3; the median is not affected.", "Run 1 (box-2): 89.7 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (233.3 MB per 24 h charged in all).", "Run 2 (box-4): 90.3 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (231.0 MB per 24 h charged in all).", "Run 3 (box-7): 139.3 MB per 24 h of filesystem-metadata writeback were charged to this unit's cgroup and are not counted as its disk writes (275.7 MB per 24 h charged in all)." ], "runs": [ { "cycle": 1, "box": "box-2", "start": 1794609557, "end": 1794699557, "metadataRecord": true }, { "cycle": 2, "box": "box-4", "start": 1794609712, "end": 1794699712, "metadataRecord": true }, { "cycle": 3, "box": "box-7", "start": 1794609863, "end": 1794699863, "metadataRecord": true } ], "values": { "pss_idle_median_mb": { "median": 358.166528, "cycles": [ 358.728704, 357.197824, 358.166528 ] }, "pss_idle_p95_mb": { "median": 367.730688, "cycles": [ 367.975424, 366.524416, 367.730688 ] }, "rss_idle_median_mb": { "median": 440.928256, "cycles": [ 441.155584, 439.771136, 440.928256 ] }, "memory_current_idle_median_mb": { "median": 252.878848, "cycles": [ 254.04416, 252.878848, 252.481536 ] }, "memory_peak_mb": { "median": 293.093376, "cycles": [ 300.122112, 290.185216, 293.093376 ] }, "first_hour_peak_mb": { "median": 312.327168, "cycles": [ 312.660992, 312.327168, 311.973888 ] }, "cpu_idle_pct": { "median": 3.89493076579297, "cycles": [ 3.8989983560816524, 3.89493076579297, 3.760002438700326 ] }, "cpu_loaded_pct": { "median": 2.9010003342618385, "cycles": [ 3.0723808913649027, 2.8993215598885795, 2.9010003342618385 ] }, "disk_written_mb_24h": { "median": 140.75840919917863, "cycles": [ 143.61384414542817, 140.75840919917863, 136.33846947698996 ] }, "disk_charged_mb_24h": { "median": 233.34204018842854, "cycles": [ 233.34204018842854, 231.01665604058462, 275.67771040946974 ] }, "disk_metadata_charged_mb_24h": { "median": 90.25824684140598, "cycles": [ 89.72819604300037, 90.25824684140598, 139.33924093247978 ] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 0.33897519990337, "cycles": [ 0.3582850102669405, 0.3382102379514434, 0.33897519990337 ] }, "net_in_mb_24h": { "median": 0.18998711921729677, "cycles": [ 0.189676125135886, 0.18998711921729677, 0.1901520086967025 ] }, "net_headline_mb_24h": { "median": 0.33897519990337, "cycles": [ 0.3582850102669405, 0.3382102379514434, 0.33897519990337 ] }, "install_seconds": { "median": 25.7, "cycles": [ 31.0, 25.7, 25.2 ] }, "disk_footprint_mb": { "median": 795.17023, "cycles": [ 795.166443, 795.17023, 796.311282 ] }, "pids_median": { "median": 17.0, "cycles": [ 17.0, 17.0, 17.0 ] }, "threads_median": { "median": 148.0, "cycles": [ 148.0, 151.0, 148.0 ] } } } ], "control": { "key": "control", "name": "No agent (harness only)", "version": "none", "versions": [ "none", "none", "none" ], "licence": "Apache-2.0", "sourceUrl": "https://github.com/monitorable-io/agent-footprint-benchmark", "installUrl": "https://github.com/monitorable-io/agent-footprint-benchmark", "unitNames": [], "reportsTo": "nothing: the control box measures the harness itself", "intervalSeconds": 10, "us": false, "footnotes": [ "Control row: no agent; the collector measures its own cgroup." ], "runs": [ { "cycle": 1, "box": "box-2", "start": 1794179024, "end": 1794269024, "metadataRecord": false }, { "cycle": 2, "box": "box-5", "start": 1794271243, "end": 1794361243, "metadataRecord": false }, { "cycle": 3, "box": "box-8", "start": 1794363552, "end": 1794453552, "metadataRecord": false } ], "values": { "pss_idle_median_mb": { "median": 12.640256, "cycles": [ 12.638208, 12.640256, 12.654592 ] }, "pss_idle_p95_mb": { "median": 12.640256, "cycles": [ 12.638208, 12.640256, 12.654592 ] }, "rss_idle_median_mb": { "median": 18.51392, "cycles": [ 18.296832, 18.51392, 18.51392 ] }, "memory_current_idle_median_mb": { "median": 11.173888, "cycles": [ 11.190272, 11.173888, 11.14112 ] }, "memory_peak_mb": { "median": 16.11776, "cycles": [ 16.146432, 16.109568, 16.11776 ] }, "first_hour_peak_mb": { "median": 12.640256, "cycles": [ 12.638208, 12.640256, 12.654592 ] }, "cpu_idle_pct": { "median": 0.06726969682328784, "cycles": [ 0.06726969682328784, 0.06995171397511778, 0.0671985686677135 ] }, "cpu_loaded_pct": { "median": 0.056501253481894144, "cycles": [ 0.05288370473537604, 0.056501253481894144, 0.0575825069637883 ] }, "disk_written_mb_24h": { "median": 10.51979850706607, "cycles": [ 10.51979850706607, 10.528347713491966, 10.51979850706607 ] }, "disk_charged_mb_24h": { "median": 10.51979850706607, "cycles": [ 10.51979850706607, 10.528347713491966, 10.51979850706607 ] }, "disk_metadata_charged_mb_24h": { "median": null, "cycles": [] }, "journal_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_out_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_in_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "net_headline_mb_24h": { "median": 0.0, "cycles": [ 0.0, 0.0, 0.0 ] }, "install_seconds": { "median": null, "cycles": [] }, "disk_footprint_mb": { "median": 0.065795, "cycles": [ 0.065795, 0.065795, 0.065795 ] }, "pids_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] }, "threads_median": { "median": 1.0, "cycles": [ 1.0, 1.0, 1.0 ] } } } }