3600, 'days' => 86400));
// Whether $every is a Repeat that runs; a dataset's None (ZFSMScheduleSnapshot.php) never does.
function zfsmScheduleRepeats($every) {
return is_string($every) && isset(ZFSM_SCHEDULE_PERIODS[$every]);
}
// How long a run waits out another replication of its dataset: less than any schedule's period, so never its own next run.
define('ZFSM_SCHEDULE_WAIT', 3300);
// On the flash drive beside the settings, so both survive a restart: Unraid builds its crontab from the .cron file at boot.
function zfsmScheduleFile() {
return zfsmStatePath('/boot/config/plugins/zfs.master/schedules.json');
}
function zfsmScheduleCronFile() {
return zfsmStatePath('/boot/config/plugins/zfs.master/schedules.cron');
}
// A run's failures in a row, in RAM: a restart counts again from none.
function zfsmScheduleStateFile($id) {
return zfsmJobDir('schedule').'/'.$id.'.json';
}
function zfsmScheduleId($id) {
return is_string($id) && preg_match('/^[0-9a-f]{8}$/D', $id) === 1;
}
// A copy's schedule, a snapshot schedule (dest and server ''; zfsmScheduleSnapshotValid) or copies under a dataset (dest ends in /;
// zfsmScheduleCopiesValid). One saved before Keep has no keep: it keeps all.
function zfsmScheduleValid($job) {
if (!is_array($job) || !zfsmScheduleId($job['id'] ?? null)) return false;
$what = ($job['dest'] ?? null) === '' ? zfsmScheduleSnapshotValid($job) : (zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesValid($job) : zfsmScheduleCopyValid($job) && zfsmScheduleRepeats($job['every'] ?? null));
$number = function($value, $top) { return is_int($value) && $value >= 0 && $value <= $top; };
return $what && $number($job['minute'] ?? null, 59) && $number($job['hour'] ?? null, 23) && zfsmScheduleDaysValid($job['days'] ?? null) && zfsmKeepValid($job['keep'] ?? 0);
}
// The copy is a dataset in a pool here, or one name inside the area on a trusted server (zfsmReplicaTargetDest).
function zfsmScheduleCopyValid($job) {
if (!zfsmValidDatasetName($job['dataset'] ?? null)) return false;
$dest = $job['dest'] ?? null;
$server = $job['server'] ?? null;
return $server === '' ? zfsmValidDatasetName($dest) && strpos($dest, '/') !== false : zfsmReplicaSenderName($server) && zfsmValidNameComponent($dest);
}
// Days of the week, 0 (Sunday) to 6: at least one, in order, each once.
function zfsmScheduleDaysValid($days) {
if (!is_array($days) || $days === array() || array_keys($days) !== range(0, count($days) - 1)) return false;
$last = -1;
foreach ($days as $day) {
if (!is_int($day) || $day <= $last || $day > 6) return false;
$last = $day;
}
return true;
}
// A record saved before start times and days of the week, with the day 0 to 6 that version needed of each: its minute came from
// its id, and a daily one ran each day, a weekly one on its day.
function zfsmScheduleUpgrade($job) {
if (!is_array($job) || array_key_exists('days', $job) || !zfsmScheduleId($job['id'] ?? null) || !in_array($job['every'] ?? null, array('hourly', 'daily', 'weekly'), true)
|| !in_array($job['day'] ?? null, range(0, 6), true)) return $job;
$days = $job['every'] === 'weekly' ? array($job['day']) : range(0, 6);
unset($job['day']);
return array('every' => $job['every'] === 'hourly' ? 'hourly' : 'days', 'minute' => hexdec(substr($job['id'], 0, 4)) % 60, 'days' => $days) + $job;
}
function zfsmScheduleUnreadable() {
return zfsmScheduleFile().' could not be read as a list of schedules; fix or remove it first';
}
// schedules.json, or $text read from it, as 'valid' schedules by id and 'other' entries kept for a write; null when unread or no list.
function zfsmScheduleRead($text = null) {
$file = zfsmScheduleFile();
if ($text === null && !file_exists($file)) return array('valid' => array(), 'other' => array());
if ($text === null) $text = (string)@file_get_contents($file);
// Read as objects first: as arrays, {} is an empty list and one schedule's record a list of its fields.
if (!is_array(json_decode($text))) return null;
$read = array('valid' => array(), 'other' => array());
foreach (json_decode($text, true) as $record) {
$job = zfsmScheduleUpgrade($record);
if (!zfsmScheduleValid($job) || isset($read['valid'][$job['id']])) {
$read['other'][] = $record;
continue;
}
$read['valid'][$job['id']] = array('id' => $job['id'], 'dataset' => $job['dataset'], 'dest' => $job['dest'], 'server' => $job['server'], 'every' => $job['every'],
'minute' => $job['minute'], 'hour' => $job['hour'], 'days' => $job['days'], 'keep' => $job['keep'] ?? 0);
}
return $read;
}
// $change gets the schedules and returns them changed, or a string saying why not; under the lock, written whole with the entries
// in other shapes as they were, then the cron file to match. A file that can't be read is never written over.
// Answers 'saved' (schedules.json was written) and 'why': '', why it was not, or what failed after.
function zfsmScheduleChange($change) {
$unsaved = function($why) { return array('saved' => false, 'why' => $why); };
$dir = zfsmJobDir('schedule');
$lock = zfsmJobLock($dir.'/settings.lock', false);
if ($lock === null) return $unsaved('Could not open the schedule lock in '.$dir);
if ($lock === false) return $unsaved('Another change to the schedules is being saved; try again');
try {
$read = zfsmScheduleRead();
if ($read === null) return $unsaved(zfsmScheduleUnreadable());
$jobs = $change($read['valid']);
if (is_string($jobs)) return $unsaved($jobs);
$file = zfsmScheduleFile();
if (!zfsmWriteFileAtomic($file, zfsmJson(array_merge(array_values($jobs), $read['other']), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n")) return $unsaved('Could not save '.$file);
return array('saved' => true, 'why' => zfsmScheduleCron($jobs));
} finally {
zfsmJobUnlock($lock);
}
}
function zfsmScheduleNewId($jobs) {
do $id = bin2hex(random_bytes(4)); while (isset($jobs[$id]));
return $id;
}
// An hourly $job's minute: $prefer, or another, that no schedule sharing a dataset with it (zfsmScheduleShare) starts at; any with
// none free.
function zfsmScheduleFreeMinute($jobs, $job, $prefer = null) {
$free = array_diff(range(0, 59), array_column(array_filter($jobs, function($other) use ($job, $jobs) { return zfsmScheduleShare($job, $other, $jobs); }), 'minute'));
if (in_array($prefer, $free, true)) return $prefer;
return $free ? $free[array_rand($free)] : random_int(0, 59);
}
// Whether $a and $b can snapshot one dataset (so not in one minute or name): both of it, or one that covers (zfsmScheduleCovers) of
// one above it or the default that it follows among $jobs (zfsmScheduleFollowed): a nearer one of its kind, None too, takes it.
function zfsmScheduleShare($a, $b, $jobs) {
if (!zfsmScheduleRepeats($a['every']) || !zfsmScheduleRepeats($b['every'])) return false;
if (zfsmScheduleCovers($a) && zfsmScheduleCovers($b) && zfsmScheduleKindOf($a) === zfsmScheduleKindOf($b)) return false;
$in = function($job, $other) use ($jobs) {
if (!zfsmScheduleCovers($job) || ($job['dataset'] !== '' && !zfsmUnderAny($other['dataset'], array($job['dataset'])))) return false;
$near = zfsmScheduleFollowed($other['dataset'], array_diff_key($jobs, array($job['id'] => 0)) + array($job['id'] => $job), zfsmScheduleKindOf($job));
return $near['id'] === $job['id'];
};
return $a['dataset'] === $b['dataset'] || $in($a, $b) || $in($b, $a);
}
// What a schedule does, inside a sentence.
function zfsmScheduleWhat($job) {
if (zfsmScheduleCopiesUnder($job)) return $job['dataset'] === '' ? 'Replicate Every Dataset' : 'the copies of '.$job['dataset'].' and below';
if (!zfsmScheduleSnapshotsOnly($job)) return 'the copy to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']);
return $job['dataset'] === '' ? 'Snapshot Every Dataset' : 'the snapshot schedule of '.$job['dataset'];
}
// Whether $a and $b ever start in the same minute.
function zfsmScheduleSameTime($a, $b) {
if ($a['minute'] !== $b['minute']) return false;
return $a['every'] === 'hourly' || $b['every'] === 'hourly' || ($a['hour'] === $b['hour'] && array_intersect($a['days'], $b['days']) !== array());
}
// When cron next runs $job after $now, in this server's time zone as cron reads it: the minute first, then each hour.
function zfsmScheduleNext($job, $now) {
$at = $now - $now % 60 + 60;
$at += ($job['minute'] - (int)date('i', $at) + 60) % 60 * 60;
while ($job['every'] !== 'hourly' && ((int)date('G', $at) !== $job['hour'] || !in_array((int)date('w', $at), $job['days'], true))) $at += 3600;
return $at;
}
// Cron's runs of $job after $from and before $until; the next is looked for from 2 hours short of a period on (hourly: a minute
// on), so a change of summer time skips none.
function zfsmScheduleRuns($job, $from, $until) {
$runs = array();
if (!zfsmScheduleRepeats($job['every'])) return $runs;
for ($at = zfsmScheduleNext($job, $from); $at < $until; $at = zfsmScheduleNext($job, $at + max(60, ZFSM_SCHEDULE_PERIODS[$job['every']] - 7200))) $runs[] = $at;
return $runs;
}
// Why Snapshots Pattern gives one of $job's runs in the next two weeks the name of another snapshot it or a schedule in $jobs
// sharing a dataset with it takes, so that run can't take its own; $first: Repeat's run at $now is one. '' when none does.
function zfsmScheduleClash($job, $jobs, $cfg, $now, $first) {
$until = $now + 14 * 86400 + 3600;
$named = $first ? array(zfsmNewSnapshotName($cfg, $now) => 'this run') : array();
foreach ($jobs as $other) {
if ($other['id'] === $job['id'] || !zfsmScheduleShare($job, $other, $jobs)) continue;
foreach (zfsmScheduleRuns($other, $now, $until) as $at) $named[zfsmNewSnapshotName($cfg, $at)] = 'the run at '.date('D j M H:i', $at).' of '.zfsmScheduleWhat($other);
}
foreach (zfsmScheduleRuns($job, $now, $until) as $at) {
$name = zfsmNewSnapshotName($cfg, $at);
if (isset($named[$name])) return 'Snapshots Pattern would give the run at '.date('D j M H:i', $at).' the same snapshot name as '.$named[$name];
$named[$name] = 'the run at '.date('D j M H:i', $at);
}
return '';
}
// Each line names only its schedule's id; None has none. '' with no schedules.
function zfsmScheduleCronText($jobs) {
$lines = array();
foreach ($jobs as $job) {
if (!zfsmScheduleRepeats($job['every'])) continue;
$hour = $job['every'] === 'hourly' ? '*' : $job['hour'];
$days = $job['every'] === 'hourly' || count($job['days']) === 7 ? '*' : implode(',', $job['days']);
$lines[] = $job['minute'].' '.$hour.' * * '.$days.' '.ZFSM_SCHEDULE_SCRIPT.' run '.$job['id'];
}
return zfsmCronText('schedules', $lines);
}
// Writes the cron file for $jobs (with none, removes it) and has Unraid rebuild its crontab from every plugin's; '' or why not.
function zfsmScheduleCron($jobs) {
$file = zfsmScheduleCronFile();
$text = zfsmScheduleCronText($jobs);
$written = $text === '' ? !file_exists($file) || @unlink($file) : zfsmWriteFileAtomic($file, $text);
if (!$written) return 'The schedules were saved, but '.$file.' could not be written, so cron still runs the ones it had';
// Unraid's update_cron begins "#/bin/bash", with no "!": PHP 8.3 and later can't start it on its own (Exec format error).
$built = runProcess('/bin/bash', array(zfsmStatePath('/usr/local/sbin/update_cron')));
if (($built['code'] ?? 1) === 0) return '';
return 'The schedules were saved, but Unraid\'s crontab could not be rebuilt ('.zfsmCommandError($built, 'update_cron failed').'), so it runs the ones it had until the server restarts';
}
// Its days Monday first, as the dialog shows them.
function zfsmScheduleWhen($job) {
if (!zfsmScheduleRepeats($job['every'])) return 'never';
$minute = sprintf('%02d', $job['minute']);
if ($job['every'] === 'hourly') return 'every hour at :'.$minute;
$at = sprintf('%02d', $job['hour']).':'.$minute;
if (count($job['days']) === 7) return 'every day at '.$at;
$days = $job['days'];
if ($days[0] === 0) $days[] = array_shift($days);
$names = array_map(function($day) { return ZFSM_SCHEDULE_DAYS[$day]; }, $days);
$last = array_pop($names);
return 'every '.($names ? implode(', ', $names).' and ' : '').$last.' at '.$at;
}
function zfsmScheduleKeepText($job) {
$kept = zfsmKeepWords($job['keep']);
if (zfsmScheduleSnapshotsOnly($job)) return $kept !== '' ? 'keeps '.$kept.' scheduled snapshots of each dataset' : 'keeps all it takes';
if (zfsmScheduleCopiesUnder($job)) return $kept !== '' ? 'keeps '.$kept.' of the snapshots of each dataset it copies' : 'keeps all its snapshots';
return $kept !== '' ? 'keeps '.$kept.' of its snapshots' : 'keeps all its snapshots';
}
// The page's $time ('13:30') and $days ('1,3,5'; 0 is Sunday) as a schedule's minute, hour and days: -1 and none when unread.
function zfsmScheduleAt($every, $time, $days) {
if ($every === 'hourly') return array('minute' => 0, 'hour' => 0, 'days' => range(0, 6));
$at = preg_match('/^([01][0-9]|2[0-3]):([0-5][0-9])$/D', $time, $parts) === 1 ? array((int)$parts[2], (int)$parts[1]) : array(-1, -1);
$days = preg_match('/^[0-6](,[0-6])*$/D', $days) === 1 ? array_values(array_unique(array_map('intval', explode(',', $days)))) : array();
sort($days);
return array('minute' => $at[0], 'hour' => $at[1], 'days' => $days);
}
// What the user fixes for $job to make a schedule: a Repeat the page no longer offers, else each wrong time, days or Keep, else the copy's name; '' when it makes one.
function zfsmScheduleWrong($job) {
if (!zfsmScheduleRepeats($job['every'])) return 'Repeat has no such choice now; reload the page and pick again';
$fix = array();
if ($job['minute'] < 0) $fix[] = 'set the time it starts at';
if ($job['days'] === array()) $fix[] = 'tick at least one day';
if (!zfsmKeepValid($job['keep'])) $fix[] = 'under Snapshots to keep, '.zfsmKeepFix();
if ($fix === array() && !zfsmScheduleValid($job)) $fix[] = 'give the copy a name as Replicate does';
return implode('; ', $fix);
}
// Saves a schedule for a copy and starts its first run now with Keep 0, as it may never be saved: one whose first run can't
// start isn't. $keep: zfsmKeepParse's, -1 when what the page sent is no Keep; an empty one is 0, which keeps all.
function zfsmScheduleAdd($zdataset, $snapshot, $zdest, $server, $every, $time, $days, $keep, $cfg) {
$job = array('id' => '00000000', 'dataset' => $zdataset, 'dest' => $zdest, 'server' => $server, 'every' => $every) + zfsmScheduleAt($every, $time, $days) + array('keep' => $keep);
$wrong = zfsmScheduleWrong($job);
if ($wrong !== '') return zfsmFailure($zdataset, 'Nothing was scheduled: '.$wrong);
if ($snapshot !== '') return zfsmFailure($zdataset, 'Nothing was scheduled: a schedule takes a new snapshot each run, so pick Take one now');
$read = zfsmScheduleRead();
if ($read === null) return zfsmFailure($zdataset, 'Nothing was scheduled: '.zfsmScheduleUnreadable());
if ($every === 'hourly') $job['minute'] = zfsmScheduleFreeMinute($read['valid'], $job);
$same = zfsmScheduleSame($read['valid'], $job);
if ($same !== '') return zfsmFailure($zdataset, 'Nothing was scheduled: '.$same);
$job['id'] = zfsmScheduleNewId($read['valid']);
$clash = zfsmScheduleClash($job, $read['valid'], $cfg, time(), true);
if ($clash !== '') return zfsmFailure($zdataset, 'Nothing was scheduled: '.$clash.'; add the time to it in Settings');
$started = zfsmStartReplicate($zdataset, '', $zdest, $cfg, $server, 0);
if (!isset($started['succeeded'][$zdataset])) return zfsmFailure($zdataset, 'Nothing was scheduled: '.($started['failed'][$zdataset] ?? 'the first run did not start'));
$changed = zfsmScheduleChange(function($jobs) use ($job) {
$same = zfsmScheduleSame($jobs, $job);
if ($same !== '') return $same;
$jobs[$job['id']] = $job;
return $jobs;
});
$said = $started['succeeded'][$zdataset];
if ($changed['why'] === '') return array('succeeded' => array($zdataset => $said.'. It runs again '.zfsmScheduleWhen($job)), 'failed' => array());
return array('succeeded' => array($zdataset => $said), 'failed' => array($zdataset => ($changed['saved'] ? '' : 'It was not scheduled: ').$changed['why']));
}
// Why $job can't be added beside $jobs: one for the same copy already runs it, or one sharing a dataset with it starts in the same
// minute (two at once can get one snapshot name). '' when none does.
function zfsmScheduleSame($jobs, $job) {
foreach ($jobs as $other) {
if ($other['dataset'] === $job['dataset'] && $other['dest'] === $job['dest'] && $other['server'] === $job['server']) return 'this copy already has a schedule ('.zfsmScheduleWhen($other).'); remove it in Settings first';
}
foreach ($jobs as $other) {
if (zfsmScheduleShare($job, $other, $jobs) && zfsmScheduleSameTime($job, $other)) return zfsmScheduleWhat($other).' ('.zfsmScheduleWhen($other).') starts then too; pick another time';
}
return '';
}
// As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt after; not when the schedule above
// that its datasets then follow clashes with one as saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg).
function zfsmScheduleRemove($id, $cfg) {
if (!zfsmScheduleId($id)) return array('saved' => false, 'why' => 'No such schedule');
$changed = zfsmScheduleChange(function($jobs) use ($id, $cfg) {
if (!isset($jobs[$id])) return 'No such schedule; it may have been removed already';
$after = array_diff_key($jobs, array($id => 0));
$clash = zfsmScheduleNewClash($jobs, $after, $cfg);
return $clash === '' ? $after : 'Not removed: '.$clash;
});
if ($changed['saved']) @unlink(zfsmScheduleStateFile($id));
return $changed;
}
// The runs in a row with a problem, and whether Unraid took the notification of the first; a record with no 'told' was told.
function zfsmScheduleState($id) {
$state = json_decode((string)@file_get_contents(zfsmScheduleStateFile($id)), true);
$failures = is_array($state) && is_int($state['failures'] ?? null) && $state['failures'] > 0 ? $state['failures'] : 0;
return array('failures' => $failures, 'told' => ($state['told'] ?? true) !== false);
}
// As Settings and diagnostics show it: None never runs to clear a count that a run going as it was saved left.
function zfsmScheduleProblems($job) {
return zfsmScheduleRepeats($job['every']) ? zfsmScheduleFailures($job['id']) : 0;
}
function zfsmScheduleFailures($id) {
return zfsmScheduleState($id)['failures'];
}
// A scheduled run notifies of its first problem (a warning; the next one does if Unraid didn't take it) and of the run that works
// again; each problem is logged with its count. A count that can't be kept notifies of every problem, rather than of none.
function zfsmScheduleNotifier($id) {
return function($title, $text, $type) use ($id) {
$state = zfsmScheduleState($id);
$file = zfsmScheduleStateFile($id);
if ($type === 'normal') {
if ($state['failures'] === 0) return;
@unlink($file);
zfsnotify($title, $text.' It works again after '.$state['failures'].' run(s) with a problem.', '', 'normal');
return;
}
$failures = $state['failures'] + 1;
$told = $failures > 1 && $state['told'];
if (!is_dir(dirname($file))) @mkdir(dirname($file), 0700, true);
$save = function($told) use ($file, $failures) { return zfsmWriteFileAtomic($file, zfsmJson(array('failures' => $failures, 'told' => $told))."\n"); };
$kept = $save($told);
zfsmLog('Schedule '.$id.': '.$failures.' run(s) in a row with a problem');
if (!$kept) zfsnotify($title, $text, '', $type);
elseif (!$told && zfsnotify($title, rtrim($text, '.').'. Later problems of this schedule are logged in /var/log/zfs.master.log, not notified, until a run works again.', '', $type)) $save(true);
};
}
// Cron's run of one schedule: the job read by its id; a copy's run takes its snapshot first, as Replicate's "Take one now" does.
function zfsmScheduleRun($id, $cfg, $now, $wait = ZFSM_SCHEDULE_WAIT) {
$notify = zfsmScheduleNotifier($id);
$read = zfsmScheduleRead();
if ($read === null) {
zfsmLog('Schedule '.$id.' not started: '.zfsmScheduleUnreadable());
$notify('ZFS Schedule', 'Scheduled run not started: '.zfsmSafeText(zfsmScheduleUnreadable()), 'warning');
return 1;
}
$job = $read['valid'][$id] ?? null;
// Its line outlived the schedule, or its Repeat is None now: say, the list was saved and Unraid's crontab was not rebuilt.
if ($job === null || !zfsmScheduleRepeats($job['every'])) {
zfsmLog('Schedule '.$id.' not started: '.($job === null ? 'Settings has no such schedule' : 'it is None now'));
return 0;
}
$copy = !zfsmScheduleSnapshotsOnly($job) && !zfsmScheduleCopiesUnder($job);
try {
if (zfsmScheduleCopiesUnder($job)) return zfsmScheduleCopiesRun($job, $read['valid'], $cfg, $now, $notify, $wait);
if (!$copy) return zfsmScheduleSnapshotRun($job, $read['valid'], $cfg, $now, $notify);
return zfsmRunReplicate($job['dataset'], zfsmNewSnapshotName($cfg, $now), $job['dest'], null, true, $job['server'], array('notify' => $notify, 'wait' => $wait, 'gone' => 'remove its schedule in Settings'), $job['keep']);
} catch (Throwable $error) {
// Cron drops what PHP prints: an error that ends a run is a problem like any other.
$what = $copy ? 'Replication of '.$job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']) : (zfsmScheduleSnapshotsOnly($job) ? 'Scheduled snapshots' : 'Scheduled copies');
zfsmLog($what.' failed: '.zfsmOneLine($error->getMessage()));
$notify(zfsmScheduleSnapshotsOnly($job) ? 'ZFS Snapshot' : 'ZFS Replication', $what.' failed: '.zfsmSafeText(zfsmOneLine($error->getMessage())), 'warning');
return 1;
}
}
// Settings' Days boxes of the row whose ids start $prefix, $days ticked.
function zfsmScheduleDaysHtml($prefix, $days) {
$html = '';
foreach (array(1, 2, 3, 4, 5, 6, 0) as $day) {
$html .= '';
}
return ''.$html.'';
}
// Settings' list; $cfg for Snapshots Pattern, which can change after a schedule was saved.
function zfsmSchedulesHtml($cfg) {
$read = zfsmScheduleRead();
if ($read === null) return ''.zfsmHtml(zfsmScheduleUnreadable()).'';
$html = $read['valid'] ? '' : 'None yet';
foreach ($read['valid'] as $job) {
$failures = zfsmScheduleProblems($job);
$clash = zfsmScheduleClash($job, $read['valid'], $cfg, time(), false);
$text = zfsmScheduleSnapshotsOnly($job) ? zfsmScheduleSnapshotText($job) : (zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesText($job)
: $job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']).', '.zfsmScheduleWhen($job).', '.zfsmScheduleKeepText($job));
// The default changes and turns off in its own row above; a dataset's goes here too, as one whose dataset is gone can't be opened.
$html .= ''.zfsmHtml($text)
.($job['dataset'] !== '' ? ' ' : '')
.($clash !== '' ? ' '.zfsmHtml($clash).', so that run fails: add the time to Snapshots Pattern' : '')
.($failures > 0 ? ' '.$failures.' run(s) in a row with a problem: the first one\'s notification and /var/log/zfs.master.log say why' : '').'';
}
if ($read['other']) $html .= ''.count($read['other']).' record(s) in '.zfsmHtml(zfsmScheduleFile()).' are not schedules the plugin can use';
return $html;
}