-- Behavioral tests for the isolated alert service. local state = {} local watchers = {} local notifications = {} local stateWrites = {} local stateRenames = {} local successfulStateCommits = 0 local failNextStateWrite = false local failNextStateRename = false local fullSmartEnabled = false local function translate(key, substitutions) local value = key for name, replacement in pairs(substitutions or {}) do value = value:gsub("{" .. name .. "}", tostring(replacement)) end if substitutions ~= nil and substitutions.count ~= nil then value = value .. ":count=" .. tostring(substitutions.count) end return value end noctalia = { getConfig = function(key) local values = { alerts_enabled = true, notify_recovery = true, warning_temperature = 65, critical_temperature = 80, life_warning_percent = 20, show_hdd = false, alert_hdd = true, drive_missing_alerts = true, missing_grace_scans = 3, use_hotspot_temperature = true, system_collector_enabled = fullSmartEnabled, } return values[key] end, pluginDataDir = function() return "/mock/plugin-data" end, readFile = function(path) return path:match("alert%-state%.json$") and "{}" or nil end, writeFile = function(path, _contents) table.insert(stateWrites, path) if failNextStateWrite then failNextStateWrite = false return false, "fixture write failure" end return true end, renameFile = function(from, to) table.insert(stateRenames, { from = from, to = to }) if failNextStateRename then failNextStateRename = false return false, "fixture rename failure" end successfulStateCommits = successfulStateCommits + 1 return true end, log = function(_message) end, notify = function(title, body) table.insert(notifications, { severity = "warning", title = title, body = body }) end, notifyError = function(title, body) table.insert(notifications, { severity = "critical", title = title, body = body }) end, tr = translate, state = { get = function(key) return state[key] end, set = function(key, value) state[key] = value end, watch = function(key, callback) watchers[key] = callback end, }, json = { decode = function(_raw) return { active = { ["SERIAL1:interface-crc"] = { id = "SERIAL1:interface-crc", kind = "interface-crc", drive = "Fixture SSD", message = "historical CRC total", severity = "warning", }, }, counters = {}, inventory = {}, availability = {}, dismissed = {}, } end, encode = function(_value, _pretty) return "{}" end, }, } local handle = assert(io.open("service.luau", "rb")) local source = handle:read("*a") handle:close() source = source:gsub("([%a_][%w_]*) %+%= ([^\n]+)", "%1 = %1 + %2") assert(load(source, "@service.luau"))() local publishSnapshot = assert(watchers.collector_snapshot, "alert service did not watch collector snapshots") local function publish(value) state.collector_snapshot = value publishSnapshot(value) end local dismiss = assert(watchers.dismiss_alert_request, "alert service did not watch dismissal requests") local function drive(temperature) return { id = "SERIAL1", device = "/dev/nvme0n1", model = "Fixture SSD", kind = "ssd", health = "passed", smart_available = true, temperature_c = temperature, hotspot_temperature_c = temperature, remaining_life_percent = 95, available_spare_percent = 100, critical_warning = 0, media_errors = 0, reallocated_sectors = 0, pending_sectors = 0, uncorrectable_errors = 0, spin_retry_count = 0, command_timeout_count = 0, interface_crc_errors = 0, unsafe_shutdowns = 12, error_log_entries = 5893, smart_completeness = "full", alerts_enabled = true, presence_alert_enabled = true, self_test_state = "passed", } end local function snapshot(disk, collectionId) return { collection_id = collectionId, disks = disk ~= nil and { disk } or {}, dependencies = { ready = true, missing = {}, missing_text = "", signature = "" }, summary = { ssd_count = disk ~= nil and 1 or 0 }, } end local function findIssue(kind) for _, issue in ipairs(state.snapshot.issues or {}) do if issue.kind == kind then return issue end end return nil end publish(snapshot(drive(45))) assert(#state.snapshot.issues == 0, "healthy drive produced an alert") assert(#notifications == 0, "healthy baseline or legacy counter cleanup produced a notification") assert(#stateWrites == 1 and #stateRenames == 1 and successfulStateCommits == 1, "first healthy baseline was not persisted as one atomic commit") publish(snapshot(drive(45))) onConfigChanged() assert(#stateWrites == 1 and #stateRenames == 1, "identical or config-only processing rewrote alert state") local writesBeforeNewAlert = #stateWrites publish(snapshot(drive(70))) assert(#state.snapshot.issues == 1 and state.snapshot.issues[1].kind == "temperature", "warning temperature was missed") assert(#notifications == 1 and notifications[1].severity == "warning", "warning notification was not sent") assert(#stateWrites == writesBeforeNewAlert + 1 and #stateRenames == writesBeforeNewAlert + 1, "new alert was not persisted as exactly one atomic commit") publish(snapshot(drive(70))) assert(#notifications == 1, "unchanged warning notification was duplicated") assert(#stateWrites == writesBeforeNewAlert + 1, "unchanged active alert rewrote alert state") publish(snapshot(drive(63))) assert(#state.snapshot.issues == 1 and state.snapshot.issues[1].message == "alerts.temperature_cooling", "temperature hysteresis displayed a contradictory threshold message") assert(#notifications == 1, "cooling hysteresis duplicated its warning notification") publish(snapshot(drive(85))) assert(state.snapshot.issues[1].severity == "critical", "critical escalation was missed") assert(#notifications == 2 and notifications[2].severity == "critical", "critical escalation did not notify") local writesBeforeRecovery = #stateWrites publish(snapshot(drive(60))) assert(#state.snapshot.issues == 0, "temperature recovery did not clear") assert(#notifications == 3 and notifications[3].severity == "warning", "recovery did not notify") assert(#stateWrites == writesBeforeRecovery + 1 and #stateRenames == writesBeforeRecovery + 1, "alert recovery was not persisted as exactly one atomic commit") publish(snapshot(drive(70))) assert(#notifications == 4, "recurring temperature issue did not notify") publish(snapshot(drive(45))) assert(#notifications == 5, "temperature recovery did not notify") local retiredFixture = drive(45) retiredFixture.presence_alert_enabled = false publish(snapshot(retiredFixture)) local hddIssue = drive(35) hddIssue.id = "HDD-SERIAL" hddIssue.kind = "hdd" hddIssue.remaining_life_percent = nil hddIssue.presence_alert_enabled = false hddIssue.interface_crc_errors = 2 local notificationsBeforeHddBaseline = #notifications publish(snapshot(hddIssue)) assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeHddBaseline, "historical HDD interface CRC errors produced a false alert") hddIssue.interface_crc_errors = 3 publish(snapshot(hddIssue)) assert(#state.snapshot.issues == 0, "new HDD interface CRC errors produced a persistent issue") assert(#notifications == notificationsBeforeHddBaseline + 1 and notifications[#notifications].severity == "warning" and notifications[#notifications].body:match("count=1"), "new HDD interface CRC error did not produce one delta notification") publish(snapshot(hddIssue)) assert(#notifications == notificationsBeforeHddBaseline + 1, "unchanged HDD interface CRC total duplicated its notification") hddIssue.interface_crc_errors = 0 publish(snapshot(hddIssue)) assert(#notifications == notificationsBeforeHddBaseline + 1, "decreased HDD interface CRC total produced a notification") local historicalErrors = drive(45) historicalErrors.id = "HISTORICAL-SERIAL" historicalErrors.presence_alert_enabled = false historicalErrors.media_errors = 8 historicalErrors.reallocated_sectors = 4 historicalErrors.pending_sectors = 2 historicalErrors.uncorrectable_errors = 3 historicalErrors.spin_retry_count = 1 historicalErrors.command_timeout_count = 6 historicalErrors.interface_crc_errors = 9 historicalErrors.error_log_entries = 20 local notificationsBeforeHistoricalBaseline = #notifications publish(snapshot(historicalErrors)) assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeHistoricalBaseline, "first observation of historical SMART counters produced false alerts") historicalErrors.uncorrectable_errors = 5 historicalErrors.error_log_entries = 21 historicalErrors.interface_crc_errors = 11 publish(snapshot(historicalErrors)) assert(#state.snapshot.issues == 0, "counter deltas produced persistent drive issues") assert(#notifications == notificationsBeforeHistoricalBaseline + 3, "new SMART counter values did not produce exactly one notification per increase") assert(notifications[notificationsBeforeHistoricalBaseline + 1].severity == "critical" and notifications[notificationsBeforeHistoricalBaseline + 1].body:match("count=2"), "uncorrectable-error delta notification was incorrect") assert(notifications[notificationsBeforeHistoricalBaseline + 2].severity == "warning" and notifications[notificationsBeforeHistoricalBaseline + 2].body:match("count=2"), "interface CRC delta notification was incorrect") assert(notifications[notificationsBeforeHistoricalBaseline + 3].severity == "warning" and notifications[notificationsBeforeHistoricalBaseline + 3].body:match("count=1"), "error-log delta notification was incorrect") publish(snapshot(historicalErrors)) local unchangedCounterNotifications = {} for index = notificationsBeforeHistoricalBaseline + 1, #notifications do table.insert(unchangedCounterNotifications, notifications[index].body) end assert(#notifications == notificationsBeforeHistoricalBaseline + 3, "unchanged SMART counter values duplicated notifications: " .. table.concat(unchangedCounterNotifications, ", ")) publish(snapshot(drive(45))) local hotspot = drive(45) hotspot.hotspot_temperature_c = 70 publish(snapshot(hotspot)) assert(state.snapshot.issues[1].kind == "temperature", "NVMe hotspot warning was missed") publish(snapshot(drive(45))) local customThreshold = drive(55) customThreshold.warning_temperature = 50 customThreshold.critical_temperature = 70 publish(snapshot(customThreshold)) assert(state.snapshot.issues[1].kind == "temperature" and state.snapshot.issues[1].severity == "warning", "per-drive temperature threshold was ignored") publish(snapshot(drive(45))) local partial = drive(45) partial.smart_completeness = "partial" publish(snapshot(partial)) assert(#state.snapshot.issues == 0, "healthy partial SMART data produced an alert") publish(snapshot(drive(45))) local unavailable = drive(45) unavailable.smart_available = false publish(snapshot(unavailable)) assert(#state.snapshot.issues == 0, "Basic mode produced a false SMART-unavailable warning") fullSmartEnabled = true local notificationsBeforeUnavailable = #notifications local writesBeforeUnavailable = #stateWrites publish(snapshot(unavailable, "smart-unavailable-transient")) assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeUnavailable, "a single transient SMART read failure produced an alert") assert(#stateWrites == writesBeforeUnavailable + 1, "the first unavailable SMART scan did not persist its pending state") publish(snapshot(unavailable, "smart-unavailable-transient")) onConfigChanged() assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeUnavailable, "reprocessing one unavailable SMART snapshot advanced its grace period") assert(#stateWrites == writesBeforeUnavailable + 1, "reprocessing one unavailable SMART snapshot rewrote pending state") publish(snapshot(drive(45), "smart-available-reset")) assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeUnavailable, "transient SMART availability recovery produced a notification") publish(snapshot(unavailable, "smart-unavailable-1")) publish(snapshot(unavailable, "smart-unavailable-2")) assert(#state.snapshot.issues == 0 and #notifications == notificationsBeforeUnavailable, "SMART unavailability alerted before three completed scans") publish(snapshot(unavailable, "smart-unavailable-3")) assert(#state.snapshot.issues == 1 and state.snapshot.issues[1].kind == "smart-unavailable", "sustained SMART unavailability did not alert after three completed scans") assert(#notifications == notificationsBeforeUnavailable + 1, "sustained SMART unavailability did not produce exactly one notification") local writesAfterConfirmedUnavailable = #stateWrites publish(snapshot(unavailable, "smart-unavailable-4")) assert(#notifications == notificationsBeforeUnavailable + 1, "continued SMART unavailability duplicated its notification") assert(#stateWrites == writesAfterConfirmedUnavailable, "confirmed SMART unavailability rewrote stable alert state") local sleeping = drive(45) sleeping.smart_available = false sleeping.smart_sleeping = true publish(snapshot(sleeping, "smart-sleeping")) assert(#state.snapshot.issues == 0, "sleeping drive produced a SMART-unavailable warning") assert(#notifications == notificationsBeforeUnavailable + 2, "confirmed SMART-unavailable recovery did not notify exactly once") fullSmartEnabled = false publish(snapshot(drive(45))) local selfTestFailure = drive(45) selfTestFailure.self_test_state = "failed" selfTestFailure.self_test_status = "Completed with read failure" publish(snapshot(selfTestFailure)) assert(state.snapshot.issues[1].kind == "self-test" and state.snapshot.issues[1].severity == "critical", "self-test failure was missed") publish(snapshot(drive(45))) local counterIncrease = drive(45) counterIncrease.unsafe_shutdowns = 13 local writesBeforeCounterIncrease = #stateWrites local notificationsBeforeCounterIncrease = #notifications publish(snapshot(counterIncrease)) assert(#stateWrites == writesBeforeCounterIncrease + 1 and #stateRenames == writesBeforeCounterIncrease + 1, "counter increase was not persisted as exactly one atomic commit") assert(#notifications == notificationsBeforeCounterIncrease + 1, "counter increase notification behavior changed") publish(snapshot(counterIncrease)) assert(#stateWrites == writesBeforeCounterIncrease + 1, "unchanged diagnostic counter rewrote alert state") publish(snapshot(drive(45))) publish(snapshot(drive(45), "scan-100")) local firstMissingScan = snapshot(nil, "scan-101") local writesBeforeMissingScan = #stateWrites publish(firstMissingScan) assert(findIssue("drive-missing") == nil, "missing drive alerted before the grace period") assert(#stateWrites == writesBeforeMissingScan + 1 and #stateRenames == writesBeforeMissingScan + 1, "missing-drive inventory change was not persisted as exactly one atomic commit") publish(firstMissingScan) assert(findIssue("drive-missing") == nil, "reprocessing one snapshot advanced the grace period") assert(#stateWrites == writesBeforeMissingScan + 1, "reprocessing one collection rewrote unchanged inventory state") publish(snapshot(nil, "scan-101")) assert(findIssue("drive-missing") == nil, "a repeated collection ID advanced the grace period") assert(#stateWrites == writesBeforeMissingScan + 1, "a repeated collection ID rewrote unchanged inventory state") onConfigChanged() assert(findIssue("drive-missing") == nil, "a config refresh advanced the grace period") assert(#stateWrites == writesBeforeMissingScan + 1, "config refresh rewrote unchanged missing-drive state") local collectingSnapshot = snapshot(nil, "scan-collecting") collectingSnapshot.collecting = true publish(collectingSnapshot) assert(findIssue("drive-missing") == nil, "an in-progress collection advanced the grace period") local failedMissingScan = snapshot(nil, "scan-failed") failedMissingScan.collector_error = "fixture failure" publish(failedMissingScan) assert(findIssue("drive-missing") == nil, "a failed collection advanced the grace period") local blockedMissingScan = snapshot(nil, "scan-blocked") blockedMissingScan.dependencies = { ready = false, blocking = true, missing = { "lsblk (lsblk)" }, missing_text = "lsblk (lsblk)", signature = "lsblk", } publish(blockedMissingScan) assert(findIssue("drive-missing") == nil, "a blocked collection advanced the grace period") publish(snapshot(nil, "scan-102")) assert(findIssue("drive-missing") == nil, "missing drive alerted after only two completed scans") publish(snapshot(nil, "scan-103")) assert(findIssue("drive-missing") ~= nil, "three unique completed scans did not trigger a missing-drive alert") publish(snapshot(nil, "scan-103")) assert(findIssue("drive-missing") ~= nil, "reprocessing a snapshot removed an active missing-drive alert") local writesBeforeReappearance = #stateWrites publish(snapshot(drive(45), "scan-104")) assert(findIssue("drive-missing") == nil, "drive reappearance did not clear its missing alert") assert(#stateWrites == writesBeforeReappearance + 1 and #stateRenames == writesBeforeReappearance + 1, "drive reappearance was not persisted as exactly one atomic commit") publish(snapshot(nil, "scan-105")) publish(snapshot(nil, "scan-106")) assert(findIssue("drive-missing") == nil, "reappearance did not reset the missing-drive grace period") publish(snapshot(nil, "scan-107")) assert(findIssue("drive-missing") ~= nil, "three new scans after reappearance did not trigger an alert") publish(snapshot(drive(45), "scan-108")) local legacyMissingScan = snapshot(nil) publish(legacyMissingScan) publish(legacyMissingScan) assert(findIssue("drive-missing") == nil, "a repeated legacy snapshot advanced the grace period") publish(snapshot(nil)) assert(findIssue("drive-missing") == nil, "two legacy snapshot objects triggered an early alert") publish(snapshot(nil)) assert(findIssue("drive-missing") ~= nil, "distinct legacy snapshot objects did not advance the grace period") local missing = snapshot(nil) missing.dependencies = { ready = false, blocking = true, missing = { "lsblk (lsblk)" }, missing_text = "lsblk (lsblk)", signature = "lsblk", } local notificationCountBeforeDependency = #notifications publish(missing) assert(findIssue("missing-dependencies") ~= nil, "dependency issue was missed") assert(findIssue("drive-missing") ~= nil, "a blocking dependency removed an active missing-drive alert") local dependencyCritical = false for index = notificationCountBeforeDependency + 1, #notifications do if notifications[index].severity == "critical" then dependencyCritical = true end end assert(dependencyCritical, "blocking dependency was not critical") local failed = snapshot(nil) failed.collector_error = "fixture failure" local notificationCountBeforeFailure = #notifications publish(failed) assert(findIssue("collector-error") ~= nil, "collector issue was missed") assert(findIssue("drive-missing") ~= nil, "a collector failure removed an active missing-drive alert") local collectorCritical = false for index = notificationCountBeforeFailure + 1, #notifications do if notifications[index].severity == "critical" then collectorCritical = true end end assert(collectorCritical, "collector failure did not notify critically") local writesBeforeWriteFailure = #stateWrites local renamesBeforeWriteFailure = #stateRenames local commitsBeforeWriteFailure = successfulStateCommits failNextStateWrite = true dismiss({ id = "collector:error", nonce = 8 }) assert(#stateWrites == writesBeforeWriteFailure + 1 and #stateRenames == renamesBeforeWriteFailure and successfulStateCommits == commitsBeforeWriteFailure, "failed temporary write attempted a rename or lost the pending state") publish(failed) assert(#stateWrites == writesBeforeWriteFailure + 2 and #stateRenames == renamesBeforeWriteFailure + 1 and successfulStateCommits == commitsBeforeWriteFailure + 1, "unchanged snapshot did not retry a failed alert-state write") publish(failed) assert(#stateWrites == writesBeforeWriteFailure + 2, "successful write retry did not clear dirty alert state") local writesBeforeRenameFailure = #stateWrites local renamesBeforeRenameFailure = #stateRenames local commitsBeforeRenameFailure = successfulStateCommits failNextStateRename = true dismiss({ all = true, nonce = 9 }) assert(#state.snapshot.issues == 0, "dismiss all did not hide every active issue") assert(#stateWrites == writesBeforeRenameFailure + 1 and #stateRenames == renamesBeforeRenameFailure + 1 and successfulStateCommits == commitsBeforeRenameFailure, "failed atomic rename was treated as a successful commit") onConfigChanged() assert(#stateWrites == writesBeforeRenameFailure + 2 and #stateRenames == renamesBeforeRenameFailure + 2 and successfulStateCommits == commitsBeforeRenameFailure + 1, "unchanged processing did not retry a failed atomic rename") onConfigChanged() assert(#stateWrites == writesBeforeRenameFailure + 2, "successful rename retry did not clear dirty alert state") publish(snapshot(drive(45), "scan-dismissal-baseline")) publish(snapshot(drive(70), "scan-dismissal-warning")) assert(findIssue("temperature") ~= nil, "permanent-dismissal fixture did not create an alert") local notificationsBeforeDismissal = #notifications local writesBeforeDismissal = #stateWrites local renamesBeforeDismissal = #stateRenames dismiss({ id = "SERIAL1:temperature", nonce = 10 }) assert(findIssue("temperature") == nil, "individual dismissal did not hide the active issue") assert(#notifications == notificationsBeforeDismissal, "dismissing an issue emitted a notification") assert(#stateWrites == writesBeforeDismissal + 1 and #stateRenames == renamesBeforeDismissal + 1, "dismissal was not persisted as exactly one atomic commit") publish(snapshot(drive(85), "scan-dismissal-critical")) assert(findIssue("temperature") == nil and #notifications == notificationsBeforeDismissal, "a dismissed alert returned or notified after escalating") publish(snapshot(drive(45), "scan-dismissal-recovery")) publish(snapshot(drive(70), "scan-dismissal-recurrence")) assert(findIssue("temperature") == nil and #notifications == notificationsBeforeDismissal, "a dismissed alert returned or notified after recurring") for _, path in ipairs(stateWrites) do assert(path:match("alert%-state%.json%.tmp$"), "alert state bypassed its temporary file") end for _, rename in ipairs(stateRenames) do assert(rename.from:match("alert%-state%.json%.tmp$") and rename.to:match("alert%-state%.json$"), "alert state was not committed with an atomic rename") end print("alert behavior tests passed")