{"status":"ok","checked_at":1787904787,"items":[{"event_id":"4d507c8a-0af3-4c99-a0a6-0a6a79d3ba52","dedup_key":"ai-runtime:BOSS_FAILED:review:9e35af171b65472b:token-budget-cleanup","scope":"ai-runtime","severity":"resolved","state":"BOSS_FAILED","previous_state":"BOSS_FAILED","maintenance_window_id":"review:9e35af171b65472b","producer":"ai-runtime","summary":"Runtime 已删除历史任务 Token 上限；当前工作台为无限制，旧预算告警已失效。","evidence_ref":"commit:455d850","occurred_at":1786955516,"received_at":1786955516,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786955516,"actions":[]},{"event_id":"c45e6a4f-610f-4338-9362-980b11d2223a","dedup_key":"ai-runtime:BOSS_FAILED:review:9e35af171b65472b","scope":"ai-runtime","severity":"critical","state":"BOSS_FAILED","previous_state":null,"maintenance_window_id":"review:9e35af171b65472b","producer":"ai-runtime","summary":"AI BOSS 审核未完成：task_token_budget_exceeded","evidence_ref":"9e35af171b65472b","occurred_at":1786943272,"received_at":1786943272,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：Runtime 已删除历史任务 Token 上限；当前工作台为无限制，旧预算告警已失效。","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786955516,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"ai-runtime","created_at":1786955516}]},{"event_id":"e1dd2154-1f28-4bb2-8447-e560b4a4fb22","dedup_key":"ai-runtime:WORKER_FAILED:task:9e35af171b65472b:classification-corrected:db4b270","scope":"ai-runtime","severity":"resolved","state":"WORKER_FAILED","previous_state":"WORKER_FAILED","maintenance_window_id":"task:9e35af171b65472b","producer":"ai-runtime","summary":"历史 Worker 失败已更正：执行完成，审核因冻结预算不足待处理；未作最终验收。","evidence_ref":"9e35af171b65472b","occurred_at":1786940955,"received_at":1786940955,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786940955,"actions":[]},{"event_id":"ec17a2df-6172-44f9-8e29-b4c68e045944","dedup_key":"ai-runtime:WORKER_FAILED:task:9e35af171b65472b","scope":"ai-runtime","severity":"critical","state":"WORKER_FAILED","previous_state":null,"maintenance_window_id":"task:9e35af171b65472b","producer":"ai-runtime","summary":"P4 主节点任务失败：HTTPException","evidence_ref":"9e35af171b65472b","occurred_at":1786881905,"received_at":1786881905,"acknowledged_at":1786940341,"acknowledged_by":"admin","read_at":1786940341,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：历史 Worker 失败已更正：执行完成，审核因冻结预算不足待处理；未作最终验收。","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786940955,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"ai-runtime","created_at":1786940955},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1786940341}]},{"event_id":"82050f56-ddce-4f9d-8839-c36b04c5dbd5","dedup_key":"ai-runtime:WORKER_FAILED:task:0e972fa36c414b6f:recovered:2ea07df","scope":"ai-runtime","severity":"resolved","state":"WORKER_FAILED","previous_state":"WORKER_FAILED","maintenance_window_id":"task:0e972fa36c414b6f","producer":"ai-runtime-recovery-controller","summary":"已修复：private CLI 任务外发分类修正后，后续同通道 P4 真实任务已完成验收。","evidence_ref":"2ea07df:e04dd51df0bd469f","occurred_at":1786876942,"received_at":1786876942,"acknowledged_at":1786939914,"acknowledged_by":"admin","read_at":1786939914,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786876942,"actions":[{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1786939914}]},{"event_id":"74afd09c-eb2e-4864-8d99-eca43ce3543b","dedup_key":"ai-runtime:WORKER_FAILED:task:abe7045e3cf440ea:recovered:2ea07df","scope":"ai-runtime","severity":"resolved","state":"WORKER_FAILED","previous_state":"WORKER_FAILED","maintenance_window_id":"task:abe7045e3cf440ea","producer":"ai-runtime-recovery-controller","summary":"已修复：private CLI 任务外发分类修正后，后续同通道 P4 真实任务已完成验收。","evidence_ref":"2ea07df:e04dd51df0bd469f","occurred_at":1786876942,"received_at":1786876942,"acknowledged_at":1786939915,"acknowledged_by":"admin","read_at":1786939915,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786876942,"actions":[{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1786939915}]},{"event_id":"60bca045-9e9b-4cce-bb62-02050d53ad8f","dedup_key":"ai-runtime:WORKER_FAILED:task:0e972fa36c414b6f","scope":"ai-runtime","severity":"critical","state":"WORKER_FAILED","previous_state":null,"maintenance_window_id":"task:0e972fa36c414b6f","producer":"ai-runtime","summary":"P4 主节点任务失败：CliAgentGatewayError","evidence_ref":"0e972fa36c414b6f","occurred_at":1786868761,"received_at":1786868761,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：已修复：private CLI 任务外发分类修正后，后续同通道 P4 真实任务已完成验收。","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786876942,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"ai-runtime-recovery-controller","created_at":1786876942}]},{"event_id":"61b0f1bd-9ba3-44d5-9831-170bb237ef76","dedup_key":"ai-runtime:WORKER_FAILED:task:abe7045e3cf440ea","scope":"ai-runtime","severity":"critical","state":"WORKER_FAILED","previous_state":null,"maintenance_window_id":"task:abe7045e3cf440ea","producer":"ai-runtime","summary":"P4 主节点任务失败：CliAgentGatewayError","evidence_ref":"abe7045e3cf440ea","occurred_at":1786868761,"received_at":1786868761,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：已修复：private CLI 任务外发分类修正后，后续同通道 P4 真实任务已完成验收。","result_evidence_ref":null,"processing_started_at":null,"handled_at":1786876942,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"ai-runtime-recovery-controller","created_at":1786876942}]},{"event_id":"1b2c8a2e-1e16-4442-9b90-104943f17635","dedup_key":"ai-runtime:NOMAD_FAILED:task:7fdd55c270c241f4","scope":"ai-runtime","severity":"critical","state":"NOMAD_FAILED","previous_state":null,"maintenance_window_id":"task:7fdd55c270c241f4","producer":"ai-runtime","summary":"Nomad 自动提交失败：RuntimeError","evidence_ref":"7fdd55c270c241f4","occurred_at":1785740179,"received_at":1785740179,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":"codex-verified-maintenance","incident_id":null,"result_summary":"历史 Nomad 磁盘下限缺陷已修复为至少 300 MiB，并由替代父子任务完成和清理验证。","result_evidence_ref":"fix:df0a478;tasks:35106cec0cb64708,114db704ef9b460b","processing_started_at":null,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"历史 Nomad 磁盘下限缺陷已修复为至少 300 MiB，并由替代父子任务完成和清理验证。","actor":"boss-agent","created_at":1785850113}]},{"event_id":"de52a730-48dd-4c40-8ccb-be4170af0fe2","dedup_key":"ai-runtime:PROVIDER_FAILED:free-route-probe:controlled_failure_20260802T171802Z","scope":"ai-runtime","severity":"critical","state":"PROVIDER_FAILED","previous_state":null,"maintenance_window_id":"free-route-probe:controlled_failure_20260802T171802Z","producer":"ai-runtime","summary":"OmniRoute 免费 Combo 测试失败：OmniRouteError","evidence_ref":null,"occurred_at":1785691084,"received_at":1785691084,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":"codex-acceptance","incident_id":null,"result_summary":"受控不存在模型调用已验证 Runtime 脱敏失败事件与全局告警链路；生产服务健康未受影响。","result_evidence_ref":"P3-6-2-stage5-controlled-failure-20260802T171802Z","processing_started_at":null,"handled_at":1785691135,"actions":[{"action":"boss_completed","note":"受控不存在模型调用已验证 Runtime 脱敏失败事件与全局告警链路；生产服务健康未受影响。","actor":"boss-agent","created_at":1785691135}]},{"event_id":"099a0511-deb3-42aa-ab31-ad6cb6020fa9","dedup_key":"ai-runtime:PROVIDER_FAILED:provider:agentrouter","scope":"ai-runtime","severity":"critical","state":"PROVIDER_FAILED","previous_state":null,"maintenance_window_id":"provider:agentrouter","producer":"ai-runtime","summary":"Provider 连通性测试失败：AuthenticationError","evidence_ref":null,"occurred_at":1785492753,"received_at":1785492753,"acknowledged_at":1785516043,"acknowledged_by":"admin","read_at":1785516043,"read_by":"admin","handling_status":"completed","handler":"codex-verified-maintenance","incident_id":null,"result_summary":"AgentRouter 已从 OmniRoute 正式数据移除，不再属于当前运行链路。","result_evidence_ref":"omniroute:agentrouter-removed","processing_started_at":null,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"AgentRouter 已从 OmniRoute 正式数据移除，不再属于当前运行链路。","actor":"boss-agent","created_at":1785850113},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785516043}]},{"event_id":"8af9e0c8-c352-4332-af3b-32eb65ad57bf","dedup_key":"ai-runtime:NOMAD_FAILED:task:93b1c29ac526486e","scope":"ai-runtime","severity":"critical","state":"NOMAD_FAILED","previous_state":null,"maintenance_window_id":"task:93b1c29ac526486e","producer":"ai-runtime","summary":"Nomad 自动提交失败：RuntimeError","evidence_ref":"93b1c29ac526486e","occurred_at":1785432607,"received_at":1785432607,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":"codex-verified-maintenance","incident_id":null,"result_summary":"旧 Nomad HCL JSON 参数编码缺陷已修复，并由后续真实 Nomad 任务完成和清理验证。","result_evidence_ref":"task:4744fcd8507942ac:completed","processing_started_at":null,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"旧 Nomad HCL JSON 参数编码缺陷已修复，并由后续真实 Nomad 任务完成和清理验证。","actor":"boss-agent","created_at":1785850113}]},{"event_id":"d2738d2a-6a3f-4099-9287-01189278b7b8","dedup_key":"storage-health:storage-health-kodbox-active:DEGRADED:storage-health/kodbox-active","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-kodbox-active","producer":"storage-health-probe","summary":"存储健康探针：kodbox-active 状态为 DEGRADED","evidence_ref":"storage-health/kodbox-active","occurred_at":1785288313,"received_at":1785288313,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：kodbox-active 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785288380,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785288380}]},{"event_id":"5be4077b-9e7d-42de-8ab8-a93a42a04f03","dedup_key":"storage-health:storage-health-disk-webdav-101:DEGRADED:storage-health/disk-webdav-101","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-disk-webdav-101","producer":"storage-health-probe","summary":"存储健康探针：disk-webdav-101 状态为 DEGRADED","evidence_ref":"storage-health/disk-webdav-101","occurred_at":1785283370,"received_at":1785283370,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：disk-webdav-101 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785283428,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785283428}]},{"event_id":"29d53c89-0cd9-4c9e-af74-4db4b9355d76","dedup_key":"storage-health:storage-health-disk-webdav-101:HEALTHY:storage-health/disk-webdav-101","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-disk-webdav-101","producer":"storage-health-probe","summary":"存储健康探针：disk-webdav-101 状态为 HEALTHY","evidence_ref":"storage-health/disk-webdav-101","occurred_at":1785283048,"received_at":1785283048,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785283048,"actions":[]},{"event_id":"e1232765-f6ad-4d73-b525-e0144f343d10","dedup_key":"storage-health:storage-health-kodbox-active:HEALTHY:storage-health/kodbox-active","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-kodbox-active","producer":"storage-health-probe","summary":"存储健康探针：kodbox-active 状态为 HEALTHY","evidence_ref":"storage-health/kodbox-active","occurred_at":1785281253,"received_at":1785281254,"acknowledged_at":1785281696,"acknowledged_by":"admin","read_at":1785281696,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785281254,"actions":[{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785281696}]},{"event_id":"d2d2652e-4bb8-4b1c-8256-f654f90f0e5e","dedup_key":"ai-runtime:PROVIDER_FAILED:provider:premium","scope":"ai-runtime","severity":"critical","state":"PROVIDER_FAILED","previous_state":null,"maintenance_window_id":"provider:premium","producer":"ai-runtime","summary":"Premium Provider 连通性测试失败：RateLimitError","evidence_ref":null,"occurred_at":1785238246,"received_at":1785238246,"acknowledged_at":1785238297,"acknowledged_by":"admin","read_at":1785238297,"read_by":"admin","handling_status":"completed","handler":"boss-agent","incident_id":"23593bb345bf4c3b","result_summary":"旧 Premium Direct Provider 路径已退役；当前 Runtime 统一通过健康的 OmniRoute 双 Combo 调用。","result_evidence_ref":"runtime-omniroute-migration","processing_started_at":1785238297,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"旧 Premium Direct Provider 路径已退役；当前 Runtime 统一通过健康的 OmniRoute 双 Combo 调用。","actor":"boss-agent","created_at":1785850113},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785238297},{"action":"boss_processing","note":"AI Boss 已接收告警，正在执行受控诊断。","actor":"boss-agent","created_at":1785238297},{"action":"boss_needs_benson_decision","note":"AI Boss 已完成 Provider 受控诊断：当前模型 gpt-5.6-terra，路由 已启用。当前没有已验证的自动回退，未改动密钥、模型或 Provider 配置。建议：在 AI Provider 管理中确认可用模型或 Provider 后重新测试。","actor":"boss-agent","created_at":1785238297}]},{"event_id":"881e2505-42bc-4989-ae74-71de255c77cb","dedup_key":"ai-runtime:PROVIDER_FAILED:review:a51ee781b0c0472a","scope":"ai-runtime","severity":"critical","state":"PROVIDER_FAILED","previous_state":null,"maintenance_window_id":"review:a51ee781b0c0472a","producer":"ai-runtime","summary":"Boss 审核模型调用失败：ValidationError","evidence_ref":"a51ee781b0c0472a","occurred_at":1785221194,"received_at":1785221194,"acknowledged_at":1785238210,"acknowledged_by":"admin","read_at":1785238210,"read_by":"admin","handling_status":"completed","handler":"boss-agent","incident_id":"760dc7c2798b4da2","result_summary":"关联任务 a51ee781b0c0472a 后续审核通过并完成；旧审核失败已失效。","result_evidence_ref":"task:a51ee781b0c0472a:completed","processing_started_at":1785238211,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"关联任务 a51ee781b0c0472a 后续审核通过并完成；旧审核失败已失效。","actor":"boss-agent","created_at":1785850113},{"action":"boss_processing","note":"AI Boss 已接收告警，正在执行受控诊断。","actor":"boss-agent","created_at":1785238211},{"action":"boss_needs_benson_decision","note":"AI Boss 已完成 Provider 受控诊断：当前模型 gpt-5.6-terra，路由 已启用。当前没有已验证的自动回退，未改动密钥、模型或 Provider 配置。建议：在 AI Provider 管理中确认可用模型或 Provider 后重新测试。","actor":"boss-agent","created_at":1785238211},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785238210}]},{"event_id":"37d78562-3429-49df-93ea-c7bbe362ad19","dedup_key":"storage-health:storage-health-disk-public:DEGRADED:storage-health/disk-public","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-disk-public","producer":"storage-health-probe","summary":"存储健康探针：disk-public 状态为 DEGRADED","evidence_ref":"storage-health/disk-public","occurred_at":1785220383,"received_at":1785220383,"acknowledged_at":1785238204,"acknowledged_by":"admin","read_at":1785238204,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：disk-public 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785234686,"actions":[{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785238204},{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785234686}]},{"event_id":"1b4ad759-7333-4b70-b214-013afe4701fd","dedup_key":"storage-health:storage-health-kodbox-101:DEGRADED:storage-health/kodbox-101","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-kodbox-101","producer":"storage-health-probe","summary":"存储健康探针：kodbox-101 状态为 DEGRADED","evidence_ref":"storage-health/kodbox-101","occurred_at":1785220373,"received_at":1785220373,"acknowledged_at":1785227280,"acknowledged_by":"admin","read_at":1785227280,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：kodbox-101 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785234631,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785234631},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785227280}]},{"event_id":"73c07f9d-d318-40d6-b57f-44b0b699d81c","dedup_key":"ai-runtime:PROVIDER_FAILED:review:379f7f4081bb4fc6","scope":"ai-runtime","severity":"critical","state":"PROVIDER_FAILED","previous_state":null,"maintenance_window_id":"review:379f7f4081bb4fc6","producer":"ai-runtime","summary":"Boss 审核模型调用失败：RuntimeError","evidence_ref":"379f7f4081bb4fc6","occurred_at":1785184230,"received_at":1785184230,"acknowledged_at":1785210795,"acknowledged_by":"admin","read_at":1785210795,"read_by":"admin","handling_status":"completed","handler":"boss-agent","incident_id":"ca6c605b831943ff","result_summary":"关联任务 379f7f4081bb4fc6 后续审核通过、任务完成且 Nomad Job 已清理；旧审核失败已失效。","result_evidence_ref":"task:379f7f4081bb4fc6:completed","processing_started_at":1785210796,"handled_at":1785850113,"actions":[{"action":"boss_completed","note":"关联任务 379f7f4081bb4fc6 后续审核通过、任务完成且 Nomad Job 已清理；旧审核失败已失效。","actor":"boss-agent","created_at":1785850113},{"action":"boss_processing","note":"AI Boss 已接收告警，正在执行受控诊断。","actor":"boss-agent","created_at":1785210796},{"action":"boss_needs_benson_decision","note":"AI Boss 已完成 Provider 受控诊断：当前模型 gpt-5.6-terra，路由 已启用。当前没有已验证的自动回退，未改动密钥、模型或 Provider 配置。建议：在 AI Provider 管理中确认可用模型或 Provider 后重新测试。","actor":"boss-agent","created_at":1785210796},{"action":"read","note":"Benson 已查看告警","actor":"admin","created_at":1785210795}]},{"event_id":"07f4309e-3caa-4d69-a6d0-607bb9695608","dedup_key":"storage-health:storage-health-galera-103:DEGRADED:storage-health/galera-103","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-galera-103","producer":"storage-health-probe","summary":"存储健康探针：galera-103 状态为 DEGRADED","evidence_ref":"storage-health/galera-103","occurred_at":1785177450,"received_at":1785177450,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：galera-103 当前已恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785178899,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785178899}]},{"event_id":"f2e2cd4d-0537-450a-a566-1280eeff042d","dedup_key":"storage-health:storage-health-galera-101:DEGRADED:storage-health/galera-101","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"HEALTHY","maintenance_window_id":"storage-health-galera-101","producer":"storage-health-probe","summary":"存储健康探针：galera-101 状态为 DEGRADED","evidence_ref":"storage-health/galera-101","occurred_at":1785177448,"received_at":1785177448,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：galera-101 当前已恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785178899,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785178899}]},{"event_id":"f151ca6e-cd26-4384-a7cf-c3b41b5e6b3b","dedup_key":"storage-health:storage-health-nomad-api:HEALTHY:storage-health/nomad-api","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"DEGRADED","maintenance_window_id":"storage-health-nomad-api","producer":"storage-health-probe","summary":"存储健康探针：nomad-api 状态为 HEALTHY","evidence_ref":"storage-health/nomad-api","occurred_at":1785173229,"received_at":1785173229,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173229,"actions":[]},{"event_id":"fa2c48b0-4f97-49cc-b0db-6932e81ee85a","dedup_key":"storage-health:storage-health-juicefs-103:HEALTHY:storage-health/juicefs-103","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"DEGRADED","maintenance_window_id":"storage-health-juicefs-103","producer":"storage-health-probe","summary":"存储健康探针：juicefs-103 状态为 HEALTHY","evidence_ref":"storage-health/juicefs-103","occurred_at":1785173074,"received_at":1785173074,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173074,"actions":[]},{"event_id":"c2d9a868-867c-4d03-81ab-df00c13cf763","dedup_key":"storage-health:storage-health-juicefs-101:HEALTHY:storage-health/juicefs-101","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"DEGRADED","maintenance_window_id":"storage-health-juicefs-101","producer":"storage-health-probe","summary":"存储健康探针：juicefs-101 状态为 HEALTHY","evidence_ref":"storage-health/juicefs-101","occurred_at":1785173072,"received_at":1785173072,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173072,"actions":[]},{"event_id":"e3893138-1529-455c-96f7-94bc06ed5c46","dedup_key":"storage-health:storage-health-disk-public:HEALTHY:storage-health/disk-public","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-disk-public","producer":"storage-health-probe","summary":"存储健康探针：disk-public 状态为 HEALTHY","evidence_ref":"storage-health/disk-public","occurred_at":1785172898,"received_at":1785172898,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172898,"actions":[]},{"event_id":"e2296f65-36a0-4b6e-bd5f-9f12036d588b","dedup_key":"storage-health:storage-health-nomad-api:DEGRADED:storage-health/nomad-api","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"","maintenance_window_id":"storage-health-nomad-api","producer":"storage-health-probe","summary":"存储健康探针：nomad-api 状态为 DEGRADED","evidence_ref":"storage-health/nomad-api","occurred_at":1785172898,"received_at":1785172898,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：nomad-api 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173229,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785173229}]},{"event_id":"598627a4-a883-45b9-b4ea-dc9d27586e8c","dedup_key":"storage-health:storage-health-kodbox-101:HEALTHY:storage-health/kodbox-101","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-kodbox-101","producer":"storage-health-probe","summary":"存储健康探针：kodbox-101 状态为 HEALTHY","evidence_ref":"storage-health/kodbox-101","occurred_at":1785172897,"received_at":1785172897,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172897,"actions":[]},{"event_id":"ed32bb0a-707f-43e8-982a-9f0fae5cdc8f","dedup_key":"storage-health:storage-health-kodbox-103:DEGRADED:storage-health/kodbox-103","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"","maintenance_window_id":"storage-health-kodbox-103","producer":"storage-health-probe","summary":"存储健康探针：kodbox-103 状态为 DEGRADED","evidence_ref":"storage-health/kodbox-103","occurred_at":1785172897,"received_at":1785172897,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"探针校正：103 为 KodBox 备用节点，当前无活动 Allocation，不是当前故障","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173247,"actions":[{"action":"probe_corrected","note":"已确认 103 为备用 KodBox 节点，关闭本次误报；后续仅监控活动 Allocation","actor":"system","created_at":1785173247}]},{"event_id":"01eaa898-9b97-43c0-8db6-9ecd5cf8cf99","dedup_key":"storage-health:storage-health-juicefs-103:DEGRADED:storage-health/juicefs-103","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"","maintenance_window_id":"storage-health-juicefs-103","producer":"storage-health-probe","summary":"存储健康探针：juicefs-103 状态为 DEGRADED","evidence_ref":"storage-health/juicefs-103","occurred_at":1785172896,"received_at":1785172896,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：juicefs-103 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173074,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785173074}]},{"event_id":"23d0b719-52a9-405a-a107-4c9353414b6f","dedup_key":"storage-health:storage-health-juicefs-101:DEGRADED:storage-health/juicefs-101","scope":"storage-health","severity":"warning","state":"DEGRADED","previous_state":"","maintenance_window_id":"storage-health-juicefs-101","producer":"storage-health-probe","summary":"存储健康探针：juicefs-101 状态为 DEGRADED","evidence_ref":"storage-health/juicefs-101","occurred_at":1785172894,"received_at":1785172894,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"已由后续恢复事件关闭：存储健康探针：juicefs-101 状态为 HEALTHY","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785173072,"actions":[{"action":"recovery_linked","note":"已关联后续恢复事件并关闭当前告警","actor":"storage-health-probe","created_at":1785173072}]},{"event_id":"8cbf338d-38bd-4284-a835-7b352f0cd8ea","dedup_key":"storage-health:storage-health-galera-103:HEALTHY:storage-health/galera-103","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-galera-103","producer":"storage-health-probe","summary":"存储健康探针：galera-103 状态为 HEALTHY","evidence_ref":"storage-health/galera-103","occurred_at":1785172889,"received_at":1785172889,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172889,"actions":[]},{"event_id":"17a08d8d-f59d-4598-8596-9b6e8719b0c9","dedup_key":"storage-health:storage-health-galera-101:HEALTHY:storage-health/galera-101","scope":"storage-health","severity":"resolved","state":"HEALTHY","previous_state":"","maintenance_window_id":"storage-health-galera-101","producer":"storage-health-probe","summary":"存储健康探针：galera-101 状态为 HEALTHY","evidence_ref":"storage-health/galera-101","occurred_at":1785172886,"received_at":1785172887,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172887,"actions":[]},{"event_id":"da7f71b6-dfcf-4532-a32d-9f81db575b4a","dedup_key":"scheduler-c3:d-cloudreve-20260723t170246Z:CAPACITY_QUEUED:resource-gate","scope":"scheduler-c3","severity":"warning","state":"CAPACITY_QUEUED","previous_state":"","maintenance_window_id":"d-cloudreve-20260723t170246Z","producer":"controlled-batch-submit","summary":"C3 当前没有可用的已验证 Worker；新 Batch 保持排队，未创建 Nomad Job。","evidence_ref":"resource-gate","occurred_at":1784826704,"received_at":1784826704,"acknowledged_at":1784941852,"acknowledged_by":"admin","read_at":1784941852,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"历史事件：对应 C2/C3 阶段已闭环，不是当前故障，不提交 AI Boss","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172856,"actions":[{"action":"historical_closed","note":"已按历史归档规则关闭，不再交给 AI Boss","actor":"system","created_at":1785172856}]},{"event_id":"09c9b054-77f5-42f6-b1e4-ecf66633ce9c","dedup_key":"scheduler-c3:d-cloudreve-copy-20260722t183000z:CAPACITY_QUEUED:resource-gate","scope":"scheduler-c3","severity":"warning","state":"CAPACITY_QUEUED","previous_state":"","maintenance_window_id":"d-cloudreve-copy-20260722t183000z","producer":"controlled-batch-submit","summary":"C3 当前没有可用的已验证 Worker；新 Batch 保持排队，未创建 Nomad Job。","evidence_ref":"resource-gate","occurred_at":1784745678,"received_at":1784745678,"acknowledged_at":1784941853,"acknowledged_by":"admin","read_at":1784941853,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"历史事件：对应 C2/C3 阶段已闭环，不是当前故障，不提交 AI Boss","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172856,"actions":[{"action":"historical_closed","note":"已按历史归档规则关闭，不再交给 AI Boss","actor":"system","created_at":1785172856}]},{"event_id":"e8fb0d0b-e4b9-4a7f-9a40-7c2250822a03","dedup_key":"storage-c2:recovery-test-20260721:RECOVERY_VALIDATING:recovery-recovery-test-20260721","scope":"storage-c2","severity":"warning","state":"RECOVERY_VALIDATING","previous_state":"CREDENTIALS_REVOKED","maintenance_window_id":"recovery-test-20260721","producer":"dd-archive-maintenance-controller","summary":"C2 已开始恢复验证，写入门禁继续保持。","evidence_ref":"recovery-recovery-test-20260721","occurred_at":1784742656,"received_at":1784742656,"acknowledged_at":1784941855,"acknowledged_by":"admin","read_at":1784941855,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"历史事件：对应 C2/C3 阶段已闭环，不是当前故障，不提交 AI Boss","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172856,"actions":[{"action":"historical_closed","note":"已按历史归档规则关闭，不再交给 AI Boss","actor":"system","created_at":1785172856}]},{"event_id":"2d286590-c418-4ef0-8d83-5081d5b1689f","dedup_key":"scheduler-c3:c3actual-20260722-batch-002:SCHEDULING_READY:isolated-batch","scope":"scheduler-c3","severity":"resolved","state":"SCHEDULING_READY","previous_state":"","maintenance_window_id":"c3actual-20260722-batch-002","producer":"controlled-batch-submit","summary":"C3 隔离 Batch 已自动选择、完成 SHA-256 回读并清理；未访问 Cloudreve。","evidence_ref":"isolated-batch","occurred_at":1784738314,"received_at":1784738314,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1784738314,"actions":[]},{"event_id":"7cb96e14-1b19-4536-84c9-801c8cecdbc5","dedup_key":"scheduler-c3:c3-20260723-plan-auto:SCHEDULING_READY:plan-auto","scope":"scheduler-c3","severity":"resolved","state":"SCHEDULING_READY","previous_state":"","maintenance_window_id":"c3-20260723-plan-auto","producer":"controlled-batch-submit","summary":"C3 受控自动选择已通过只计划验收；未创建 Nomad Job。","evidence_ref":"plan-auto","occurred_at":1784737493,"received_at":1784737493,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1784737493,"actions":[]},{"event_id":"fc11cc0d-6395-4a0f-9c8e-a2756ee63b32","dedup_key":"c2-alert-20260722-b:HEALTHY:console-alert-4434d2df","scope":"storage-c2","severity":"resolved","state":"HEALTHY","previous_state":"MANUAL_APPROVAL_REQUIRED","maintenance_window_id":"c2-alert-20260722-b","producer":"dd-archive-maintenance-controller","summary":"C2 已经人工批准恢复健康，新的受控写入可重新接受。","evidence_ref":"console-alert-4434d2df","occurred_at":1784730445,"received_at":1784730445,"acknowledged_at":null,"acknowledged_by":null,"read_at":null,"read_by":null,"handling_status":"completed","handler":null,"incident_id":null,"result_summary":"来源控制器已报告恢复","result_evidence_ref":null,"processing_started_at":null,"handled_at":1784730445,"actions":[]},{"event_id":"4434d2df-23c7-45c2-b551-579588ade358","dedup_key":"c2-alert-20260722-b:MANUAL_APPROVAL_REQUIRED:recovery-probe-passed","scope":"storage-c2","severity":"critical","state":"MANUAL_APPROVAL_REQUIRED","previous_state":"","maintenance_window_id":"c2-alert-20260722-b","producer":"dd-archive-maintenance-controller","summary":"C2 当前状态为 MANUAL_APPROVAL_REQUIRED；写入或恢复需要人工处理。","evidence_ref":"recovery-probe-passed","occurred_at":1784730303,"received_at":1784730303,"acknowledged_at":1784730676,"acknowledged_by":"admin","read_at":1784730676,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"历史事件：对应 C2/C3 阶段已闭环，不是当前故障，不提交 AI Boss","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172856,"actions":[{"action":"historical_closed","note":"已按历史归档规则关闭，不再交给 AI Boss","actor":"system","created_at":1785172856}]},{"event_id":"e8cbf523-87c6-4314-8031-87f6efed9ddb","dedup_key":"c2-alert-20260722:MANUAL_APPROVAL_REQUIRED:recovery-probe-passed","scope":"storage-c2","severity":"critical","state":"MANUAL_APPROVAL_REQUIRED","previous_state":"","maintenance_window_id":"c2-alert-20260722","producer":"dd-archive-maintenance-controller","summary":"C2 当前状态为 MANUAL_APPROVAL_REQUIRED；写入或恢复需要人工处理。","evidence_ref":"recovery-probe-passed","occurred_at":1784730266,"received_at":1784730266,"acknowledged_at":1784730677,"acknowledged_by":"admin","read_at":1784730677,"read_by":"admin","handling_status":"completed","handler":null,"incident_id":null,"result_summary":"历史事件：对应 C2/C3 阶段已闭环，不是当前故障，不提交 AI Boss","result_evidence_ref":null,"processing_started_at":null,"handled_at":1785172856,"actions":[{"action":"historical_closed","note":"已按历史归档规则关闭，不再交给 AI Boss","actor":"system","created_at":1785172856}]}],"counts":{"unread":27,"pending":0,"processing":0,"completed":42,"failed":0,"needs_benson_decision":0}}