feat: storage health monitor with UI alert on drive loss
Detect a disconnected or read-only recordings drive and surface it without stopping recording or live view: - storage.go: probe path existence, mount point, writability and free space every 30s; capture mount expectation at startup so a clean unmount (writes falling through to root fs) is also caught - GET /api/storage endpoint; storage fields in /api/status - red UI banner while storage is unavailable - log ALERT on loss, OK on recovery
This commit is contained in:
parent
679c6ae7d6
commit
a54ff08ebb
7 changed files with 257 additions and 0 deletions
21
api.go
21
api.go
|
|
@ -364,9 +364,30 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
"cameras_active": 0, // TODO: real counts in M2
|
"cameras_active": 0, // TODO: real counts in M2
|
||||||
"uptime": "n/a",
|
"uptime": "n/a",
|
||||||
}
|
}
|
||||||
|
if s.storage != nil {
|
||||||
|
st := s.storage.Status()
|
||||||
|
status["storage_ok"] = st.OK
|
||||||
|
status["storage"] = st
|
||||||
|
}
|
||||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: status})
|
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: status})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleStorage returns recordings-storage health.
|
||||||
|
// GET /api/storage
|
||||||
|
func (s *Server) handleStorage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
jsonResponse(w, http.StatusMethodNotAllowed, APIResponse{Error: "method not allowed"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s.storage == nil {
|
||||||
|
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: StorageStatus{
|
||||||
|
OK: true, Message: "storage monitor disabled",
|
||||||
|
}})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: s.storage.Status()})
|
||||||
|
}
|
||||||
|
|
||||||
// handleRecordings returns available recordings for playback.
|
// handleRecordings returns available recordings for playback.
|
||||||
// GET /api/recordings?cam={id}&preset={today|yesterday|week}&from={date}&to={date}
|
// GET /api/recordings?cam={id}&preset={today|yesterday|week}&from={date}&to={date}
|
||||||
//
|
//
|
||||||
|
|
|
||||||
18
main.go
18
main.go
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|
@ -76,8 +77,14 @@ func main() {
|
||||||
go cln.Start()
|
go cln.Start()
|
||||||
app.cleaner = cln
|
app.cleaner = cln
|
||||||
|
|
||||||
|
// Start storage monitor (alert-only; never stops recording or live view).
|
||||||
|
storeMon := NewStorageMonitor(cfg.Storage.RecordingsPath, 30*time.Second)
|
||||||
|
storeMon.Start()
|
||||||
|
app.storage = storeMon
|
||||||
|
|
||||||
// Start the HTTP server (main port, with auth).
|
// Start the HTTP server (main port, with auth).
|
||||||
app.server, _ = NewServer(cfg)
|
app.server, _ = NewServer(cfg)
|
||||||
|
app.server.storage = storeMon
|
||||||
app.server.onReload = func() {
|
app.server.onReload = func() {
|
||||||
log.Println("Config reload — restarting services...")
|
log.Println("Config reload — restarting services...")
|
||||||
if app.go2rtc != nil { app.go2rtc.Stop() }
|
if app.go2rtc != nil { app.go2rtc.Stop() }
|
||||||
|
|
@ -100,6 +107,16 @@ func main() {
|
||||||
rec := NewRecorderManager(liveCfg.Storage)
|
rec := NewRecorderManager(liveCfg.Storage)
|
||||||
rec.StartAll(liveCfg.Cameras)
|
rec.StartAll(liveCfg.Cameras)
|
||||||
app.recorder = rec
|
app.recorder = rec
|
||||||
|
|
||||||
|
// Re-target the storage monitor if the recordings path changed.
|
||||||
|
if app.storage == nil || app.storage.Path() != liveCfg.Storage.RecordingsPath {
|
||||||
|
if app.storage != nil {
|
||||||
|
app.storage.Stop()
|
||||||
|
}
|
||||||
|
app.storage = NewStorageMonitor(liveCfg.Storage.RecordingsPath, 30*time.Second)
|
||||||
|
app.storage.Start()
|
||||||
|
app.server.storage = app.storage
|
||||||
|
}
|
||||||
log.Println("Config reload — services restarted.")
|
log.Println("Config reload — services restarted.")
|
||||||
}
|
}
|
||||||
errCh := make(chan error, 1)
|
errCh := make(chan error, 1)
|
||||||
|
|
@ -159,6 +176,7 @@ type App struct {
|
||||||
snapshots *SnapshotEngine
|
snapshots *SnapshotEngine
|
||||||
recorder *RecorderManager
|
recorder *RecorderManager
|
||||||
cleaner *Cleaner
|
cleaner *Cleaner
|
||||||
|
storage *StorageMonitor
|
||||||
viewerSrv *http.Server
|
viewerSrv *http.Server
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,10 @@ document.addEventListener('DOMContentLoaded', async () => {
|
||||||
renderPlaybackCameras();
|
renderPlaybackCameras();
|
||||||
renderCameraCards();
|
renderCameraCards();
|
||||||
|
|
||||||
|
// Storage health alert (never stops recording; just warns the operator).
|
||||||
|
checkStorage();
|
||||||
|
setInterval(checkStorage, 15000);
|
||||||
|
|
||||||
// Read role from meta tag (set by server).
|
// Read role from meta tag (set by server).
|
||||||
const role = document.querySelector('meta[name="nextnvr-role"]')?.content || '';
|
const role = document.querySelector('meta[name="nextnvr-role"]')?.content || '';
|
||||||
applyRole(role);
|
applyRole(role);
|
||||||
|
|
@ -66,6 +70,26 @@ async function loadStatus() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Storage Health Alert ──
|
||||||
|
async function checkStorage() {
|
||||||
|
const banner = document.getElementById('storage-alert');
|
||||||
|
if (!banner) return;
|
||||||
|
try {
|
||||||
|
const r = await fetch(API + '/storage');
|
||||||
|
const j = await r.json();
|
||||||
|
const st = j.data || {};
|
||||||
|
if (j.success && st.ok === false) {
|
||||||
|
banner.textContent = '⚠ RECORDING STORAGE PROBLEM — ' + (st.message || 'storage unavailable') +
|
||||||
|
' — live view keeps running, but recordings are NOT being saved.';
|
||||||
|
banner.classList.remove('hidden');
|
||||||
|
} else {
|
||||||
|
banner.classList.add('hidden');
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
// Server unreachable — the status dot already reflects that.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadConfig() {
|
async function loadConfig() {
|
||||||
try {
|
try {
|
||||||
const r = await fetch(API + '/config');
|
const r = await fetch(API + '/config');
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,8 @@
|
||||||
<span id="status-dot" class="status offline" title="Server status">●</span>
|
<span id="status-dot" class="status offline" title="Server status">●</span>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
<div id="storage-alert" class="storage-alert hidden"></div>
|
||||||
|
|
||||||
<main id="content">
|
<main id="content">
|
||||||
<!-- Live Wall: 4×2 grid + focus overlay -->
|
<!-- Live Wall: 4×2 grid + focus overlay -->
|
||||||
<section id="tab-live" class="tab-panel active">
|
<section id="tab-live" class="tab-panel active">
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,20 @@ body {
|
||||||
.tab-panel { display: none; padding: 16px; }
|
.tab-panel { display: none; padding: 16px; }
|
||||||
.tab-panel.active { display: block; }
|
.tab-panel.active { display: block; }
|
||||||
|
|
||||||
|
/* ── Storage Alert Banner ── */
|
||||||
|
.storage-alert {
|
||||||
|
background: var(--red);
|
||||||
|
color: #fff;
|
||||||
|
padding: 10px 20px;
|
||||||
|
font-size: 14px;
|
||||||
|
font-weight: 600;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
border-bottom: 1px solid rgba(0,0,0,.3);
|
||||||
|
}
|
||||||
|
.storage-alert.hidden { display: none; }
|
||||||
|
|
||||||
/* ── Live Grid ── */
|
/* ── Live Grid ── */
|
||||||
.grid-4x2 {
|
.grid-4x2 {
|
||||||
display: grid;
|
display: grid;
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ type Server struct {
|
||||||
appConfig *Config
|
appConfig *Config
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
sessions *SessionStore
|
sessions *SessionStore
|
||||||
|
storage *StorageMonitor
|
||||||
onReload func() // called when config is saved via API
|
onReload func() // called when config is saved via API
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -121,6 +122,7 @@ func (s *Server) registerRoutes() {
|
||||||
s.mux.HandleFunc("/api/config/reload", s.handleConfigReload)
|
s.mux.HandleFunc("/api/config/reload", s.handleConfigReload)
|
||||||
s.mux.HandleFunc("/api/scan", s.handleScan)
|
s.mux.HandleFunc("/api/scan", s.handleScan)
|
||||||
s.mux.HandleFunc("/api/status", s.handleStatus)
|
s.mux.HandleFunc("/api/status", s.handleStatus)
|
||||||
|
s.mux.HandleFunc("/api/storage", s.handleStorage)
|
||||||
s.mux.HandleFunc("/api/recordings", s.handleRecordings)
|
s.mux.HandleFunc("/api/recordings", s.handleRecordings)
|
||||||
|
|
||||||
// Auth endpoints.
|
// Auth endpoints.
|
||||||
|
|
|
||||||
176
storage.go
Normal file
176
storage.go
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
// NextNVR — MIT License
|
||||||
|
// Copyright (c) 2026 NextNVR Contributors
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
// storage.go — monitors the recordings storage and raises a visible alert when
|
||||||
|
// the drive disappears or becomes read-only. Recording and live view are NEVER
|
||||||
|
// stopped: the goal is to warn the operator, not to go dark.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StorageStatus is a point-in-time snapshot of recordings-storage health.
|
||||||
|
type StorageStatus struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
Exists bool `json:"exists"`
|
||||||
|
Mounted bool `json:"mounted"`
|
||||||
|
Writable bool `json:"writable"`
|
||||||
|
FreeBytes uint64 `json:"free_bytes"`
|
||||||
|
TotalBytes uint64 `json:"total_bytes"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
CheckedAt string `json:"checked_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageMonitor periodically probes the recordings path and logs transitions.
|
||||||
|
type StorageMonitor struct {
|
||||||
|
path string
|
||||||
|
interval time.Duration
|
||||||
|
expectMount bool
|
||||||
|
mu sync.RWMutex
|
||||||
|
status StorageStatus
|
||||||
|
stopCh chan struct{}
|
||||||
|
started bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorageMonitor creates a monitor. The mount-point expectation is captured
|
||||||
|
// now, while storage is assumed healthy, so a later unmount is detected.
|
||||||
|
func NewStorageMonitor(path string, interval time.Duration) *StorageMonitor {
|
||||||
|
if interval <= 0 {
|
||||||
|
interval = 30 * time.Second
|
||||||
|
}
|
||||||
|
m := &StorageMonitor{
|
||||||
|
path: path,
|
||||||
|
interval: interval,
|
||||||
|
expectMount: isMountPoint(path),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
m.status = checkStorage(path, m.expectMount)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// Path returns the monitored recordings path.
|
||||||
|
func (m *StorageMonitor) Path() string { return m.path }
|
||||||
|
|
||||||
|
// Start begins periodic checks and logs an alert on state changes.
|
||||||
|
func (m *StorageMonitor) Start() {
|
||||||
|
m.started = true
|
||||||
|
prev := m.Status()
|
||||||
|
m.logStatus("startup", prev)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(m.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-m.stopCh:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
st := checkStorage(m.path, m.expectMount)
|
||||||
|
m.mu.Lock()
|
||||||
|
m.status = st
|
||||||
|
m.mu.Unlock()
|
||||||
|
if st.OK != prev.OK {
|
||||||
|
m.logStatus("change", st)
|
||||||
|
}
|
||||||
|
prev = st
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop halts the monitor.
|
||||||
|
func (m *StorageMonitor) Stop() {
|
||||||
|
if m.started {
|
||||||
|
close(m.stopCh)
|
||||||
|
m.started = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the latest cached status.
|
||||||
|
func (m *StorageMonitor) Status() StorageStatus {
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
return m.status
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StorageMonitor) logStatus(tag string, st StorageStatus) {
|
||||||
|
if st.OK {
|
||||||
|
log.Printf("storage: OK (%s) — %s, %s free", tag, st.Path, formatBytes(int64(st.FreeBytes)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Printf("storage: ALERT (%s) — recordings storage problem: %s", tag, st.Message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkStorage probes existence, mount status, free space and writability.
|
||||||
|
func checkStorage(path string, expectMount bool) StorageStatus {
|
||||||
|
st := StorageStatus{Path: path, CheckedAt: time.Now().Format(time.RFC3339)}
|
||||||
|
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
st.Message = "storage path not found: " + err.Error()
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
st.Message = "storage path is not a directory"
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
st.Exists = true
|
||||||
|
st.Mounted = isMountPoint(path)
|
||||||
|
|
||||||
|
var varfs syscall.Statfs_t
|
||||||
|
if syscall.Statfs(path, &varfs) == nil {
|
||||||
|
st.TotalBytes = varfs.Blocks * uint64(varfs.Bsize)
|
||||||
|
st.FreeBytes = varfs.Bavail * uint64(varfs.Bsize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writability probe — catches dead mounts that return EIO on write.
|
||||||
|
probe := filepath.Join(path, ".nextnvr_write_test")
|
||||||
|
f, err := os.OpenFile(probe, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||||
|
if err != nil {
|
||||||
|
st.Message = "storage is not writable: " + err.Error()
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
os.Remove(probe)
|
||||||
|
st.Writable = true
|
||||||
|
|
||||||
|
if expectMount && !st.Mounted {
|
||||||
|
st.Message = "recordings drive is not mounted (it may have been disconnected)"
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
st.OK = true
|
||||||
|
st.Message = "ok"
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// isMountPoint reports whether path is a mount point by comparing its device
|
||||||
|
// ID with that of its parent directory.
|
||||||
|
func isMountPoint(path string) bool {
|
||||||
|
abs, err := filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
child, err := os.Stat(abs)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
parent, err := os.Stat(filepath.Dir(abs))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
cs, ok1 := child.Sys().(*syscall.Stat_t)
|
||||||
|
ps, ok2 := parent.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok1 || !ok2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return cs.Dev != ps.Dev
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue