feat: storage health monitor with UI alert on drive loss
Detect a disconnected or read-only recordings drive and surface it without stopping recording or live view: - storage.go: probe path existence, mount point, writability and free space every 30s; capture mount expectation at startup so a clean unmount (writes falling through to root fs) is also caught - GET /api/storage endpoint; storage fields in /api/status - red UI banner while storage is unavailable - log ALERT on loss, OK on recovery
This commit is contained in:
parent
679c6ae7d6
commit
a54ff08ebb
7 changed files with 257 additions and 0 deletions
21
api.go
21
api.go
|
|
@ -364,9 +364,30 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
|
|||
"cameras_active": 0, // TODO: real counts in M2
|
||||
"uptime": "n/a",
|
||||
}
|
||||
if s.storage != nil {
|
||||
st := s.storage.Status()
|
||||
status["storage_ok"] = st.OK
|
||||
status["storage"] = st
|
||||
}
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: status})
|
||||
}
|
||||
|
||||
// handleStorage returns recordings-storage health.
|
||||
// GET /api/storage
|
||||
func (s *Server) handleStorage(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
jsonResponse(w, http.StatusMethodNotAllowed, APIResponse{Error: "method not allowed"})
|
||||
return
|
||||
}
|
||||
if s.storage == nil {
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: StorageStatus{
|
||||
OK: true, Message: "storage monitor disabled",
|
||||
}})
|
||||
return
|
||||
}
|
||||
jsonResponse(w, http.StatusOK, APIResponse{Success: true, Data: s.storage.Status()})
|
||||
}
|
||||
|
||||
// handleRecordings returns available recordings for playback.
|
||||
// GET /api/recordings?cam={id}&preset={today|yesterday|week}&from={date}&to={date}
|
||||
//
|
||||
|
|
|
|||
18
main.go
18
main.go
|
|
@ -12,6 +12,7 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -76,8 +77,14 @@ func main() {
|
|||
go cln.Start()
|
||||
app.cleaner = cln
|
||||
|
||||
// Start storage monitor (alert-only; never stops recording or live view).
|
||||
storeMon := NewStorageMonitor(cfg.Storage.RecordingsPath, 30*time.Second)
|
||||
storeMon.Start()
|
||||
app.storage = storeMon
|
||||
|
||||
// Start the HTTP server (main port, with auth).
|
||||
app.server, _ = NewServer(cfg)
|
||||
app.server.storage = storeMon
|
||||
app.server.onReload = func() {
|
||||
log.Println("Config reload — restarting services...")
|
||||
if app.go2rtc != nil { app.go2rtc.Stop() }
|
||||
|
|
@ -100,6 +107,16 @@ func main() {
|
|||
rec := NewRecorderManager(liveCfg.Storage)
|
||||
rec.StartAll(liveCfg.Cameras)
|
||||
app.recorder = rec
|
||||
|
||||
// Re-target the storage monitor if the recordings path changed.
|
||||
if app.storage == nil || app.storage.Path() != liveCfg.Storage.RecordingsPath {
|
||||
if app.storage != nil {
|
||||
app.storage.Stop()
|
||||
}
|
||||
app.storage = NewStorageMonitor(liveCfg.Storage.RecordingsPath, 30*time.Second)
|
||||
app.storage.Start()
|
||||
app.server.storage = app.storage
|
||||
}
|
||||
log.Println("Config reload — services restarted.")
|
||||
}
|
||||
errCh := make(chan error, 1)
|
||||
|
|
@ -159,6 +176,7 @@ type App struct {
|
|||
snapshots *SnapshotEngine
|
||||
recorder *RecorderManager
|
||||
cleaner *Cleaner
|
||||
storage *StorageMonitor
|
||||
viewerSrv *http.Server
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ document.addEventListener('DOMContentLoaded', async () => {
|
|||
renderPlaybackCameras();
|
||||
renderCameraCards();
|
||||
|
||||
// Storage health alert (never stops recording; just warns the operator).
|
||||
checkStorage();
|
||||
setInterval(checkStorage, 15000);
|
||||
|
||||
// Read role from meta tag (set by server).
|
||||
const role = document.querySelector('meta[name="nextnvr-role"]')?.content || '';
|
||||
applyRole(role);
|
||||
|
|
@ -66,6 +70,26 @@ async function loadStatus() {
|
|||
}
|
||||
}
|
||||
|
||||
// ── Storage Health Alert ──
|
||||
async function checkStorage() {
|
||||
const banner = document.getElementById('storage-alert');
|
||||
if (!banner) return;
|
||||
try {
|
||||
const r = await fetch(API + '/storage');
|
||||
const j = await r.json();
|
||||
const st = j.data || {};
|
||||
if (j.success && st.ok === false) {
|
||||
banner.textContent = '⚠ RECORDING STORAGE PROBLEM — ' + (st.message || 'storage unavailable') +
|
||||
' — live view keeps running, but recordings are NOT being saved.';
|
||||
banner.classList.remove('hidden');
|
||||
} else {
|
||||
banner.classList.add('hidden');
|
||||
}
|
||||
} catch(e) {
|
||||
// Server unreachable — the status dot already reflects that.
|
||||
}
|
||||
}
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
const r = await fetch(API + '/config');
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@
|
|||
<span id="status-dot" class="status offline" title="Server status">●</span>
|
||||
</header>
|
||||
|
||||
<div id="storage-alert" class="storage-alert hidden"></div>
|
||||
|
||||
<main id="content">
|
||||
<!-- Live Wall: 4×2 grid + focus overlay -->
|
||||
<section id="tab-live" class="tab-panel active">
|
||||
|
|
|
|||
|
|
@ -48,6 +48,20 @@ body {
|
|||
.tab-panel { display: none; padding: 16px; }
|
||||
.tab-panel.active { display: block; }
|
||||
|
||||
/* ── Storage Alert Banner ── */
|
||||
.storage-alert {
|
||||
background: var(--red);
|
||||
color: #fff;
|
||||
padding: 10px 20px;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
border-bottom: 1px solid rgba(0,0,0,.3);
|
||||
}
|
||||
.storage-alert.hidden { display: none; }
|
||||
|
||||
/* ── Live Grid ── */
|
||||
.grid-4x2 {
|
||||
display: grid;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ type Server struct {
|
|||
appConfig *Config
|
||||
mu sync.RWMutex
|
||||
sessions *SessionStore
|
||||
storage *StorageMonitor
|
||||
onReload func() // called when config is saved via API
|
||||
}
|
||||
|
||||
|
|
@ -121,6 +122,7 @@ func (s *Server) registerRoutes() {
|
|||
s.mux.HandleFunc("/api/config/reload", s.handleConfigReload)
|
||||
s.mux.HandleFunc("/api/scan", s.handleScan)
|
||||
s.mux.HandleFunc("/api/status", s.handleStatus)
|
||||
s.mux.HandleFunc("/api/storage", s.handleStorage)
|
||||
s.mux.HandleFunc("/api/recordings", s.handleRecordings)
|
||||
|
||||
// Auth endpoints.
|
||||
|
|
|
|||
176
storage.go
Normal file
176
storage.go
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
// NextNVR — MIT License
|
||||
// Copyright (c) 2026 NextNVR Contributors
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// storage.go — monitors the recordings storage and raises a visible alert when
|
||||
// the drive disappears or becomes read-only. Recording and live view are NEVER
|
||||
// stopped: the goal is to warn the operator, not to go dark.
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StorageStatus is a point-in-time snapshot of recordings-storage health.
|
||||
type StorageStatus struct {
|
||||
OK bool `json:"ok"`
|
||||
Path string `json:"path"`
|
||||
Exists bool `json:"exists"`
|
||||
Mounted bool `json:"mounted"`
|
||||
Writable bool `json:"writable"`
|
||||
FreeBytes uint64 `json:"free_bytes"`
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
Message string `json:"message"`
|
||||
CheckedAt string `json:"checked_at"`
|
||||
}
|
||||
|
||||
// StorageMonitor periodically probes the recordings path and logs transitions.
|
||||
type StorageMonitor struct {
|
||||
path string
|
||||
interval time.Duration
|
||||
expectMount bool
|
||||
mu sync.RWMutex
|
||||
status StorageStatus
|
||||
stopCh chan struct{}
|
||||
started bool
|
||||
}
|
||||
|
||||
// NewStorageMonitor creates a monitor. The mount-point expectation is captured
|
||||
// now, while storage is assumed healthy, so a later unmount is detected.
|
||||
func NewStorageMonitor(path string, interval time.Duration) *StorageMonitor {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
m := &StorageMonitor{
|
||||
path: path,
|
||||
interval: interval,
|
||||
expectMount: isMountPoint(path),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
m.status = checkStorage(path, m.expectMount)
|
||||
return m
|
||||
}
|
||||
|
||||
// Path returns the monitored recordings path.
|
||||
func (m *StorageMonitor) Path() string { return m.path }
|
||||
|
||||
// Start begins periodic checks and logs an alert on state changes.
|
||||
func (m *StorageMonitor) Start() {
|
||||
m.started = true
|
||||
prev := m.Status()
|
||||
m.logStatus("startup", prev)
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
st := checkStorage(m.path, m.expectMount)
|
||||
m.mu.Lock()
|
||||
m.status = st
|
||||
m.mu.Unlock()
|
||||
if st.OK != prev.OK {
|
||||
m.logStatus("change", st)
|
||||
}
|
||||
prev = st
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop halts the monitor.
|
||||
func (m *StorageMonitor) Stop() {
|
||||
if m.started {
|
||||
close(m.stopCh)
|
||||
m.started = false
|
||||
}
|
||||
}
|
||||
|
||||
// Status returns the latest cached status.
|
||||
func (m *StorageMonitor) Status() StorageStatus {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.status
|
||||
}
|
||||
|
||||
func (m *StorageMonitor) logStatus(tag string, st StorageStatus) {
|
||||
if st.OK {
|
||||
log.Printf("storage: OK (%s) — %s, %s free", tag, st.Path, formatBytes(int64(st.FreeBytes)))
|
||||
return
|
||||
}
|
||||
log.Printf("storage: ALERT (%s) — recordings storage problem: %s", tag, st.Message)
|
||||
}
|
||||
|
||||
// checkStorage probes existence, mount status, free space and writability.
|
||||
func checkStorage(path string, expectMount bool) StorageStatus {
|
||||
st := StorageStatus{Path: path, CheckedAt: time.Now().Format(time.RFC3339)}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
st.Message = "storage path not found: " + err.Error()
|
||||
return st
|
||||
}
|
||||
if !info.IsDir() {
|
||||
st.Message = "storage path is not a directory"
|
||||
return st
|
||||
}
|
||||
st.Exists = true
|
||||
st.Mounted = isMountPoint(path)
|
||||
|
||||
var varfs syscall.Statfs_t
|
||||
if syscall.Statfs(path, &varfs) == nil {
|
||||
st.TotalBytes = varfs.Blocks * uint64(varfs.Bsize)
|
||||
st.FreeBytes = varfs.Bavail * uint64(varfs.Bsize)
|
||||
}
|
||||
|
||||
// Writability probe — catches dead mounts that return EIO on write.
|
||||
probe := filepath.Join(path, ".nextnvr_write_test")
|
||||
f, err := os.OpenFile(probe, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
st.Message = "storage is not writable: " + err.Error()
|
||||
return st
|
||||
}
|
||||
f.Close()
|
||||
os.Remove(probe)
|
||||
st.Writable = true
|
||||
|
||||
if expectMount && !st.Mounted {
|
||||
st.Message = "recordings drive is not mounted (it may have been disconnected)"
|
||||
return st
|
||||
}
|
||||
|
||||
st.OK = true
|
||||
st.Message = "ok"
|
||||
return st
|
||||
}
|
||||
|
||||
// isMountPoint reports whether path is a mount point by comparing its device
|
||||
// ID with that of its parent directory.
|
||||
func isMountPoint(path string) bool {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
child, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
parent, err := os.Stat(filepath.Dir(abs))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
cs, ok1 := child.Sys().(*syscall.Stat_t)
|
||||
ps, ok2 := parent.Sys().(*syscall.Stat_t)
|
||||
if !ok1 || !ok2 {
|
||||
return false
|
||||
}
|
||||
return cs.Dev != ps.Dev
|
||||
}
|
||||
Loading…
Reference in a new issue