A power cut makes the recordings drive's fsck slow, so the mount can appear tens of seconds after nextnvr starts. The recorders then hit the unmounted, root-owned mountpoint, failed mkdir, and exited permanently — recording stayed dead until a manual restart. - recorder: on mkdir failure, log and retry every 15s instead of exiting, so recording resumes once storage returns - storage: add waitForStorage() and call it before the writers start, so startup waits (up to 120s) for the recordings dir to be present and writable; live view still starts if storage never appears
202 lines
5.2 KiB
Go
202 lines
5.2 KiB
Go
// NextNVR — MIT License
|
|
// Copyright (c) 2026 NextNVR Contributors
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
// storage.go — monitors the recordings storage and raises a visible alert when
|
|
// the drive disappears or becomes read-only. Recording and live view are NEVER
|
|
// stopped: the goal is to warn the operator, not to go dark.
|
|
package main
|
|
|
|
import (
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// StorageStatus is a point-in-time snapshot of recordings-storage health.
|
|
type StorageStatus struct {
|
|
OK bool `json:"ok"`
|
|
Path string `json:"path"`
|
|
Exists bool `json:"exists"`
|
|
Mounted bool `json:"mounted"`
|
|
Writable bool `json:"writable"`
|
|
FreeBytes uint64 `json:"free_bytes"`
|
|
TotalBytes uint64 `json:"total_bytes"`
|
|
Message string `json:"message"`
|
|
CheckedAt string `json:"checked_at"`
|
|
}
|
|
|
|
// StorageMonitor periodically probes the recordings path and logs transitions.
|
|
type StorageMonitor struct {
|
|
path string
|
|
interval time.Duration
|
|
expectMount bool
|
|
mu sync.RWMutex
|
|
status StorageStatus
|
|
stopCh chan struct{}
|
|
started bool
|
|
}
|
|
|
|
// NewStorageMonitor creates a monitor. The mount-point expectation is captured
|
|
// now, while storage is assumed healthy, so a later unmount is detected.
|
|
func NewStorageMonitor(path string, interval time.Duration) *StorageMonitor {
|
|
if interval <= 0 {
|
|
interval = 30 * time.Second
|
|
}
|
|
m := &StorageMonitor{
|
|
path: path,
|
|
interval: interval,
|
|
expectMount: isMountPoint(path),
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
m.status = checkStorage(path, m.expectMount)
|
|
return m
|
|
}
|
|
|
|
// Path returns the monitored recordings path.
|
|
func (m *StorageMonitor) Path() string { return m.path }
|
|
|
|
// Start begins periodic checks and logs an alert on state changes.
|
|
func (m *StorageMonitor) Start() {
|
|
m.started = true
|
|
prev := m.Status()
|
|
m.logStatus("startup", prev)
|
|
|
|
go func() {
|
|
ticker := time.NewTicker(m.interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-m.stopCh:
|
|
return
|
|
case <-ticker.C:
|
|
st := checkStorage(m.path, m.expectMount)
|
|
m.mu.Lock()
|
|
m.status = st
|
|
m.mu.Unlock()
|
|
if st.OK != prev.OK {
|
|
m.logStatus("change", st)
|
|
}
|
|
prev = st
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Stop halts the monitor.
|
|
func (m *StorageMonitor) Stop() {
|
|
if m.started {
|
|
close(m.stopCh)
|
|
m.started = false
|
|
}
|
|
}
|
|
|
|
// Status returns the latest cached status.
|
|
func (m *StorageMonitor) Status() StorageStatus {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.status
|
|
}
|
|
|
|
func (m *StorageMonitor) logStatus(tag string, st StorageStatus) {
|
|
if st.OK {
|
|
log.Printf("storage: OK (%s) — %s, %s free", tag, st.Path, formatBytes(int64(st.FreeBytes)))
|
|
return
|
|
}
|
|
log.Printf("storage: ALERT (%s) — recordings storage problem: %s", tag, st.Message)
|
|
}
|
|
|
|
// checkStorage probes existence, mount status, free space and writability.
|
|
func checkStorage(path string, expectMount bool) StorageStatus {
|
|
st := StorageStatus{Path: path, CheckedAt: time.Now().Format(time.RFC3339)}
|
|
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
st.Message = "storage path not found: " + err.Error()
|
|
return st
|
|
}
|
|
if !info.IsDir() {
|
|
st.Message = "storage path is not a directory"
|
|
return st
|
|
}
|
|
st.Exists = true
|
|
st.Mounted = isMountPoint(path)
|
|
|
|
var varfs syscall.Statfs_t
|
|
if syscall.Statfs(path, &varfs) == nil {
|
|
st.TotalBytes = varfs.Blocks * uint64(varfs.Bsize)
|
|
st.FreeBytes = varfs.Bavail * uint64(varfs.Bsize)
|
|
}
|
|
|
|
// Writability probe — catches dead mounts that return EIO on write.
|
|
probe := filepath.Join(path, ".nextnvr_write_test")
|
|
f, err := os.OpenFile(probe, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
|
if err != nil {
|
|
st.Message = "storage is not writable: " + err.Error()
|
|
return st
|
|
}
|
|
f.Close()
|
|
os.Remove(probe)
|
|
st.Writable = true
|
|
|
|
if expectMount && !st.Mounted {
|
|
st.Message = "recordings drive is not mounted (it may have been disconnected)"
|
|
return st
|
|
}
|
|
|
|
st.OK = true
|
|
st.Message = "ok"
|
|
return st
|
|
}
|
|
|
|
// isMountPoint reports whether path is a mount point by comparing its device
|
|
// ID with that of its parent directory.
|
|
func isMountPoint(path string) bool {
|
|
abs, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
child, err := os.Stat(abs)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
parent, err := os.Stat(filepath.Dir(abs))
|
|
if err != nil {
|
|
return false
|
|
}
|
|
cs, ok1 := child.Sys().(*syscall.Stat_t)
|
|
ps, ok2 := parent.Sys().(*syscall.Stat_t)
|
|
if !ok1 || !ok2 {
|
|
return false
|
|
}
|
|
return cs.Dev != ps.Dev
|
|
}
|
|
|
|
// waitForStorage blocks until the recordings path exists and is writable, or
|
|
// until the timeout elapses. It is meant to run before the writers start: on a
|
|
// slow boot (e.g. an fsck on the recordings drive after a power cut) the mount
|
|
// can appear tens of seconds late, and without this the recorders would stumble
|
|
// against an unmounted, root-owned mountpoint. Returns the final status; if the
|
|
// timeout expires the caller may still proceed (live view does not need storage,
|
|
// and the recorder retries on its own).
|
|
func waitForStorage(path string, timeout time.Duration) StorageStatus {
|
|
deadline := time.Now().Add(timeout)
|
|
warned := false
|
|
for {
|
|
st := checkStorage(path, false)
|
|
if st.OK {
|
|
return st
|
|
}
|
|
if !warned {
|
|
log.Printf("startup: waiting for recordings storage at %s (%s)", path, st.Message)
|
|
warned = true
|
|
}
|
|
if !time.Now().Before(deadline) {
|
|
return st
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
}
|