fix: config reload with health polling, not blind timeout

writeConfigAndReload: 发送 reload 后每 2s 轮询 media-server
健康状态,60s 兜底回滚。替代原来的 10s 硬超时。
This commit is contained in:
tian 2026-05-12 15:00:09 +08:00
parent e8cc66636c
commit ae7e9e8bfc

View File

@ -417,24 +417,54 @@ func (s *Server) writeConfigAndReload(ctx context.Context, body []byte, restoreB
return fmt.Errorf("write config failed: %w", err)
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := s.ms.Reload(ctx); err != nil {
rerr := err
if len(restoreBody) == 0 {
return fmt.Errorf("reload failed: %v", rerr)
// Signal media-server to reload (short timeout — just the HTTP call).
reloadCtx, reloadCancel := context.WithTimeout(ctx, 5*time.Second)
defer reloadCancel()
_ = s.ms.Reload(reloadCtx) // ignore immediate result; poll status instead
// Wait for media-server to recover with the new config.
// The media-server may restart its pipeline, which can take 10-30 seconds.
deadline := time.After(60 * time.Second)
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-deadline:
// Timed out — roll back.
if len(restoreBody) == 0 {
return fmt.Errorf("media-server did not recover within 60s")
}
if werr := files.WriteFileAtomic(s.agentCfg.ConfigPath, append(restoreBody, '\n'), 0o644); werr != nil {
return fmt.Errorf("media-server recovery timeout; restore write failed: %v", werr)
}
_ = s.ms.Reload(context.Background())
return fmt.Errorf("media-server did not recover within 60s; restored previous config")
case <-ticker.C:
if s.mediaServerHealthy() {
return nil
}
case <-ctx.Done():
return ctx.Err()
}
if werr := files.WriteFileAtomic(s.agentCfg.ConfigPath, append(restoreBody, '\n'), 0o644); werr != nil {
return fmt.Errorf("reload failed: %v; restore write failed: %v", rerr, werr)
}
restoreCtx, restoreCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer restoreCancel()
if restoreErr := s.ms.Reload(restoreCtx); restoreErr != nil {
return fmt.Errorf("reload failed: %v; restore reload failed: %v", rerr, restoreErr)
}
return fmt.Errorf("reload failed: %v; restored previous config", rerr)
}
return nil
}
// mediaServerHealthy checks whether the media-server process is running
// and responding to API requests.
func (s *Server) mediaServerHealthy() bool {
if s.proc == nil || !s.proc.Enabled() {
return true // no process management — assume OK
}
st, err := s.proc.Status()
if err != nil || !st.Running {
return false
}
// Quick API health check (1s timeout).
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
_, _, err = s.ms.GetGraphs(ctx)
return err == nil
}
func (s *Server) applyCandidateConfigBytes(ctx context.Context, body []byte) error {