fix: config reload with health polling, not blind timeout
writeConfigAndReload: 发送 reload 后每 2s 轮询 media-server 健康状态,60s 兜底回滚。替代原来的 10s 硬超时。
This commit is contained in:
parent
e8cc66636c
commit
ae7e9e8bfc
@ -417,24 +417,54 @@ func (s *Server) writeConfigAndReload(ctx context.Context, body []byte, restoreB
|
||||
return fmt.Errorf("write config failed: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.ms.Reload(ctx); err != nil {
|
||||
rerr := err
|
||||
if len(restoreBody) == 0 {
|
||||
return fmt.Errorf("reload failed: %v", rerr)
|
||||
// Signal media-server to reload (short timeout — just the HTTP call).
|
||||
reloadCtx, reloadCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer reloadCancel()
|
||||
_ = s.ms.Reload(reloadCtx) // ignore immediate result; poll status instead
|
||||
|
||||
// Wait for media-server to recover with the new config.
|
||||
// The media-server may restart its pipeline, which can take 10-30 seconds.
|
||||
deadline := time.After(60 * time.Second)
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-deadline:
|
||||
// Timed out — roll back.
|
||||
if len(restoreBody) == 0 {
|
||||
return fmt.Errorf("media-server did not recover within 60s")
|
||||
}
|
||||
if werr := files.WriteFileAtomic(s.agentCfg.ConfigPath, append(restoreBody, '\n'), 0o644); werr != nil {
|
||||
return fmt.Errorf("media-server recovery timeout; restore write failed: %v", werr)
|
||||
}
|
||||
_ = s.ms.Reload(context.Background())
|
||||
return fmt.Errorf("media-server did not recover within 60s; restored previous config")
|
||||
case <-ticker.C:
|
||||
if s.mediaServerHealthy() {
|
||||
return nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if werr := files.WriteFileAtomic(s.agentCfg.ConfigPath, append(restoreBody, '\n'), 0o644); werr != nil {
|
||||
return fmt.Errorf("reload failed: %v; restore write failed: %v", rerr, werr)
|
||||
}
|
||||
restoreCtx, restoreCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer restoreCancel()
|
||||
if restoreErr := s.ms.Reload(restoreCtx); restoreErr != nil {
|
||||
return fmt.Errorf("reload failed: %v; restore reload failed: %v", rerr, restoreErr)
|
||||
}
|
||||
return fmt.Errorf("reload failed: %v; restored previous config", rerr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mediaServerHealthy checks whether the media-server process is running
|
||||
// and responding to API requests.
|
||||
func (s *Server) mediaServerHealthy() bool {
|
||||
if s.proc == nil || !s.proc.Enabled() {
|
||||
return true // no process management — assume OK
|
||||
}
|
||||
st, err := s.proc.Status()
|
||||
if err != nil || !st.Running {
|
||||
return false
|
||||
}
|
||||
// Quick API health check (1s timeout).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
_, _, err = s.ms.GetGraphs(ctx)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s *Server) applyCandidateConfigBytes(ctx context.Context, body []byte) error {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user