supervisor.go 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. package main
  2. import (
  3. "bufio"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "os/exec"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "time"
  12. )
  13. // supervisor.go —— dsh web 守护核心:端口预检、隐藏启动、指数退避重启、状态快照。
  14. // healthyUptime 稳定运行判定阈值:超过它则清零失败计数(退避复位)。
  15. const healthyUptime = 30 * time.Second
  16. // Status 服务状态快照。
  17. type Status struct {
  18. Running bool
  19. PID int
  20. Restarts int
  21. Uptime time.Duration
  22. LastError string
  23. URL string
  24. Updating bool
  25. External bool // 端口被外部实例占用(非本程序启动)
  26. }
  27. // Supervisor 守护 dsh web。
  28. type Supervisor struct {
  29. mu sync.Mutex
  30. probeMu sync.Mutex // 串行化入口探测(见 probe.go)
  31. cfg *Config
  32. log *Logger
  33. pid int
  34. running bool
  35. desired bool
  36. restarts int
  37. failures int
  38. startedAt time.Time
  39. lastError string
  40. updating bool
  41. external bool
  42. binCache string
  43. nodeCache string
  44. webURL string // dsh web 输出的完整 URL(含访问 token,仅存内存)
  45. autoMu sync.Mutex
  46. autoLoop bool // 定期更新循环是否已启动(进程生命周期内单例)
  47. }
  48. func NewSupervisor(cfg *Config, log *Logger) *Supervisor {
  49. return &Supervisor{cfg: cfg, log: log}
  50. }
  51. // Start 设置期望运行并拉起守护循环。
  52. func (s *Supervisor) Start() {
  53. s.mu.Lock()
  54. if s.desired {
  55. s.mu.Unlock()
  56. return
  57. }
  58. s.desired = true
  59. s.failures = 0
  60. s.mu.Unlock()
  61. go s.monitor()
  62. }
  63. // Stop 同步停止:结束进程树并等待其真正退出(最多 8 秒)。
  64. func (s *Supervisor) Stop() {
  65. s.mu.Lock()
  66. s.desired = false
  67. pid := s.pid
  68. s.webURL = ""
  69. s.mu.Unlock()
  70. if pid > 0 {
  71. s.log.Printf("停止 dsh web pid=%d", pid)
  72. _ = killTree(pid)
  73. s.waitStopped(pid, 8*time.Second)
  74. }
  75. }
  76. // Restart 重启并等待端口释放,避免新旧实例争抢端口。
  77. func (s *Supervisor) Restart() {
  78. s.log.Printf("重启 dsh web")
  79. s.Stop()
  80. s.waitPortFree(5 * time.Second)
  81. s.Start()
  82. }
  83. func (s *Supervisor) Status() Status {
  84. s.mu.Lock()
  85. defer s.mu.Unlock()
  86. url := s.baseURL()
  87. if s.webURL != "" {
  88. url = s.webURL // 优先使用带 token 的地址
  89. }
  90. var uptime time.Duration
  91. if s.running && !s.startedAt.IsZero() {
  92. uptime = time.Since(s.startedAt)
  93. }
  94. return Status{
  95. Running: s.running,
  96. PID: s.pid,
  97. Restarts: s.restarts,
  98. Uptime: uptime,
  99. LastError: s.lastError,
  100. URL: url,
  101. Updating: s.updating,
  102. External: s.external,
  103. }
  104. }
  105. func (s *Supervisor) baseURL() string {
  106. host := s.cfg.WebHost
  107. if host == "" {
  108. host = "127.0.0.1"
  109. }
  110. return fmt.Sprintf("http://%s:%d", host, s.cfg.WebPort)
  111. }
  112. func (s *Supervisor) setError(err error) {
  113. s.mu.Lock()
  114. s.lastError = err.Error()
  115. s.mu.Unlock()
  116. s.log.Printf("守护错误: %v", err)
  117. }
  118. // maxBackoff 重启退避上限。
  119. const maxBackoff = 2 * time.Minute
  120. // backoff 指数退避:base * 2^failures,封顶 maxBackoff。
  121. // 循环内提前 break,既保证封顶语义,也避免超大 failures 造成数值溢出。
  122. func (s *Supervisor) backoff() time.Duration {
  123. s.mu.Lock()
  124. failures := s.failures
  125. base := s.cfg.RestartDelaySec
  126. s.mu.Unlock()
  127. if base <= 0 {
  128. base = 5
  129. }
  130. d := time.Duration(base) * time.Second
  131. for i := 0; i < failures; i++ {
  132. if d >= maxBackoff {
  133. break
  134. }
  135. d *= 2
  136. }
  137. if d > maxBackoff {
  138. d = maxBackoff
  139. }
  140. return d
  141. }
  142. // portBusyBackoff 端口占用时的温和退避(上限 30 秒):不是故障,只是等待外部实例释放。
  143. func (s *Supervisor) portBusyBackoff() time.Duration {
  144. d := s.backoff()
  145. if d > 30*time.Second {
  146. d = 30 * time.Second
  147. }
  148. return d
  149. }
  150. // monitor 守护循环:端口预检 -> 启动 -> 等待退出 -> 期望运行时退避重启。
  151. func (s *Supervisor) monitor() {
  152. for {
  153. if !s.isDesired() {
  154. return
  155. }
  156. // 端口预检:已有实例监听时不再启动,避免端口冲突导致的重启风暴
  157. if isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  158. s.mu.Lock()
  159. s.running = false
  160. s.pid = 0
  161. s.external = true
  162. s.lastError = "端口已被占用(可能已有 dsh web 在运行)"
  163. s.mu.Unlock()
  164. s.log.Printf("端口 %s 已被占用,跳过启动(等待释放)", s.baseURL())
  165. s.bumpFailure()
  166. s.sleepInterruptible(s.portBusyBackoff())
  167. continue
  168. }
  169. s.mu.Lock()
  170. s.external = false
  171. s.mu.Unlock()
  172. exe, args, err := s.resolveCommand()
  173. if err != nil {
  174. s.setError(err)
  175. s.bumpFailure()
  176. s.sleepInterruptible(s.backoff())
  177. continue
  178. }
  179. cmd := exec.Command(exe, args...)
  180. hideWindow(cmd)
  181. stdout, errOut := cmd.StdoutPipe()
  182. stderr, errErr := cmd.StderrPipe()
  183. if errOut != nil || errErr != nil {
  184. s.setError(errors.New("无法创建输出管道"))
  185. s.bumpFailure()
  186. s.sleepInterruptible(s.backoff())
  187. continue
  188. }
  189. if err := cmd.Start(); err != nil {
  190. s.setError(err)
  191. s.bumpFailure()
  192. s.sleepInterruptible(s.backoff())
  193. continue
  194. }
  195. pid := cmd.Process.Pid
  196. startedAt := time.Now()
  197. s.mu.Lock()
  198. s.pid = pid
  199. s.running = true
  200. s.startedAt = startedAt
  201. s.lastError = ""
  202. s.webURL = ""
  203. s.mu.Unlock()
  204. s.log.Printf("dsh web 已启动 pid=%d: %s %s", pid, exe, strings.Join(args, " "))
  205. go s.pipeLog("out", stdout)
  206. go s.pipeLog("err", stderr)
  207. waitErr := cmd.Wait()
  208. ranFor := time.Since(startedAt)
  209. s.mu.Lock()
  210. s.running = false
  211. s.pid = 0
  212. stillDesired := s.desired
  213. if ranFor >= healthyUptime {
  214. s.failures = 0 // 稳定运行过,退避复位
  215. } else {
  216. s.failures++
  217. }
  218. if stillDesired {
  219. s.restarts++
  220. }
  221. s.mu.Unlock()
  222. s.log.Printf("dsh web 退出(err=%v,运行 %s),期望运行=%v", waitErr, ranFor.Round(time.Second), stillDesired)
  223. if !stillDesired {
  224. return
  225. }
  226. delay := s.backoff()
  227. s.log.Printf("%s 后自动重启", delay)
  228. s.sleepInterruptible(delay)
  229. }
  230. }
  231. func (s *Supervisor) isDesired() bool {
  232. s.mu.Lock()
  233. defer s.mu.Unlock()
  234. return s.desired
  235. }
  236. // bumpFailure 增加失败计数(触发退避放大)。
  237. func (s *Supervisor) bumpFailure() {
  238. s.mu.Lock()
  239. s.failures++
  240. s.mu.Unlock()
  241. }
  242. func (s *Supervisor) sleepInterruptible(d time.Duration) {
  243. deadline := time.Now().Add(d)
  244. for time.Now().Before(deadline) {
  245. if !s.isDesired() {
  246. return
  247. }
  248. time.Sleep(200 * time.Millisecond)
  249. }
  250. }
  251. func (s *Supervisor) waitStopped(pid int, timeout time.Duration) {
  252. deadline := time.Now().Add(timeout)
  253. for time.Now().Before(deadline) {
  254. if !processAlive(pid) {
  255. return
  256. }
  257. time.Sleep(100 * time.Millisecond)
  258. }
  259. s.log.Printf("等待 pid=%d 退出超时(继续)", pid)
  260. }
  261. func (s *Supervisor) waitPortFree(timeout time.Duration) {
  262. deadline := time.Now().Add(timeout)
  263. for time.Now().Before(deadline) {
  264. if !isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  265. return
  266. }
  267. time.Sleep(150 * time.Millisecond)
  268. }
  269. }
  270. // pipeLog 把子进程输出写入日志(token 由 Logger 统一脱敏),并捕获访问地址。
  271. func (s *Supervisor) pipeLog(tag string, r io.ReadCloser) {
  272. if r == nil {
  273. return
  274. }
  275. sc := bufio.NewScanner(r)
  276. sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
  277. for sc.Scan() {
  278. line := sc.Text()
  279. s.log.Printf("[dsh %s] %s", tag, line)
  280. if url := extractHTTPURL(line); url != "" {
  281. s.mu.Lock()
  282. if url != s.webURL {
  283. s.webURL = url
  284. s.log.Printf("捕获 Web 访问地址(含 token,仅存内存)")
  285. }
  286. s.mu.Unlock()
  287. }
  288. }
  289. }
  290. // resolveCommand 解析启动命令:优先 node + dsh 入口 JS(无 shell 包装,避免弹窗与注入面)。
  291. func (s *Supervisor) resolveCommand() (string, []string, error) {
  292. node := s.cfg.NodePath
  293. if node == "" {
  294. node = s.nodePath()
  295. }
  296. binJS := s.cfg.DshBinJS
  297. if binJS == "" {
  298. binJS = s.dshBinJS()
  299. }
  300. port := strconv.Itoa(s.cfg.WebPort)
  301. if node != "" && binJS != "" {
  302. return node, []string{binJS, "web", "--no-open", "--host", s.cfg.WebHost, "--port", port}, nil
  303. }
  304. if p, err := exec.LookPath("dsh.cmd"); err == nil {
  305. return "cmd.exe", []string{"/c", p, "web", "--no-open", "--port", port}, nil
  306. }
  307. if p, err := exec.LookPath("dsh"); err == nil {
  308. return p, []string{"web", "--no-open", "--port", port}, nil
  309. }
  310. return "", nil, errors.New("未找到 dsh 入口:请确认已全局安装 @deepseek-ai/dsh,或在配置中指定 nodePath / dshBinJs")
  311. }