supervisor.go 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. package main
  2. import (
  3. "bufio"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "os/exec"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "time"
  12. )
  13. // supervisor.go —— dsh web 守护核心:端口预检、隐藏启动、指数退避重启、状态快照。
  14. // healthyUptime 稳定运行判定阈值:超过它则清零失败计数(退避复位)。
  15. const healthyUptime = 30 * time.Second
  16. // Status 服务状态快照。
  17. type Status struct {
  18. Running bool
  19. PID int
  20. Restarts int
  21. Uptime time.Duration
  22. LastError string
  23. URL string
  24. Updating bool
  25. External bool // 端口被外部实例占用(非本程序启动)
  26. }
  27. // Supervisor 守护 dsh web。
  28. type Supervisor struct {
  29. mu sync.Mutex
  30. probeMu sync.Mutex // 串行化入口探测(见 probe.go)
  31. cfg *Config
  32. log *Logger
  33. pid int
  34. running bool
  35. desired bool
  36. restarts int
  37. failures int
  38. startedAt time.Time
  39. lastError string
  40. updating bool
  41. external bool
  42. binCache string
  43. nodeCache string
  44. webURL string // dsh web 输出的完整 URL(含访问 token,仅存内存)
  45. autoMu sync.Mutex
  46. autoLoop bool // 定期更新循环是否已启动(进程生命周期内单例)
  47. }
  48. func NewSupervisor(cfg *Config, log *Logger) *Supervisor {
  49. return &Supervisor{cfg: cfg, log: log}
  50. }
  51. // Start 设置期望运行并拉起守护循环。
  52. func (s *Supervisor) Start() {
  53. s.mu.Lock()
  54. if s.desired {
  55. s.mu.Unlock()
  56. return
  57. }
  58. s.desired = true
  59. s.failures = 0
  60. s.mu.Unlock()
  61. go s.monitor()
  62. }
  63. // Stop 同步停止:结束进程树并等待其真正退出(最多 8 秒)。
  64. func (s *Supervisor) Stop() {
  65. s.mu.Lock()
  66. s.desired = false
  67. pid := s.pid
  68. s.webURL = ""
  69. s.mu.Unlock()
  70. if pid > 0 {
  71. s.log.Printf("停止 dsh web pid=%d", pid)
  72. _ = killTree(pid)
  73. s.waitStopped(pid, 8*time.Second)
  74. }
  75. }
  76. // SetAutoUpdate 设置定期更新开关(加锁 + 立即持久化;开启时确保循环已启动)。
  77. func (s *Supervisor) SetAutoUpdate(enabled bool) error {
  78. s.mu.Lock()
  79. s.cfg.AutoUpdate = enabled
  80. snapshot := *s.cfg
  81. s.mu.Unlock()
  82. if enabled {
  83. s.startAutoUpdate()
  84. }
  85. return saveConfig(&snapshot)
  86. }
  87. // SetAutoStart 设置开机自启开关(加锁 + 立即持久化;注册表由调用方处理)。
  88. func (s *Supervisor) SetAutoStart(enabled bool) error {
  89. s.mu.Lock()
  90. s.cfg.AutoStart = enabled
  91. snapshot := *s.cfg
  92. s.mu.Unlock()
  93. return saveConfig(&snapshot)
  94. }
  95. // UpdateAndRestart 先停止服务再更新,避免 Windows 上 native 模块文件被占用导致安装失败;
  96. // 无论更新成功与否都恢复到原有运行意图。
  97. func (s *Supervisor) UpdateAndRestart() (string, error) {
  98. wasDesired := s.isDesired()
  99. if wasDesired {
  100. s.Stop()
  101. }
  102. out, err := s.Update()
  103. if wasDesired {
  104. s.Start()
  105. }
  106. return out, err
  107. }
  108. // Restart 重启并等待端口释放,避免新旧实例争抢端口。
  109. func (s *Supervisor) Restart() {
  110. s.log.Printf("重启 dsh web")
  111. s.Stop()
  112. s.waitPortFree(5 * time.Second)
  113. s.Start()
  114. }
  115. func (s *Supervisor) Status() Status {
  116. s.mu.Lock()
  117. defer s.mu.Unlock()
  118. url := s.baseURL()
  119. if s.webURL != "" {
  120. url = s.webURL // 优先使用带 token 的地址
  121. }
  122. var uptime time.Duration
  123. if s.running && !s.startedAt.IsZero() {
  124. uptime = time.Since(s.startedAt)
  125. }
  126. return Status{
  127. Running: s.running,
  128. PID: s.pid,
  129. Restarts: s.restarts,
  130. Uptime: uptime,
  131. LastError: s.lastError,
  132. URL: url,
  133. Updating: s.updating,
  134. External: s.external,
  135. }
  136. }
  137. func (s *Supervisor) baseURL() string {
  138. host := s.cfg.WebHost
  139. if host == "" {
  140. host = "127.0.0.1"
  141. }
  142. return fmt.Sprintf("http://%s:%d", host, s.cfg.WebPort)
  143. }
  144. func (s *Supervisor) setError(err error) {
  145. s.mu.Lock()
  146. s.lastError = err.Error()
  147. s.mu.Unlock()
  148. s.log.Printf("守护错误: %v", err)
  149. }
  150. // maxBackoff 重启退避上限。
  151. const maxBackoff = 2 * time.Minute
  152. // backoff 指数退避:base * 2^failures,封顶 maxBackoff。
  153. // 循环内提前 break,既保证封顶语义,也避免超大 failures 造成数值溢出。
  154. func (s *Supervisor) backoff() time.Duration {
  155. s.mu.Lock()
  156. failures := s.failures
  157. base := s.cfg.RestartDelaySec
  158. s.mu.Unlock()
  159. if base <= 0 {
  160. base = 5
  161. }
  162. d := time.Duration(base) * time.Second
  163. for i := 0; i < failures; i++ {
  164. if d >= maxBackoff {
  165. break
  166. }
  167. d *= 2
  168. }
  169. if d > maxBackoff {
  170. d = maxBackoff
  171. }
  172. return d
  173. }
  174. // portBusyBackoff 端口占用时的温和退避(上限 30 秒):不是故障,只是等待外部实例释放。
  175. func (s *Supervisor) portBusyBackoff() time.Duration {
  176. d := s.backoff()
  177. if d > 30*time.Second {
  178. d = 30 * time.Second
  179. }
  180. return d
  181. }
  182. // monitor 守护循环:端口预检 -> 启动 -> 等待退出 -> 期望运行时退避重启。
  183. func (s *Supervisor) monitor() {
  184. for {
  185. if !s.isDesired() {
  186. return
  187. }
  188. // 端口预检:已有实例监听时不再启动,避免端口冲突导致的重启风暴
  189. if isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  190. s.mu.Lock()
  191. s.running = false
  192. s.pid = 0
  193. s.external = true
  194. s.lastError = "端口已被占用(可能已有 dsh web 在运行)"
  195. s.mu.Unlock()
  196. s.log.Printf("端口 %s 已被占用,跳过启动(等待释放)", s.baseURL())
  197. s.bumpFailure()
  198. s.sleepInterruptible(s.portBusyBackoff())
  199. continue
  200. }
  201. s.mu.Lock()
  202. s.external = false
  203. s.mu.Unlock()
  204. exe, args, err := s.resolveCommand()
  205. if err != nil {
  206. s.setError(err)
  207. s.bumpFailure()
  208. s.sleepInterruptible(s.backoff())
  209. continue
  210. }
  211. cmd := exec.Command(exe, args...)
  212. hideWindow(cmd)
  213. stdout, errOut := cmd.StdoutPipe()
  214. stderr, errErr := cmd.StderrPipe()
  215. if errOut != nil || errErr != nil {
  216. s.setError(errors.New("无法创建输出管道"))
  217. s.bumpFailure()
  218. s.sleepInterruptible(s.backoff())
  219. continue
  220. }
  221. if err := cmd.Start(); err != nil {
  222. s.setError(err)
  223. s.bumpFailure()
  224. s.sleepInterruptible(s.backoff())
  225. continue
  226. }
  227. pid := cmd.Process.Pid
  228. startedAt := time.Now()
  229. s.mu.Lock()
  230. s.pid = pid
  231. s.running = true
  232. s.startedAt = startedAt
  233. s.lastError = ""
  234. s.webURL = ""
  235. desiredNow := s.desired
  236. s.mu.Unlock()
  237. s.log.Printf("dsh web 已启动 pid=%d: %s %s", pid, exe, strings.Join(args, " "))
  238. if !desiredNow {
  239. // 竞态窗口:停止请求发生在 cmd.Start 之后、pid 赋值之前。
  240. // 此时 Stop 读到的 pid 为 0 不会终止进程,这里补一次终止,避免残留。
  241. s.log.Printf("启动期间收到停止请求,立即结束 pid=%d", pid)
  242. _ = killTree(pid)
  243. }
  244. go s.pipeLog("out", stdout)
  245. go s.pipeLog("err", stderr)
  246. waitErr := cmd.Wait()
  247. ranFor := time.Since(startedAt)
  248. s.mu.Lock()
  249. s.running = false
  250. s.pid = 0
  251. stillDesired := s.desired
  252. if ranFor >= healthyUptime {
  253. s.failures = 0 // 稳定运行过,退避复位
  254. } else {
  255. s.failures++
  256. }
  257. if stillDesired {
  258. s.restarts++
  259. }
  260. s.mu.Unlock()
  261. s.log.Printf("dsh web 退出(err=%v,运行 %s),期望运行=%v", waitErr, ranFor.Round(time.Second), stillDesired)
  262. if !stillDesired {
  263. return
  264. }
  265. delay := s.backoff()
  266. s.log.Printf("%s 后自动重启", delay)
  267. s.sleepInterruptible(delay)
  268. }
  269. }
  270. func (s *Supervisor) isDesired() bool {
  271. s.mu.Lock()
  272. defer s.mu.Unlock()
  273. return s.desired
  274. }
  275. // bumpFailure 增加失败计数(触发退避放大)。
  276. func (s *Supervisor) bumpFailure() {
  277. s.mu.Lock()
  278. s.failures++
  279. s.mu.Unlock()
  280. }
  281. func (s *Supervisor) sleepInterruptible(d time.Duration) {
  282. deadline := time.Now().Add(d)
  283. for time.Now().Before(deadline) {
  284. if !s.isDesired() {
  285. return
  286. }
  287. time.Sleep(200 * time.Millisecond)
  288. }
  289. }
  290. func (s *Supervisor) waitStopped(pid int, timeout time.Duration) {
  291. deadline := time.Now().Add(timeout)
  292. for time.Now().Before(deadline) {
  293. if !processAlive(pid) {
  294. return
  295. }
  296. time.Sleep(100 * time.Millisecond)
  297. }
  298. s.log.Printf("等待 pid=%d 退出超时(继续)", pid)
  299. }
  300. func (s *Supervisor) waitPortFree(timeout time.Duration) {
  301. deadline := time.Now().Add(timeout)
  302. for time.Now().Before(deadline) {
  303. if !isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  304. return
  305. }
  306. time.Sleep(150 * time.Millisecond)
  307. }
  308. }
  309. // pipeLog 把子进程输出写入日志(token 由 Logger 统一脱敏),并捕获访问地址。
  310. func (s *Supervisor) pipeLog(tag string, r io.ReadCloser) {
  311. if r == nil {
  312. return
  313. }
  314. sc := bufio.NewScanner(r)
  315. sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
  316. for sc.Scan() {
  317. line := sc.Text()
  318. s.log.Printf("[dsh %s] %s", tag, line)
  319. if url := extractHTTPURL(line); url != "" {
  320. s.mu.Lock()
  321. if url != s.webURL {
  322. s.webURL = url
  323. s.log.Printf("捕获 Web 访问地址(含 token,仅存内存)")
  324. }
  325. s.mu.Unlock()
  326. }
  327. }
  328. }
  329. // resolveCommand 解析启动命令:优先 node + dsh 入口 JS(无 shell 包装,避免弹窗与注入面)。
  330. func (s *Supervisor) resolveCommand() (string, []string, error) {
  331. node := s.cfg.NodePath
  332. if node == "" {
  333. node = s.nodePath()
  334. }
  335. binJS := s.cfg.DshBinJS
  336. if binJS == "" {
  337. binJS = s.dshBinJS()
  338. }
  339. port := strconv.Itoa(s.cfg.WebPort)
  340. if node != "" && binJS != "" {
  341. return node, []string{binJS, "web", "--no-open", "--host", s.cfg.WebHost, "--port", port}, nil
  342. }
  343. if p, err := exec.LookPath("dsh.cmd"); err == nil {
  344. return "cmd.exe", []string{"/c", p, "web", "--no-open", "--port", port}, nil
  345. }
  346. if p, err := exec.LookPath("dsh"); err == nil {
  347. return p, []string{"web", "--no-open", "--port", port}, nil
  348. }
  349. return "", nil, errors.New("未找到 dsh 入口:请确认已全局安装 @deepseek-ai/dsh,或在配置中指定 nodePath / dshBinJs")
  350. }