supervisor.go 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373
  1. package main
  2. import (
  3. "bufio"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "os/exec"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "time"
  12. )
  13. // supervisor.go —— dsh web 守护核心:端口预检、隐藏启动、指数退避重启、状态快照。
  14. // healthyUptime 稳定运行判定阈值:超过它则清零失败计数(退避复位)。
  15. const healthyUptime = 30 * time.Second
  16. // Status 服务状态快照。
  17. type Status struct {
  18. Running bool
  19. Ready bool // 已捕获带 token 的访问地址,Web 界面可打开
  20. PID int
  21. Restarts int
  22. Uptime time.Duration
  23. LastError string
  24. URL string
  25. Updating bool
  26. External bool // 端口被外部实例占用(非本程序启动)
  27. }
  28. // Supervisor 守护 dsh web。
  29. type Supervisor struct {
  30. mu sync.Mutex
  31. probeMu sync.Mutex // 串行化入口探测(见 probe.go)
  32. cfg *Config
  33. log *Logger
  34. pid int
  35. running bool
  36. desired bool
  37. restarts int
  38. failures int
  39. startedAt time.Time
  40. lastError string
  41. updating bool
  42. external bool
  43. binCache string
  44. nodeCache string
  45. webURL string // dsh web 输出的完整 URL(含访问 token,仅存内存)
  46. autoMu sync.Mutex
  47. autoLoop bool // 定期更新循环是否已启动(进程生命周期内单例)
  48. }
  49. func NewSupervisor(cfg *Config, log *Logger) *Supervisor {
  50. return &Supervisor{cfg: cfg, log: log}
  51. }
  52. // Start 设置期望运行并拉起守护循环。
  53. func (s *Supervisor) Start() {
  54. s.mu.Lock()
  55. if s.desired {
  56. s.mu.Unlock()
  57. return
  58. }
  59. s.desired = true
  60. s.failures = 0
  61. s.mu.Unlock()
  62. go s.monitor()
  63. }
  64. // Stop 同步停止:结束进程树并等待其真正退出(最多 8 秒)。
  65. func (s *Supervisor) Stop() {
  66. s.mu.Lock()
  67. s.desired = false
  68. pid := s.pid
  69. s.webURL = ""
  70. s.mu.Unlock()
  71. if pid > 0 {
  72. s.log.Printf("停止 dsh web pid=%d", pid)
  73. _ = killTree(pid)
  74. s.waitStopped(pid, 8*time.Second)
  75. }
  76. }
  77. // SetAutoUpdate 设置定期更新开关(加锁 + 立即持久化;开启时确保循环已启动)。
  78. func (s *Supervisor) SetAutoUpdate(enabled bool) error {
  79. s.mu.Lock()
  80. s.cfg.AutoUpdate = enabled
  81. snapshot := *s.cfg
  82. s.mu.Unlock()
  83. if enabled {
  84. s.startAutoUpdate()
  85. }
  86. return saveConfig(&snapshot)
  87. }
  88. // SetAutoStart 设置开机自启开关(加锁 + 立即持久化;注册表由调用方处理)。
  89. func (s *Supervisor) SetAutoStart(enabled bool) error {
  90. s.mu.Lock()
  91. s.cfg.AutoStart = enabled
  92. snapshot := *s.cfg
  93. s.mu.Unlock()
  94. return saveConfig(&snapshot)
  95. }
  96. // Restart 重启并等待端口释放,避免新旧实例争抢端口。
  97. func (s *Supervisor) Restart() {
  98. s.log.Printf("重启 dsh web")
  99. s.Stop()
  100. s.waitPortFree(5 * time.Second)
  101. s.Start()
  102. }
  103. func (s *Supervisor) Status() Status {
  104. s.mu.Lock()
  105. defer s.mu.Unlock()
  106. url := s.baseURL()
  107. if s.webURL != "" {
  108. url = s.webURL // 优先使用带 token 的地址
  109. }
  110. var uptime time.Duration
  111. if s.running && !s.startedAt.IsZero() {
  112. uptime = time.Since(s.startedAt)
  113. }
  114. return Status{
  115. Running: s.running,
  116. Ready: s.running && s.webURL != "",
  117. PID: s.pid,
  118. Restarts: s.restarts,
  119. Uptime: uptime,
  120. LastError: s.lastError,
  121. URL: url,
  122. Updating: s.updating,
  123. External: s.external,
  124. }
  125. }
  126. func (s *Supervisor) baseURL() string {
  127. host := s.cfg.WebHost
  128. if host == "" {
  129. host = "127.0.0.1"
  130. }
  131. return fmt.Sprintf("http://%s:%d", host, s.cfg.WebPort)
  132. }
  133. func (s *Supervisor) setError(err error) {
  134. s.mu.Lock()
  135. s.lastError = err.Error()
  136. s.mu.Unlock()
  137. s.log.Printf("守护错误: %v", err)
  138. }
  139. // maxBackoff 重启退避上限。
  140. const maxBackoff = 2 * time.Minute
  141. // backoff 指数退避:base * 2^failures,封顶 maxBackoff。
  142. // 循环内提前 break,既保证封顶语义,也避免超大 failures 造成数值溢出。
  143. func (s *Supervisor) backoff() time.Duration {
  144. s.mu.Lock()
  145. failures := s.failures
  146. base := s.cfg.RestartDelaySec
  147. s.mu.Unlock()
  148. if base <= 0 {
  149. base = 5
  150. }
  151. d := time.Duration(base) * time.Second
  152. for i := 0; i < failures; i++ {
  153. if d >= maxBackoff {
  154. break
  155. }
  156. d *= 2
  157. }
  158. if d > maxBackoff {
  159. d = maxBackoff
  160. }
  161. return d
  162. }
  163. // portBusyBackoff 端口占用时的温和退避(上限 30 秒):不是故障,只是等待外部实例释放。
  164. func (s *Supervisor) portBusyBackoff() time.Duration {
  165. d := s.backoff()
  166. if d > 30*time.Second {
  167. d = 30 * time.Second
  168. }
  169. return d
  170. }
  171. // monitor 守护循环:端口预检 -> 启动 -> 等待退出 -> 期望运行时退避重启。
  172. func (s *Supervisor) monitor() {
  173. for {
  174. if !s.isDesired() {
  175. return
  176. }
  177. // 端口预检:已有实例监听时不再启动,避免端口冲突导致的重启风暴
  178. if isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  179. s.mu.Lock()
  180. s.running = false
  181. s.pid = 0
  182. s.external = true
  183. s.lastError = "端口已被占用(可能已有 dsh web 在运行)"
  184. s.mu.Unlock()
  185. s.log.Printf("端口 %s 已被占用,跳过启动(等待释放)", s.baseURL())
  186. s.bumpFailure()
  187. s.sleepInterruptible(s.portBusyBackoff())
  188. continue
  189. }
  190. s.mu.Lock()
  191. s.external = false
  192. s.mu.Unlock()
  193. exe, args, err := s.resolveCommand()
  194. if err != nil {
  195. s.setError(err)
  196. s.bumpFailure()
  197. s.sleepInterruptible(s.backoff())
  198. continue
  199. }
  200. cmd := exec.Command(exe, args...)
  201. hideWindow(cmd)
  202. stdout, errOut := cmd.StdoutPipe()
  203. stderr, errErr := cmd.StderrPipe()
  204. if errOut != nil || errErr != nil {
  205. s.setError(errors.New("无法创建输出管道"))
  206. s.bumpFailure()
  207. s.sleepInterruptible(s.backoff())
  208. continue
  209. }
  210. if err := cmd.Start(); err != nil {
  211. s.setError(err)
  212. s.bumpFailure()
  213. s.sleepInterruptible(s.backoff())
  214. continue
  215. }
  216. pid := cmd.Process.Pid
  217. startedAt := time.Now()
  218. s.mu.Lock()
  219. s.pid = pid
  220. s.running = true
  221. s.startedAt = startedAt
  222. s.lastError = ""
  223. s.webURL = ""
  224. desiredNow := s.desired
  225. s.mu.Unlock()
  226. s.log.Printf("dsh web 已启动 pid=%d: %s %s", pid, exe, strings.Join(args, " "))
  227. if !desiredNow {
  228. // 竞态窗口:停止请求发生在 cmd.Start 之后、pid 赋值之前。
  229. // 此时 Stop 读到的 pid 为 0 不会终止进程,这里补一次终止,避免残留。
  230. s.log.Printf("启动期间收到停止请求,立即结束 pid=%d", pid)
  231. _ = killTree(pid)
  232. }
  233. go s.pipeLog("out", stdout)
  234. go s.pipeLog("err", stderr)
  235. waitErr := cmd.Wait()
  236. ranFor := time.Since(startedAt)
  237. s.mu.Lock()
  238. s.running = false
  239. s.pid = 0
  240. stillDesired := s.desired
  241. if ranFor >= healthyUptime {
  242. s.failures = 0 // 稳定运行过,退避复位
  243. } else {
  244. s.failures++
  245. }
  246. if stillDesired {
  247. s.restarts++
  248. }
  249. s.mu.Unlock()
  250. s.log.Printf("dsh web 退出(err=%v,运行 %s),期望运行=%v", waitErr, ranFor.Round(time.Second), stillDesired)
  251. if !stillDesired {
  252. return
  253. }
  254. delay := s.backoff()
  255. s.log.Printf("%s 后自动重启", delay)
  256. s.sleepInterruptible(delay)
  257. }
  258. }
  259. func (s *Supervisor) isDesired() bool {
  260. s.mu.Lock()
  261. defer s.mu.Unlock()
  262. return s.desired
  263. }
  264. // bumpFailure 增加失败计数(触发退避放大)。
  265. func (s *Supervisor) bumpFailure() {
  266. s.mu.Lock()
  267. s.failures++
  268. s.mu.Unlock()
  269. }
  270. func (s *Supervisor) sleepInterruptible(d time.Duration) {
  271. deadline := time.Now().Add(d)
  272. for time.Now().Before(deadline) {
  273. if !s.isDesired() {
  274. return
  275. }
  276. time.Sleep(200 * time.Millisecond)
  277. }
  278. }
  279. func (s *Supervisor) waitStopped(pid int, timeout time.Duration) {
  280. deadline := time.Now().Add(timeout)
  281. for time.Now().Before(deadline) {
  282. if !processAlive(pid) {
  283. return
  284. }
  285. time.Sleep(100 * time.Millisecond)
  286. }
  287. s.log.Printf("等待 pid=%d 退出超时(继续)", pid)
  288. }
  289. func (s *Supervisor) waitPortFree(timeout time.Duration) {
  290. deadline := time.Now().Add(timeout)
  291. for time.Now().Before(deadline) {
  292. if !isPortOpen(s.cfg.WebHost, s.cfg.WebPort) {
  293. return
  294. }
  295. time.Sleep(150 * time.Millisecond)
  296. }
  297. }
  298. // pipeLog 把子进程输出写入日志(token 由 Logger 统一脱敏),并捕获访问地址。
  299. func (s *Supervisor) pipeLog(tag string, r io.ReadCloser) {
  300. if r == nil {
  301. return
  302. }
  303. sc := bufio.NewScanner(r)
  304. sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
  305. for sc.Scan() {
  306. line := sc.Text()
  307. s.log.Printf("[dsh %s] %s", tag, line)
  308. if url := extractHTTPURL(line); url != "" {
  309. s.mu.Lock()
  310. if url != s.webURL {
  311. s.webURL = url
  312. s.log.Printf("捕获 Web 访问地址(含 token,仅存内存);启动耗时 %s", time.Since(s.startedAt).Round(time.Second))
  313. }
  314. s.mu.Unlock()
  315. }
  316. }
  317. }
  318. // resolveCommand 解析启动命令:优先 node + dsh 入口 JS(无 shell 包装,避免弹窗与注入面)。
  319. func (s *Supervisor) resolveCommand() (string, []string, error) {
  320. node := s.cfg.NodePath
  321. if node == "" {
  322. node = s.nodePath()
  323. }
  324. binJS := s.cfg.DshBinJS
  325. if binJS == "" {
  326. binJS = s.dshBinJS()
  327. }
  328. port := strconv.Itoa(s.cfg.WebPort)
  329. if node != "" && binJS != "" {
  330. return node, []string{binJS, "web", "--no-open", "--host", s.cfg.WebHost, "--port", port}, nil
  331. }
  332. if p, err := exec.LookPath("dsh.cmd"); err == nil {
  333. return "cmd.exe", []string{"/c", p, "web", "--no-open", "--port", port}, nil
  334. }
  335. if p, err := exec.LookPath("dsh"); err == nil {
  336. return p, []string{"web", "--no-open", "--port", port}, nil
  337. }
  338. return "", nil, errors.New("未找到 dsh 入口:请确认已全局安装 @deepseek-ai/dsh,或在配置中指定 nodePath / dshBinJs")
  339. }