fwhTaskQueue.go 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. package fwhQueue
  2. import (
  3. "context"
  4. "fmt"
  5. "time"
  6. "fwh-toolbox-server/dto"
  7. task_core "fwh-toolbox-server/task/core"
  8. "github.com/hibiken/asynq"
  9. "go.uber.org/zap"
  10. )
  11. // 任务类型常量
  12. const (
  13. TaskTypeBatchDownload = "fwh:BatchDownload"
  14. TaskTypeCallFile = "fwh:CallFile"
  15. TaskTypeSiteAnalysis = "fwh:SiteAnalysis"
  16. TaskTypeShutdownStats = "fwh:ShutdownStatistics"
  17. TaskTypeSiteInfoUpdate = "fwh:SiteInfoUpdate"
  18. )
  19. // 队列名称
  20. const (
  21. QueueMain = "fwh:main" // 主任务队列(顺序执行)
  22. QueueDownload = "fwh:download" // 文件下载队列(可并发)
  23. )
  24. // 模块级变量
  25. var (
  26. TaskClient *asynq.Client
  27. inspector *asynq.Inspector
  28. )
  29. // InitEnqueueClient 只创建入队客户端,不启动 worker、不注册处理函数。
  30. // 供 fwh-agent-mcp 等独立进程把任务投递给主服务执行;主服务使用 InitFwhTaskQueue。
  31. func InitEnqueueClient(redisOpt asynq.RedisClientOpt) {
  32. TaskClient = asynq.NewClient(redisOpt)
  33. }
  34. // InitFwhTaskQueue 初始化新的 FWH 任务队列
  35. // 通过 TaskCore 创建 Asynq Server(worker)、Client(入队)、Inspector(状态查询)
  36. func InitFwhTaskQueue(tc *task_core.TaskCore) {
  37. // 获取 Inspector(用于主任务轮询子任务状态)
  38. inspector = tc.Inspector.GetInspector()
  39. // 注册处理函数
  40. mux := asynq.NewServeMux()
  41. mux.HandleFunc(TaskTypeBatchDownload, handleBatchDownload)
  42. mux.HandleFunc(TaskTypeCallFile, handleCallFile)
  43. mux.HandleFunc(TaskTypeSiteAnalysis, handleSiteAnalysis)
  44. mux.HandleFunc(TaskTypeShutdownStats, handleShutdownStatistics)
  45. mux.HandleFunc(TaskTypeSiteInfoUpdate, handleSiteInfoUpdate)
  46. // 通过 TaskCore 创建任务 Server 和 Client
  47. TaskClient = tc.CreateTask(task_core.TaskConfig{
  48. Name: "FwhTask",
  49. SvrCfg: asynq.Config{
  50. Concurrency: 6, // 总并发数
  51. Queues: map[string]int{
  52. QueueDownload: 4, // 文件下载队列:4 个并发 worker
  53. QueueMain: 2, // 主任务队列:2 个并发 worker
  54. },
  55. // 错误处理
  56. ErrorHandler: asynq.ErrorHandlerFunc(func(ctx context.Context, task *asynq.Task, err error) {
  57. zap.L().Error("[fwhQueue] task failed",
  58. zap.String("type", task.Type()),
  59. zap.String("err", err.Error()),
  60. )
  61. }),
  62. // 重试延迟策略(指数退避)
  63. RetryDelayFunc: func(n int, e error, t *asynq.Task) time.Duration {
  64. return time.Duration(n*n) * 30 * time.Second // 30s, 2min, 4.5min
  65. },
  66. },
  67. Mux: mux,
  68. })
  69. zap.L().Info("[fwhQueue] initialized successfully")
  70. }
  71. // GetClient 获取 Asynq Client(供内部使用)
  72. func GetClient() *asynq.Client {
  73. return TaskClient
  74. }
  75. // GetInspector 获取 Asynq Inspector(供内部使用)
  76. func GetInspector() *asynq.Inspector {
  77. return inspector
  78. }
  79. // GetTaskInfoList 获取所有 FWH 队列中的任务列表(供外部查询)
  80. func GetTaskInfoList() ([]dto.BackgroundTaskItem, error) {
  81. if inspector == nil {
  82. return nil, fmt.Errorf("inspector not initialized")
  83. }
  84. var result []dto.BackgroundTaskItem
  85. queues := []string{QueueMain, QueueDownload}
  86. for _, queue := range queues {
  87. // 查询活跃任务(running)
  88. activeTasks, err := inspector.ListActiveTasks(queue, asynq.PageSize(100))
  89. if err == nil {
  90. for _, t := range activeTasks {
  91. result = append(result, dto.BackgroundTaskItem{
  92. Id: t.ID,
  93. Status: "running",
  94. Retry: int64(t.Retried),
  95. Payload: string(t.Payload),
  96. })
  97. }
  98. }
  99. // 查询等待中的任务(pending)
  100. pendingTasks, err := inspector.ListPendingTasks(queue, asynq.PageSize(100))
  101. if err == nil {
  102. for _, t := range pendingTasks {
  103. result = append(result, dto.BackgroundTaskItem{
  104. Id: t.ID,
  105. Status: "pending",
  106. Retry: int64(t.Retried),
  107. Payload: string(t.Payload),
  108. })
  109. }
  110. }
  111. // 查询重试中的任务(retry)
  112. retryTasks, err := inspector.ListRetryTasks(queue, asynq.PageSize(100))
  113. if err == nil {
  114. for _, t := range retryTasks {
  115. result = append(result, dto.BackgroundTaskItem{
  116. Id: t.ID,
  117. Status: "retry",
  118. Retry: int64(t.Retried),
  119. Payload: string(t.Payload),
  120. })
  121. }
  122. }
  123. // 查询已完成的任务(completed)
  124. completedTasks, err := inspector.ListCompletedTasks(queue, asynq.PageSize(100))
  125. if err == nil {
  126. for _, t := range completedTasks {
  127. result = append(result, dto.BackgroundTaskItem{
  128. Id: t.ID,
  129. Status: "completed",
  130. Retry: int64(t.Retried),
  131. Payload: string(t.Payload),
  132. })
  133. }
  134. }
  135. // 查询失败的任务(archived/failed)
  136. archivedTasks, err := inspector.ListArchivedTasks(queue, asynq.PageSize(100))
  137. if err == nil {
  138. for _, t := range archivedTasks {
  139. result = append(result, dto.BackgroundTaskItem{
  140. Id: t.ID,
  141. Status: "failed",
  142. Retry: int64(t.Retried),
  143. Payload: string(t.Payload),
  144. })
  145. }
  146. }
  147. }
  148. return result, nil
  149. }
  150. // newTaskID 生成固定格式的 TaskID,方便追踪
  151. func newTaskID(taskType string, id string) string {
  152. return fmt.Sprintf("%s:%s", taskType, id)
  153. }