fwhTaskQueue.go 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176
  1. package fwhQueue
  2. import (
  3. "context"
  4. "fmt"
  5. "time"
  6. "fwh-toolbox-server/dto"
  7. task_core "fwh-toolbox-server/task/core"
  8. "github.com/hibiken/asynq"
  9. "go.uber.org/zap"
  10. )
  11. // 任务类型常量
  12. const (
  13. TaskTypeBatchDownload = "fwh:BatchDownload"
  14. TaskTypeCallFile = "fwh:CallFile"
  15. TaskTypeSiteAnalysis = "fwh:SiteAnalysis"
  16. TaskTypeShutdownStats = "fwh:ShutdownStatistics"
  17. TaskTypeSiteInfoUpdate = "fwh:SiteInfoUpdate"
  18. )
  19. // 队列名称
  20. const (
  21. QueueMain = "fwh:main" // 主任务队列(顺序执行)
  22. QueueDownload = "fwh:download" // 文件下载队列(可并发)
  23. )
  24. // 模块级变量
  25. var (
  26. TaskClient *asynq.Client
  27. inspector *asynq.Inspector
  28. )
  29. // InitEnqueueClient 只创建入队客户端与 Inspector(查询、取消),不启动 worker、不注册处理函数。
  30. // 供 fwh-agent-mcp 等独立进程把任务投递给主服务执行;主服务使用 InitFwhTaskQueue。
  31. func InitEnqueueClient(redisOpt asynq.RedisClientOpt) {
  32. TaskClient = asynq.NewClient(redisOpt)
  33. inspector = asynq.NewInspector(redisOpt)
  34. }
  35. // InitFwhTaskQueue 初始化新的 FWH 任务队列
  36. // 通过 TaskCore 创建 Asynq Server(worker)、Client(入队)、Inspector(状态查询)
  37. func InitFwhTaskQueue(tc *task_core.TaskCore) {
  38. // 获取 Inspector(用于主任务轮询子任务状态)
  39. inspector = tc.Inspector.GetInspector()
  40. // 注册处理函数
  41. mux := asynq.NewServeMux()
  42. mux.HandleFunc(TaskTypeBatchDownload, handleBatchDownload)
  43. mux.HandleFunc(TaskTypeCallFile, handleCallFile)
  44. mux.HandleFunc(TaskTypeSiteAnalysis, handleSiteAnalysis)
  45. mux.HandleFunc(TaskTypeShutdownStats, handleShutdownStatistics)
  46. mux.HandleFunc(TaskTypeSiteInfoUpdate, handleSiteInfoUpdate)
  47. // 通过 TaskCore 创建任务 Server 和 Client
  48. TaskClient = tc.CreateTask(task_core.TaskConfig{
  49. Name: "FwhTask",
  50. SvrCfg: asynq.Config{
  51. Concurrency: 6, // 总并发数
  52. Queues: map[string]int{
  53. QueueDownload: 4, // 文件下载队列:4 个并发 worker
  54. QueueMain: 2, // 主任务队列:2 个并发 worker
  55. },
  56. // 错误处理
  57. ErrorHandler: asynq.ErrorHandlerFunc(func(ctx context.Context, task *asynq.Task, err error) {
  58. zap.L().Error("[fwhQueue] task failed",
  59. zap.String("type", task.Type()),
  60. zap.String("err", err.Error()),
  61. )
  62. }),
  63. // 重试延迟策略(指数退避)
  64. RetryDelayFunc: func(n int, e error, t *asynq.Task) time.Duration {
  65. return time.Duration(n*n) * 30 * time.Second // 30s, 2min, 4.5min
  66. },
  67. },
  68. Mux: mux,
  69. })
  70. zap.L().Info("[fwhQueue] initialized successfully")
  71. }
  72. // GetClient 获取 Asynq Client(供内部使用)
  73. func GetClient() *asynq.Client {
  74. return TaskClient
  75. }
  76. // GetInspector 获取 Asynq Inspector(供内部使用)
  77. func GetInspector() *asynq.Inspector {
  78. return inspector
  79. }
  80. // GetTaskInfoList 获取所有 FWH 队列中的任务列表(供外部查询)
  81. func GetTaskInfoList() ([]dto.BackgroundTaskItem, error) {
  82. if inspector == nil {
  83. return nil, fmt.Errorf("inspector not initialized")
  84. }
  85. var result []dto.BackgroundTaskItem
  86. queues := []string{QueueMain, QueueDownload}
  87. for _, queue := range queues {
  88. // 查询活跃任务(running)
  89. activeTasks, err := inspector.ListActiveTasks(queue, asynq.PageSize(100))
  90. if err == nil {
  91. for _, t := range activeTasks {
  92. result = append(result, dto.BackgroundTaskItem{
  93. Id: t.ID,
  94. Status: "running",
  95. Retry: int64(t.Retried),
  96. Payload: string(t.Payload),
  97. })
  98. }
  99. }
  100. // 查询等待中的任务(pending)
  101. pendingTasks, err := inspector.ListPendingTasks(queue, asynq.PageSize(100))
  102. if err == nil {
  103. for _, t := range pendingTasks {
  104. result = append(result, dto.BackgroundTaskItem{
  105. Id: t.ID,
  106. Status: "pending",
  107. Retry: int64(t.Retried),
  108. Payload: string(t.Payload),
  109. })
  110. }
  111. }
  112. // 查询重试中的任务(retry)
  113. retryTasks, err := inspector.ListRetryTasks(queue, asynq.PageSize(100))
  114. if err == nil {
  115. for _, t := range retryTasks {
  116. result = append(result, dto.BackgroundTaskItem{
  117. Id: t.ID,
  118. Status: "retry",
  119. Retry: int64(t.Retried),
  120. Payload: string(t.Payload),
  121. })
  122. }
  123. }
  124. // 查询已完成的任务(completed)
  125. completedTasks, err := inspector.ListCompletedTasks(queue, asynq.PageSize(100))
  126. if err == nil {
  127. for _, t := range completedTasks {
  128. result = append(result, dto.BackgroundTaskItem{
  129. Id: t.ID,
  130. Status: "completed",
  131. Retry: int64(t.Retried),
  132. Payload: string(t.Payload),
  133. })
  134. }
  135. }
  136. // 查询失败的任务(archived/failed)
  137. archivedTasks, err := inspector.ListArchivedTasks(queue, asynq.PageSize(100))
  138. if err == nil {
  139. for _, t := range archivedTasks {
  140. result = append(result, dto.BackgroundTaskItem{
  141. Id: t.ID,
  142. Status: "failed",
  143. Retry: int64(t.Retried),
  144. Payload: string(t.Payload),
  145. })
  146. }
  147. }
  148. }
  149. return result, nil
  150. }
  151. // newTaskID 生成固定格式的 TaskID,方便追踪
  152. func newTaskID(taskType string, id string) string {
  153. return fmt.Sprintf("%s:%s", taskType, id)
  154. }