fwhTaskQueue.go 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169
  1. package fwhQueue
  2. import (
  3. "context"
  4. "fmt"
  5. "time"
  6. "fwh-toolbox-server/dto"
  7. task_core "fwh-toolbox-server/task/core"
  8. "github.com/hibiken/asynq"
  9. "go.uber.org/zap"
  10. )
  11. // 任务类型常量
  12. const (
  13. TaskTypeBatchDownload = "fwh:BatchDownload"
  14. TaskTypeCallFile = "fwh:CallFile"
  15. TaskTypeSiteAnalysis = "fwh:SiteAnalysis"
  16. TaskTypeShutdownStats = "fwh:ShutdownStatistics"
  17. TaskTypeSiteInfoUpdate = "fwh:SiteInfoUpdate"
  18. )
  19. // 队列名称
  20. const (
  21. QueueMain = "fwh:main" // 主任务队列(顺序执行)
  22. QueueDownload = "fwh:download" // 文件下载队列(可并发)
  23. )
  24. // 模块级变量
  25. var (
  26. TaskClient *asynq.Client
  27. inspector *asynq.Inspector
  28. )
  29. // InitFwhTaskQueue 初始化新的 FWH 任务队列
  30. // 通过 TaskCore 创建 Asynq Server(worker)、Client(入队)、Inspector(状态查询)
  31. func InitFwhTaskQueue(tc *task_core.TaskCore) {
  32. // 获取 Inspector(用于主任务轮询子任务状态)
  33. inspector = tc.Inspector.GetInspector()
  34. // 注册处理函数
  35. mux := asynq.NewServeMux()
  36. mux.HandleFunc(TaskTypeBatchDownload, handleBatchDownload)
  37. mux.HandleFunc(TaskTypeCallFile, handleCallFile)
  38. mux.HandleFunc(TaskTypeSiteAnalysis, handleSiteAnalysis)
  39. mux.HandleFunc(TaskTypeShutdownStats, handleShutdownStatistics)
  40. mux.HandleFunc(TaskTypeSiteInfoUpdate, handleSiteInfoUpdate)
  41. // 通过 TaskCore 创建任务 Server 和 Client
  42. TaskClient = tc.CreateTask(task_core.TaskConfig{
  43. Name: "FwhTask",
  44. SvrCfg: asynq.Config{
  45. Concurrency: 6, // 总并发数
  46. Queues: map[string]int{
  47. QueueDownload: 4, // 文件下载队列:4 个并发 worker
  48. QueueMain: 2, // 主任务队列:2 个并发 worker
  49. },
  50. // 错误处理
  51. ErrorHandler: asynq.ErrorHandlerFunc(func(ctx context.Context, task *asynq.Task, err error) {
  52. zap.L().Error("[fwhQueue] task failed",
  53. zap.String("type", task.Type()),
  54. zap.String("err", err.Error()),
  55. )
  56. }),
  57. // 重试延迟策略(指数退避)
  58. RetryDelayFunc: func(n int, e error, t *asynq.Task) time.Duration {
  59. return time.Duration(n*n) * 30 * time.Second // 30s, 2min, 4.5min
  60. },
  61. },
  62. Mux: mux,
  63. })
  64. zap.L().Info("[fwhQueue] initialized successfully")
  65. }
  66. // GetClient 获取 Asynq Client(供内部使用)
  67. func GetClient() *asynq.Client {
  68. return TaskClient
  69. }
  70. // GetInspector 获取 Asynq Inspector(供内部使用)
  71. func GetInspector() *asynq.Inspector {
  72. return inspector
  73. }
  74. // GetTaskInfoList 获取所有 FWH 队列中的任务列表(供外部查询)
  75. func GetTaskInfoList() ([]dto.BackgroundTaskItem, error) {
  76. if inspector == nil {
  77. return nil, fmt.Errorf("inspector not initialized")
  78. }
  79. var result []dto.BackgroundTaskItem
  80. queues := []string{QueueMain, QueueDownload}
  81. for _, queue := range queues {
  82. // 查询活跃任务(running)
  83. activeTasks, err := inspector.ListActiveTasks(queue, asynq.PageSize(100))
  84. if err == nil {
  85. for _, t := range activeTasks {
  86. result = append(result, dto.BackgroundTaskItem{
  87. Id: t.ID,
  88. Status: "running",
  89. Retry: int64(t.Retried),
  90. Payload: string(t.Payload),
  91. })
  92. }
  93. }
  94. // 查询等待中的任务(pending)
  95. pendingTasks, err := inspector.ListPendingTasks(queue, asynq.PageSize(100))
  96. if err == nil {
  97. for _, t := range pendingTasks {
  98. result = append(result, dto.BackgroundTaskItem{
  99. Id: t.ID,
  100. Status: "pending",
  101. Retry: int64(t.Retried),
  102. Payload: string(t.Payload),
  103. })
  104. }
  105. }
  106. // 查询重试中的任务(retry)
  107. retryTasks, err := inspector.ListRetryTasks(queue, asynq.PageSize(100))
  108. if err == nil {
  109. for _, t := range retryTasks {
  110. result = append(result, dto.BackgroundTaskItem{
  111. Id: t.ID,
  112. Status: "retry",
  113. Retry: int64(t.Retried),
  114. Payload: string(t.Payload),
  115. })
  116. }
  117. }
  118. // 查询已完成的任务(completed)
  119. completedTasks, err := inspector.ListCompletedTasks(queue, asynq.PageSize(100))
  120. if err == nil {
  121. for _, t := range completedTasks {
  122. result = append(result, dto.BackgroundTaskItem{
  123. Id: t.ID,
  124. Status: "completed",
  125. Retry: int64(t.Retried),
  126. Payload: string(t.Payload),
  127. })
  128. }
  129. }
  130. // 查询失败的任务(archived/failed)
  131. archivedTasks, err := inspector.ListArchivedTasks(queue, asynq.PageSize(100))
  132. if err == nil {
  133. for _, t := range archivedTasks {
  134. result = append(result, dto.BackgroundTaskItem{
  135. Id: t.ID,
  136. Status: "failed",
  137. Retry: int64(t.Retried),
  138. Payload: string(t.Payload),
  139. })
  140. }
  141. }
  142. }
  143. return result, nil
  144. }
  145. // newTaskID 生成固定格式的 TaskID,方便追踪
  146. func newTaskID(taskType string, id string) string {
  147. return fmt.Sprintf("%s:%s", taskType, id)
  148. }