queue.go 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. // Package faultRecordQueue 故障录波 MQTT 操作的异步执行器(asynq)
  2. package faultRecordQueue
  3. import (
  4. "context"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "time"
  9. mongoRepo "fwh-toolbox-server/repositories/mongo"
  10. task_core "fwh-toolbox-server/task/core"
  11. "github.com/hibiken/asynq"
  12. "go.mongodb.org/mongo-driver/bson/primitive"
  13. "go.uber.org/zap"
  14. )
  15. const (
  16. TaskTypeRun = "fault-record:run"
  17. QueueName = "fault-record"
  18. )
  19. var client *asynq.Client
  20. type runPayload struct {
  21. OperationID string `json:"operationId"`
  22. }
  23. func Init(tc *task_core.TaskCore) {
  24. mux := asynq.NewServeMux()
  25. mux.HandleFunc(TaskTypeRun, handleRun)
  26. client = tc.CreateTask(task_core.TaskConfig{
  27. Name: "FaultRecord",
  28. SvrCfg: asynq.Config{
  29. Concurrency: 10, // 不同网关的操作可并行;同一网关由网关锁串行
  30. Queues: map[string]int{QueueName: 1},
  31. ErrorHandler: asynq.ErrorHandlerFunc(func(_ context.Context, task *asynq.Task, err error) {
  32. zap.L().Error("fault-record task failed", zap.String("type", task.Type()), zap.String("err", err.Error()))
  33. }),
  34. },
  35. Mux: mux,
  36. })
  37. }
  38. // InitEnqueueClient 只创建入队客户端,不启动执行器。供 fwh-agent-mcp 等独立进程提交操作,由主服务执行
  39. func InitEnqueueClient(redisOpt asynq.RedisClientOpt) {
  40. client = asynq.NewClient(redisOpt)
  41. }
  42. // Enqueue 提交操作。TaskID 固定为操作 ID,重复提交(如启动恢复)不会重复执行。
  43. // 业务失败由执行器写入操作状态,不依赖 asynq 重试;MaxRetry 只用于进程崩溃后重新投递。
  44. func Enqueue(id primitive.ObjectID) error {
  45. if client == nil {
  46. return fmt.Errorf("fault record queue not initialized")
  47. }
  48. payload, _ := json.Marshal(runPayload{OperationID: id.Hex()})
  49. _, err := client.Enqueue(asynq.NewTask(TaskTypeRun, payload),
  50. asynq.Queue(QueueName),
  51. asynq.MaxRetry(3),
  52. asynq.Timeout(15*time.Minute),
  53. asynq.TaskID("fault-record-"+id.Hex()),
  54. asynq.Retention(24*time.Hour),
  55. )
  56. if errors.Is(err, asynq.ErrTaskIDConflict) {
  57. return nil
  58. }
  59. return err
  60. }
  61. // EnqueueDownloadRetry 提交手动重试下载;每次重试使用新的 TaskID(原任务的 TaskID 仍在保留期内)
  62. func EnqueueDownloadRetry(id primitive.ObjectID, attempt int) error {
  63. if client == nil {
  64. return fmt.Errorf("fault record queue not initialized")
  65. }
  66. payload, _ := json.Marshal(runPayload{OperationID: id.Hex()})
  67. _, err := client.Enqueue(asynq.NewTask(TaskTypeRun, payload),
  68. asynq.Queue(QueueName),
  69. asynq.MaxRetry(3),
  70. asynq.Timeout(15*time.Minute),
  71. asynq.TaskID(fmt.Sprintf("fault-record-%s-retry-%d", id.Hex(), attempt)),
  72. asynq.Retention(24*time.Hour),
  73. )
  74. if errors.Is(err, asynq.ErrTaskIDConflict) {
  75. return nil
  76. }
  77. return err
  78. }
  79. func handleRun(ctx context.Context, t *asynq.Task) error {
  80. var p runPayload
  81. if err := json.Unmarshal(t.Payload(), &p); err != nil {
  82. return fmt.Errorf("%w: %v", asynq.SkipRetry, err)
  83. }
  84. id, err := primitive.ObjectIDFromHex(p.OperationID)
  85. if err != nil {
  86. return fmt.Errorf("%w: %v", asynq.SkipRetry, err)
  87. }
  88. return defaultDeps().run(ctx, id)
  89. }
  90. // RecoverActive 启动时把未结束的操作重新提交;仍在队列中的会因 TaskID 冲突被忽略
  91. func RecoverActive(ctx context.Context) {
  92. list, err := mongoRepo.ListActiveFaultRecordOps(ctx)
  93. if err != nil {
  94. zap.L().Error("fault record recover: list active failed", zap.Error(err))
  95. return
  96. }
  97. for _, op := range list {
  98. enqueue := func() error { return Enqueue(op.ID) }
  99. if op.DownloadRetryCount > 0 {
  100. // 手动重试下载使用独立的 TaskID,按当前重试序号重新提交
  101. enqueue = func() error { return EnqueueDownloadRetry(op.ID, op.DownloadRetryCount) }
  102. }
  103. if err := enqueue(); err != nil {
  104. zap.L().Error("fault record recover: enqueue failed", zap.String("id", op.ID.Hex()), zap.Error(err))
  105. }
  106. }
  107. if len(list) > 0 {
  108. zap.L().Info("fault record recover: resubmitted active operations", zap.Int("count", len(list)))
  109. }
  110. }