queue.go 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117
  1. // Package faultRecordQueue 故障录波 MQTT 操作的异步执行器(asynq)
  2. package faultRecordQueue
  3. import (
  4. "context"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "time"
  9. mongoRepo "fwh-toolbox-server/repositories/mongo"
  10. task_core "fwh-toolbox-server/task/core"
  11. "github.com/hibiken/asynq"
  12. "go.mongodb.org/mongo-driver/bson/primitive"
  13. "go.uber.org/zap"
  14. )
  15. const (
  16. TaskTypeRun = "fault-record:run"
  17. QueueName = "fault-record"
  18. )
  19. var client *asynq.Client
  20. type runPayload struct {
  21. OperationID string `json:"operationId"`
  22. }
  23. func Init(tc *task_core.TaskCore) {
  24. mux := asynq.NewServeMux()
  25. mux.HandleFunc(TaskTypeRun, handleRun)
  26. client = tc.CreateTask(task_core.TaskConfig{
  27. Name: "FaultRecord",
  28. SvrCfg: asynq.Config{
  29. Concurrency: 10, // 不同网关的操作可并行;同一网关由网关锁串行
  30. Queues: map[string]int{QueueName: 1},
  31. ErrorHandler: asynq.ErrorHandlerFunc(func(_ context.Context, task *asynq.Task, err error) {
  32. zap.L().Error("fault-record task failed", zap.String("type", task.Type()), zap.String("err", err.Error()))
  33. }),
  34. },
  35. Mux: mux,
  36. })
  37. }
  38. // Enqueue 提交操作。TaskID 固定为操作 ID,重复提交(如启动恢复)不会重复执行。
  39. // 业务失败由执行器写入操作状态,不依赖 asynq 重试;MaxRetry 只用于进程崩溃后重新投递。
  40. func Enqueue(id primitive.ObjectID) error {
  41. if client == nil {
  42. return fmt.Errorf("fault record queue not initialized")
  43. }
  44. payload, _ := json.Marshal(runPayload{OperationID: id.Hex()})
  45. _, err := client.Enqueue(asynq.NewTask(TaskTypeRun, payload),
  46. asynq.Queue(QueueName),
  47. asynq.MaxRetry(3),
  48. asynq.Timeout(15*time.Minute),
  49. asynq.TaskID("fault-record-"+id.Hex()),
  50. asynq.Retention(24*time.Hour),
  51. )
  52. if errors.Is(err, asynq.ErrTaskIDConflict) {
  53. return nil
  54. }
  55. return err
  56. }
  57. // EnqueueDownloadRetry 提交手动重试下载;每次重试使用新的 TaskID(原任务的 TaskID 仍在保留期内)
  58. func EnqueueDownloadRetry(id primitive.ObjectID, attempt int) error {
  59. if client == nil {
  60. return fmt.Errorf("fault record queue not initialized")
  61. }
  62. payload, _ := json.Marshal(runPayload{OperationID: id.Hex()})
  63. _, err := client.Enqueue(asynq.NewTask(TaskTypeRun, payload),
  64. asynq.Queue(QueueName),
  65. asynq.MaxRetry(3),
  66. asynq.Timeout(15*time.Minute),
  67. asynq.TaskID(fmt.Sprintf("fault-record-%s-retry-%d", id.Hex(), attempt)),
  68. asynq.Retention(24*time.Hour),
  69. )
  70. if errors.Is(err, asynq.ErrTaskIDConflict) {
  71. return nil
  72. }
  73. return err
  74. }
  75. func handleRun(ctx context.Context, t *asynq.Task) error {
  76. var p runPayload
  77. if err := json.Unmarshal(t.Payload(), &p); err != nil {
  78. return fmt.Errorf("%w: %v", asynq.SkipRetry, err)
  79. }
  80. id, err := primitive.ObjectIDFromHex(p.OperationID)
  81. if err != nil {
  82. return fmt.Errorf("%w: %v", asynq.SkipRetry, err)
  83. }
  84. return defaultDeps().run(ctx, id)
  85. }
  86. // RecoverActive 启动时把未结束的操作重新提交;仍在队列中的会因 TaskID 冲突被忽略
  87. func RecoverActive(ctx context.Context) {
  88. list, err := mongoRepo.ListActiveFaultRecordOps(ctx)
  89. if err != nil {
  90. zap.L().Error("fault record recover: list active failed", zap.Error(err))
  91. return
  92. }
  93. for _, op := range list {
  94. enqueue := func() error { return Enqueue(op.ID) }
  95. if op.DownloadRetryCount > 0 {
  96. // 手动重试下载使用独立的 TaskID,按当前重试序号重新提交
  97. enqueue = func() error { return EnqueueDownloadRetry(op.ID, op.DownloadRetryCount) }
  98. }
  99. if err := enqueue(); err != nil {
  100. zap.L().Error("fault record recover: enqueue failed", zap.String("id", op.ID.Hex()), zap.Error(err))
  101. }
  102. }
  103. if len(list) > 0 {
  104. zap.L().Info("fault record recover: resubmitted active operations", zap.Int("count", len(list)))
  105. }
  106. }