batchDownloadRepo.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. package mongo
  2. import (
  3. "context"
  4. "fwh-toolbox-server/database"
  5. "fwh-toolbox-server/domain"
  6. "fwh-toolbox-server/dto"
  7. "time"
  8. "go.mongodb.org/mongo-driver/bson"
  9. "go.mongodb.org/mongo-driver/bson/primitive"
  10. "go.mongodb.org/mongo-driver/mongo"
  11. "go.mongodb.org/mongo-driver/mongo/options"
  12. )
  13. const (
  14. CollectionName_BatchDownloadConfig = "BatchDownloadConfig"
  15. )
  16. // 从数据库获取批量下载数据列表
  17. func GetBatchDownloadManageList(page, size uint, sn, progress *string) (*dto.ResPage[domain.BatchDownloadInfo], error) {
  18. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  19. defer cancel()
  20. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  21. filter := bson.M{}
  22. // 条件
  23. if sn != nil {
  24. filter["sn"] = *sn
  25. }
  26. if progress != nil {
  27. filter["progress"] = *progress
  28. }
  29. findOptions := options.Find().
  30. SetProjection(bson.M{
  31. "_id": 1,
  32. "sn": 1,
  33. "description": 1,
  34. "timePeriod": 1,
  35. "options": 1,
  36. "progress": 1,
  37. "failReasons": 1,
  38. "items": 1,
  39. "totalCount": 1,
  40. "doneCount": 1,
  41. }).
  42. SetSkip(int64((page - 1) * size)).
  43. SetLimit(int64(size)).
  44. SetSort(bson.D{
  45. {Key: "_id", Value: -1}, // 倒序
  46. })
  47. cursor, err := collection.Find(ctx, filter, findOptions)
  48. if err != nil {
  49. return nil, err
  50. }
  51. defer cursor.Close(ctx)
  52. var configList []domain.BatchDownloadInfo
  53. for cursor.Next(ctx) {
  54. var cfg domain.BatchDownloadInfo
  55. if err := cursor.Decode(&cfg); err != nil {
  56. return nil, err
  57. }
  58. configList = append(configList, cfg)
  59. }
  60. // 查询总数
  61. total, err := collection.CountDocuments(ctx, filter)
  62. if err != nil {
  63. return nil, err
  64. }
  65. res := dto.ResPage[domain.BatchDownloadInfo]{
  66. Data: configList,
  67. PageSize: size,
  68. PageNum: page,
  69. Total: uint(total),
  70. }
  71. return &res, nil
  72. }
  73. // 从数据库获取一条批量下载数据配置
  74. func GetBatchDownloadConfig(id primitive.ObjectID) (*domain.BatchDownloadConfig, error) {
  75. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  76. defer cancel()
  77. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  78. var cfg domain.BatchDownloadConfig
  79. err := collection.FindOne(ctx, bson.M{"_id": id}).Decode(&cfg)
  80. if err != nil {
  81. return nil, err
  82. }
  83. return &cfg, nil
  84. }
  85. // 插入一条批量下载配置到数据库
  86. func InsertBatchDownloadConfig(writeConfig domain.BatchDownloadConfigWrite) (primitive.ObjectID, error) {
  87. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  88. defer cancel()
  89. config := domain.BatchDownloadConfig{
  90. Sn: writeConfig.Sn,
  91. Description: writeConfig.Description,
  92. TimePeriod: writeConfig.TimePeriod,
  93. Options: writeConfig.Options,
  94. Progress: domain.ProgressPending,
  95. Data: map[string]domain.BatchDownloadData{},
  96. FailReasons: []string{},
  97. }
  98. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  99. res, err := collection.InsertOne(ctx, config)
  100. if err != nil {
  101. return primitive.ObjectID{}, err
  102. }
  103. return res.InsertedID.(primitive.ObjectID), nil
  104. }
  105. // 更新一条批量下载描述描述到数据库
  106. func UpdateBatchDownloadDescription(id primitive.ObjectID, desc string) error {
  107. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  108. defer cancel()
  109. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  110. // _, err := collection.InsertOne(ctx, config)
  111. filter := bson.M{"_id": id} // 按ID匹配
  112. update := bson.M{
  113. // 如果存在就更新全部字段
  114. "$set": bson.M{
  115. "description": desc,
  116. },
  117. }
  118. opts := options.Update().SetUpsert(true)
  119. _, err := collection.UpdateOne(ctx, filter, update, opts)
  120. if err != nil {
  121. return err
  122. }
  123. return nil
  124. }
  125. // 更新一条批量下载数据owner到数据库
  126. func UpdateBatchDownloadOwner(id primitive.ObjectID, owner []string) error {
  127. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  128. defer cancel()
  129. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  130. // _, err := collection.InsertOne(ctx, config)
  131. filter := bson.M{"_id": id} // 按ID匹配
  132. update := bson.M{
  133. // 如果存在就更新全部字段
  134. "$set": bson.M{
  135. "owner": owner,
  136. },
  137. }
  138. opts := options.Update().SetUpsert(true)
  139. _, err := collection.UpdateOne(ctx, filter, update, opts)
  140. if err != nil {
  141. return err
  142. }
  143. return nil
  144. }
  145. // 更新一条批量下载结果到数据库
  146. func UpdateBatchDownloadResult(id primitive.ObjectID, result domain.Progress, reasons []string) error {
  147. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  148. defer cancel()
  149. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  150. // _, err := collection.InsertOne(ctx, config)
  151. filter := bson.M{"_id": id} // 按ID匹配
  152. update := bson.M{
  153. // 如果存在就更新非progress字段
  154. "$set": bson.M{
  155. "progress": result,
  156. "failReasons": reasons,
  157. },
  158. }
  159. opts := options.Update().SetUpsert(true)
  160. _, err := collection.UpdateOne(ctx, filter, update, opts)
  161. if err != nil {
  162. return err
  163. }
  164. return nil
  165. }
  166. // 删除一条批量下载配置
  167. func DeleteBatchDownloadConfig(id primitive.ObjectID) error {
  168. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  169. defer cancel()
  170. filter := bson.M{"_id": id} // 删除条件
  171. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  172. _, err := collection.DeleteOne(ctx, filter)
  173. if err != nil {
  174. return err
  175. }
  176. return nil
  177. }
  178. // 插入批量下载结果数据
  179. func InsertBatchDownloadData(id primitive.ObjectID, data domain.BatchDownloadData) error {
  180. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  181. defer cancel()
  182. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  183. // _, err := collection.InsertOne(ctx, config)
  184. filter := bson.M{"_id": id} // 按ID匹配
  185. update := bson.M{
  186. // 如果存在就更新data字段
  187. "$set": bson.M{
  188. "data": data,
  189. },
  190. }
  191. opts := options.Update().SetUpsert(true)
  192. _, err := collection.UpdateOne(ctx, filter, update, opts)
  193. if err != nil {
  194. return err
  195. }
  196. return nil
  197. }
  198. // 获取下载数据包路径
  199. func GetBatchDownloadData(id primitive.ObjectID) (map[string]domain.BatchDownloadData, error) {
  200. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  201. defer cancel()
  202. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  203. var cfg domain.BatchDownloadConfig
  204. err := collection.FindOne(ctx, bson.M{"_id": id}).Decode(&cfg)
  205. if err != nil {
  206. return nil, err
  207. }
  208. return cfg.Data, nil
  209. }
  210. // 插入批量下载结果数据 - PowerFlow
  211. func InsertBatchDownloadPathPowerFlow(id primitive.ObjectID, sn string, date string, path string) error {
  212. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  213. defer cancel()
  214. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  215. // _, err := collection.InsertOne(ctx, config)
  216. filter := bson.M{"_id": id} // 按ID匹配
  217. update := bson.M{
  218. // 如果存在就更新data字段
  219. "$set": bson.M{
  220. "data." + sn + ".powerFlow." + date: path,
  221. },
  222. }
  223. opts := options.Update().SetUpsert(true)
  224. _, err := collection.UpdateOne(ctx, filter, update, opts)
  225. if err != nil {
  226. return err
  227. }
  228. return nil
  229. }
  230. // 插入批量下载结果数据 - MqttLogs
  231. func InsertBatchDownloadPathMqttLogs(id primitive.ObjectID, sn string, date string, logType string, path string) error {
  232. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  233. defer cancel()
  234. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  235. // _, err := collection.InsertOne(ctx, config)
  236. filter := bson.M{"_id": id} // 按ID匹配
  237. update := bson.M{
  238. // 如果存在就更新data字段
  239. "$set": bson.M{
  240. "data." + sn + ".mqttLogs." + date + "." + logType: path,
  241. },
  242. }
  243. opts := options.Update().SetUpsert(true)
  244. _, err := collection.UpdateOne(ctx, filter, update, opts)
  245. if err != nil {
  246. return err
  247. }
  248. return nil
  249. }
  250. // 插入批量下载结果数据 - EventAlarm
  251. func InsertBatchDownloadPathEventAlarm(id primitive.ObjectID, sn string, date string, path string) error {
  252. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  253. defer cancel()
  254. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  255. // _, err := collection.InsertOne(ctx, config)
  256. filter := bson.M{"_id": id} // 按ID匹配
  257. update := bson.M{
  258. // 如果存在就更新data字段
  259. "$set": bson.M{
  260. "data." + sn + ".eventAlarm." + date: path,
  261. },
  262. }
  263. opts := options.Update().SetUpsert(true)
  264. _, err := collection.UpdateOne(ctx, filter, update, opts)
  265. if err != nil {
  266. return err
  267. }
  268. return nil
  269. }
  270. // 插入批量下载结果数据 - AlarmSnapshot
  271. func InsertBatchDownloadPathAlarmSnapshot(id primitive.ObjectID, sn string, date string, path string) error {
  272. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  273. defer cancel()
  274. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  275. // _, err := collection.InsertOne(ctx, config)
  276. filter := bson.M{"_id": id} // 按ID匹配
  277. update := bson.M{
  278. // 如果存在就更新data字段
  279. "$set": bson.M{
  280. "data." + sn + ".alarmSnapshot." + date: path,
  281. },
  282. }
  283. opts := options.Update().SetUpsert(true)
  284. _, err := collection.UpdateOne(ctx, filter, update, opts)
  285. if err != nil {
  286. return err
  287. }
  288. return nil
  289. }
  290. // 插入批量下载结果数据 - EmsLogs
  291. func InsertBatchDownloadPathEmsLogs(id primitive.ObjectID, sn string, date string, logType string, path string) error {
  292. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  293. defer cancel()
  294. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  295. // _, err := collection.InsertOne(ctx, config)
  296. filter := bson.M{"_id": id} // 按ID匹配
  297. update := bson.M{
  298. // 如果存在就更新data字段
  299. "$set": bson.M{
  300. "data." + sn + ".emsLogs." + date + "." + logType: path,
  301. },
  302. }
  303. opts := options.Update().SetUpsert(true)
  304. _, err := collection.UpdateOne(ctx, filter, update, opts)
  305. if err != nil {
  306. return err
  307. }
  308. return nil
  309. }
  310. // 插入批量下载结果数据 - FaultRecord
  311. func InsertBatchDownloadPathFaultRecord(id primitive.ObjectID, sn string, fileName string, path string) error {
  312. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  313. defer cancel()
  314. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  315. // _, err := collection.InsertOne(ctx, config)
  316. filter := bson.M{"_id": id} // 按ID匹配
  317. update := bson.M{
  318. // 如果存在就更新data字段
  319. "$set": bson.M{
  320. "data." + sn + ".faultRecord." + fileName: path,
  321. },
  322. }
  323. opts := options.Update().SetUpsert(true)
  324. _, err := collection.UpdateOne(ctx, filter, update, opts)
  325. if err != nil {
  326. return err
  327. }
  328. return nil
  329. }
  330. // 插入批量下载结果数据 - UpdateLog
  331. func InsertBatchDownloadPathUpdateLog(id primitive.ObjectID, sn string, path string) error {
  332. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  333. defer cancel()
  334. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  335. // _, err := collection.InsertOne(ctx, config)
  336. filter := bson.M{"_id": id} // 按ID匹配
  337. update := bson.M{
  338. // 如果存在就更新data字段
  339. "$set": bson.M{
  340. "data." + sn + ".updateLog": path,
  341. },
  342. }
  343. opts := options.Update().SetUpsert(true)
  344. _, err := collection.UpdateOne(ctx, filter, update, opts)
  345. if err != nil {
  346. return err
  347. }
  348. return nil
  349. }
  350. // 插入批量下载结果数据 - Settings
  351. func InsertBatchDownloadPathSettings(id primitive.ObjectID, sn string, path string) error {
  352. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  353. defer cancel()
  354. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  355. // _, err := collection.InsertOne(ctx, config)
  356. filter := bson.M{"_id": id} // 按ID匹配
  357. update := bson.M{
  358. // 如果存在就更新data字段
  359. "$set": bson.M{
  360. "data." + sn + ".settings": path,
  361. },
  362. }
  363. opts := options.Update().SetUpsert(true)
  364. _, err := collection.UpdateOne(ctx, filter, update, opts)
  365. if err != nil {
  366. return err
  367. }
  368. return nil
  369. }
  370. // ==================== 结构化 item 结果(B2 完成判定) ====================
  371. // InitBatchDownloadItems 初始化 items 列表,并设置 totalCount / doneCount。
  372. // doneCount 初始化为已经处于终态的 item 数量(构建期就 failed/skipped 的),
  373. // 使得后续所有 item 转终态后 doneCount 恰好等于 totalCount。
  374. func InitBatchDownloadItems(id primitive.ObjectID, items []domain.BatchDownloadItem) error {
  375. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  376. defer cancel()
  377. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  378. doneInit := 0
  379. now := time.Now()
  380. for i := range items {
  381. if items[i].UpdatedAt.IsZero() {
  382. items[i].UpdatedAt = now
  383. }
  384. if items[i].Status != domain.ItemPending {
  385. doneInit++
  386. }
  387. }
  388. filter := bson.M{"_id": id}
  389. update := bson.M{
  390. "$set": bson.M{
  391. "items": items,
  392. "totalCount": len(items),
  393. "doneCount": doneInit,
  394. "progress": domain.ProgressRunning,
  395. },
  396. }
  397. opts := options.Update().SetUpsert(true)
  398. _, err := collection.UpdateOne(ctx, filter, update, opts)
  399. if err != nil {
  400. return err
  401. }
  402. return nil
  403. }
  404. // FinishBatchDownloadItem 原子地把某个 pending item 置为终态并递增 doneCount。
  405. // 幂等:filter 要求该 item 当前为 pending,重复调用(Asynq 重试/重复回调)不会重复计数。
  406. // 返回 (doneCount, totalCount, error);若该 item 已非 pending(幂等跳过),返回 (-1, -1, nil)。
  407. func FinishBatchDownloadItem(id primitive.ObjectID, key string, status domain.ItemStatus, stage, reason string) (int, int, error) {
  408. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  409. defer cancel()
  410. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  411. filter := bson.M{
  412. "_id": id,
  413. "items": bson.M{
  414. "$elemMatch": bson.M{"key": key, "status": domain.ItemPending},
  415. },
  416. }
  417. update := bson.M{
  418. "$set": bson.M{
  419. "items.$.status": status,
  420. "items.$.stage": stage,
  421. "items.$.reason": reason,
  422. "items.$.updatedAt": time.Now(),
  423. },
  424. "$inc": bson.M{"doneCount": 1},
  425. }
  426. opts := options.FindOneAndUpdate().SetReturnDocument(options.After)
  427. var doc domain.BatchDownloadConfig
  428. err := collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&doc)
  429. if err != nil {
  430. if err == mongo.ErrNoDocuments {
  431. // item 已非 pending(重复回调/重试),幂等跳过
  432. return -1, -1, nil
  433. }
  434. return 0, 0, err
  435. }
  436. return doc.DoneCount, doc.TotalCount, nil
  437. }
  438. // GetBatchDownloadItems 读取某任务的全部 item 结果,用于聚合派生终态。
  439. func GetBatchDownloadItems(id primitive.ObjectID) ([]domain.BatchDownloadItem, error) {
  440. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  441. defer cancel()
  442. collection := database.GetCollection(CollectionName_BatchDownloadConfig)
  443. var cfg domain.BatchDownloadConfig
  444. err := collection.FindOne(ctx, bson.M{"_id": id},
  445. options.FindOne().SetProjection(bson.M{"items": 1})).Decode(&cfg)
  446. if err != nil {
  447. return nil, err
  448. }
  449. return cfg.Items, nil
  450. }