diff --git a/.gitignore b/.gitignore index 8da5a52..d884903 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ build/ ### Mac OS ### .DS_Store /.idea/encodings.xml + +# Migration script bytecode +scripts/__pycache__/ diff --git a/docs/GALLERY_INDEX_REWRITE.md b/docs/GALLERY_INDEX_REWRITE.md new file mode 100644 index 0000000..151bde1 --- /dev/null +++ b/docs/GALLERY_INDEX_REWRITE.md @@ -0,0 +1,150 @@ +# 图片索引缓存（GalleryIndex）— 重写模块说明 + +## 问题 + +在线阅读的索引是「gid → 每一页 imgkey」，存在 `cache.db` 的两张表里： + +| 表 | 列 | 含义 | +|---|---|---| +| `gidToKey` | `gid`, `key`, `pages` | 画廊头部：gid、画廊 key、总页数 | +| `ImageKeyCache` | `gid`, `imgkey`, `page` | 每页的 imgkey | + +旧实现（`GalleryManageService.cacheImagesKey`）依次做四件事：查缓存、判断是否需要重建、 +外呼抓 mpv 页、逐页 `INSERT`。由此产生的实际问题： + +1. **没有唯一约束**。实测生产缓存库有 4 组重复 gid、97 组重复 `(gid,page)`。 +2. **整组替换不是原子的**。先写头部再逐页写，中途失败会留下「头部命中、页 key 缺失」的 + 半截索引；后续请求直接判定为已缓存，永远取不到图。 +3. **用行数判断完整性**。`count(*) == pages` 无法发现「页码 1、3 而 pages=2」这种跳号。 +4. **没有并发合并**。同一本画廊被并发打开时，两个构建者会互相删除对方刚写入的行。 +5. **逐页写入放大写事务**，抓取与写入之间没有事务边界。 + +## 交付物与职责 + +| 文件 | 职责 | +|---|---| +| `Service/GalleryIndexService.java` | 索引的唯一读写入口：同 gid 合并、抓取在事务外、校验后整组短事务替换 | +| `Exception/GalleryIndexException.java` | 分类失败原因：INVALID_LINK / UNAVAILABLE / MALFORMED / TRANSIENT | +| `Configuration/GalleryIndexConfiguration.java` | 缓存库的事务管理器与读写模板（独立文件，未改 SqlConfiguration） | +| `Dao/cache/ImageCacheMapper.java` | 新增页码查询、脏行计数、批量写入（原有方法全部保留） | +| `Maintenance/GalleryIndexMaintenance.java` | 离线体检与修复工具（只删不健康 gid 的整组索引） | +| `scripts/migrate_image_index_cache.{sh,sql}` | 迁移：默认 dry-run、备份、清理、建唯一索引 | +| `Service/GalleryIndexServiceTest.java` | 服务编排的行为约束（无 Spring、无网络） | +| `Maintenance/GalleryIndexMaintenanceTest.java` | 工具在真实临时 SQLite 上的判定与删除范围 | +| `Dao/cache/ImageCacheMapperSqlTest.java` | 注解 SQL 在真实 MyBatis + SQLite 上的验证 | + +## 接口 + +```java +GidToKey ensureIndex(String url) throws GalleryIndexException; +``` + +返回该画廊的 `gid` / `key` / `pages`。三条不变量： + +- **抓取在事务外**：`upstream.fetchPageKeys(url)` 先于任何写事务执行；写事务只在 + `publish(...)` 里开启，删除旧索引与批量写入在同一个事务内整体生效。 +- **校验先于写入**：行必须满足 gid 与请求一致、页码恰好为 1..N、不重复、imgkey 非空， + 否则抛 MALFORMED，此时尚未开启写事务，库中旧索引原样保留。 +- **读在快照内**：`lookup` 在同一个读事务里读头部与页码，避免读到整组替换的中间状态。 + +失败原因： + +| Reason | 触发条件 | isPermanent() | 建议处置 | +|---|---|---|---| +| INVALID_LINK | 链接里取不到 gid 或 key | true | 直接回「链接错误」 | +| UNAVAILABLE | 上游返回空页列表（下架/删除） | true | 回「该图片已下架或已被删除」 | +| MALFORMED | 上游页 key 不自洽 | false | 可重试，回「网络波动或其他异常」 | +| TRANSIENT | 上游异常或写库失败 | false | 可重试，回「网络波动或其他异常」 | + +## 接入方式（由主模型完成 GalleryManageService 侧改动） + +构造函数注入： + +```java +final GalleryIndexService galleryIndexService; +``` + +`cacheImagesKey` 改为： + +```java +public String cacheImagesKey(String url) { + Response response = Response.generateResponse(); + try { + GidToKey gidToKey = galleryIndexService.ensureIndex(url); + return response.success(objectMapper.valueToTree(gidToKey)).toJSONString(); + } catch (GalleryIndexException e) { + response.failure(e.getReason() == GalleryIndexException.Reason.INVALID_LINK + ? "链接错误" + : e.isPermanent() ? "该图片已下架或已被删除" : "网络波动或其他异常"); + return response.toJSONString(); + } +} +``` + +Spring 装配无需额外配置：`GalleryIndexConfiguration` 已把 `GalleryIndexService` 注册为 bean， +依赖 `ImageCacheMapper`（cache 数据源）与两个 TransactionTemplate。原来的 +`isIndexComplete` / `rollbackImageIndex` 私有方法与 `ImageCacheMapper` 的 +`countImageKeyCacheByGid` / `delete*ByGid` 仍保留，供旧调用方与非索引路径使用。 + +读取路径 `getCachedImage` 不需要改动：它按 `(gid,page)` 查 ImageKeyCache，现在会走新加的 +`ux_ImageKeyCache_gid_page` 唯一索引。 + +## 迁移 + +```bash +# 1. dry-run（只读，不带 --apply 绝不写入） +scripts/migrate_image_index_cache.sh --db /path/to/copy.db + +# 2. 先在 SQLite 备份副本上演练 +sqlite3 -readonly /home/lionwebsite/cache.db ".backup '/tmp/cache-copy.db'" + +# 3. 确认后执行（自动做 .backup 备份并校验） +scripts/migrate_image_index_cache.sh --db /tmp/cache-copy.db --apply +``` + +迁移只做两类写入：删除不健康 gid 的整组索引（重复头部、半截、跳号、空 imgkey）、 +删除孤儿页 key；随后建立 `ux_gidToKey_gid` 与 `ux_ImageKeyCache_gid_page`。 +若清理后仍有重复，建唯一索引会失败并整体回滚。真实的 `/home/lionwebsite/cache.db` 未被本次工作修改。 + +离线修复工具： + +```bash +java -cp sqlite-jdbc.jar Maintenance/GalleryIndexMaintenance.java --db /tmp/cache-copy.db # dry-run +java -cp sqlite-jdbc.jar Maintenance/GalleryIndexMaintenance.java --db /tmp/cache-copy.db --apply # 执行 +``` + +对 `/home/lionwebsite` 下的库运行 `--apply` 会被拒绝，除非显式加 `--allow-production`。 + +## 测试 + +在 `/tmp` 的独立源码副本中运行（未复制任何生产 runtime config、未改生产库）： + +``` +mvn -o -Djacoco.skip=true \ + -Dtest='GalleryIndexServiceTest,GalleryIndexMaintenanceTest,ImageCacheMapperSqlTest' \ + -DfailIfNoSpecifiedTests=false test + +Tests run: 9, Failures: 0, Errors: 0 -- GalleryIndexMaintenanceTest +Tests run: 21, Failures: 0, Errors: 0 -- GalleryIndexServiceTest +Tests run: 7, Failures: 0, Errors: 0 -- ImageCacheMapperSqlTest +Tests run: 37, Failures: 0, Errors: 0 +BUILD SUCCESS +``` + +覆盖的关键场景： + +- 同 gid 24 线程并发只抓取一次、只发布一次；失败结果也被等待者共享；失败后可重试。 +- 抓取发生在事务之外；所有删除与写入都在事务内；整组替换只有一个写事务。 +- 行数相同但页码跳号、空 imgkey、空 key、pages<=0 均判定为不完整并重建。 +- 写入失败整体回滚，旧索引与失败前逐字段一致。 +- 上游畸形数据（gid 不符、页码重复/跳号/越界、imgkey 为空）在开启写事务之前被拒绝。 +- 大规模索引按 BATCH_SIZE 分片写入；重复发布替换旧行而非累积。 +- 注解 SQL 在真实 SQLite 上验证，含唯一约束确实拒绝重复写入。 + +## 已知边界 + +- 迁移后 gidToKey 433 条、ImageKeyCache 43704 条（原 443 / 43918）。 + 被删除的是 4 个冲突或不完整 gid 的整组索引与 214 行关联页 key，由下一次在线阅读重抓。 +- 未做性能压测，不承诺加索引后的具体延迟收益，只确认查询计划已由全表扫描变为索引查找。 +- lookup 的读事务只是为了拿到一致快照；实测 sqlite-jdbc 不允许在已建立的连接上设置 + readOnly，因此读模板只设超时、不设只读标志。 diff --git a/docs/SUBSCRIPTION_CACHE_REWRITE.md b/docs/SUBSCRIPTION_CACHE_REWRITE.md new file mode 100644 index 0000000..c03f191 --- /dev/null +++ b/docs/SUBSCRIPTION_CACHE_REWRITE.md @@ -0,0 +1,61 @@ +# 订阅缓存分代化 — 重写模块说明 + +## 问题 + +一个订阅账号对应两份内容：`v2ray.txt`（Base64 节点）与 `clash.yaml`（Clash 配置）。 +旧实现先 `atomicWrite` 写 v2ray，再 `atomicWrite` 写 clash，两次替换彼此独立： + +1. **两份文件可以不同版本**。第二次写入失败（磁盘满、进程被杀）时，用户会拿到 + 「新 v2ray + 旧 clash」的混合版本，且没有任何状态能表明它已经撕裂。 +2. **读取方无法保证配对**。分发路径与备机快照各自 `Files.readAllBytes` 两个路径， + 在这两次读之间完成一次刷新，就会读到跨版本组合。 +3. **过期刷新可能复活**。在途刷新返回时只比较上游 key 与过滤开关；一次 + `invalidateCache` 之后，迟到的响应仍会把内容写回被清空的路径。 +4. **失败留半成品**。第一次写成功、第二次失败时，磁盘上就是只有一份文件的账号。 + +## 交付物与职责 + +| 文件 | 职责 | +|---|---| +| `Util/SubscriptionCacheStore.java` | 分代存储：两份文件同一分代写入 → 校验 → 指针原子发布；读取始终来自指针指向的同一分代 | +| `Service/SubscriptionRefreshService.java` | 抓取与解析在协调器锁外进行，发布短持锁；`invalidateCache` 递增版本使在途刷新失效 | +| `Service/SubscriptionStandbySnapshotService.java` | 快照构建在读锁内通过同一 store 解析成对内容，保证同版本 | +| `Util/ClashSubscriptionParser.java` | 按 YAML 语义过滤高倍节点并清理 `proxy-groups` 引用（由主模型提供） | + +## 目录布局 + +``` +//current 指针（Properties 文本，原子替换） +//generations//v2ray.txt +//generations//clash.yaml +//v2ray.txt + clash.yaml 旧结构，仅在两份都在时按 legacy 读 +``` + +`current` 记录分代名与两份文件的 SHA-256。分代名是定宽 `时间戳-序号`， +因此字典序即时间序；默认只保留最近两个分代（当前与上一个）。 + +## 不变量 + +- **发布最后一步才切指针**：先写两份文件，读回校验长度与内容，再原子替换 `current`。 + 抓取失败、解析失败、任一份写入失败、指针写入失败，都会回滚新分代目录并保留上一个完整版本。 +- **读取配对**：`read` / `resolve` / `hasComplete` 走同一份 `load` 契约，每个调用只读一次 + 指针快照，再按该快照校验双文件哈希。只有两份齐全且匹配才算一个有效版本； + 指针缺失才回退 legacy（两份齐全时），指针存在但读不出完整版本（半写、篡改、缺字段） + 时按无可用缓存处理，不回退到更旧的半份内容。 +- **分发不会发出半份缓存**：`cachedPath` 在无有效版本时返回保证不存在的哨兵路径 + （`.unavailable/`），使 `SubService` 判定为 503，而不会被单份 legacy 文件骗过。 + `SubService.updateSub` 在读锁内完成 resolve 与 `FileDownload.export`，与 publish/invalidate 互斥。 +- **失效不可复活**：`invalidateCache` 在写锁内删除指针/分代/旧结构文件，并移除该账号的 + 刷新版本标记；在途响应返回时因版本标记对不上而被丢弃，不会重新发布。 +- **兼容旧结构**：只有当账号根下 `v2ray.txt` 与 `clash.yaml` 都在时才读为 legacy 版本； + 新分代发布成功后旧结构文件被移除，避免两份来源并存。 + +## 测试 + +| 测试 | 锁定行为 | +|---|---| +| `Util/SubscriptionCacheStoreTest` | 发布后两份同分代；第二份写入失败 / 指针发布失败均保留旧版本；legacy 双文件才有效；半份 legacy、缺一份分代、内容被篡改、坏指针、指针指向缺失分代均判定不可用；每次查询只读一次指针；哨兵路径保证不存在 | +| `Service/SubscriptionCacheGenerationTest` | 在途旧刷新在失效与新版发布后被丢弃；被丢弃的刷新不留任何缓存 | +| `Service/SubscriptionFilteringTest` | 过滤规则；失败保留上次完整版本；`cachedPath` / `hasCompleteCache` 新契约；legacy 路径定位 | +| `Service/SubscriptionStandbySnapshotServiceTest` | 快照从单一分代读成对内容；不完整 legacy 账号不入快照 | +| `Util/ClashSubscriptionParserTest` | YAML 语义过滤（内联节点与引用）与畸形正文拒绝 | diff --git a/docs/TASK_PROTOCOL_V2.md b/docs/TASK_PROTOCOL_V2.md new file mode 100644 index 0000000..acb4435 --- /dev/null +++ b/docs/TASK_PROTOCOL_V2.md @@ -0,0 +1,80 @@ +# 任务生命周期与协议 v2 + +范围：主站下载任务的持久化生命周期、与存储节点的可靠上报协议。对应评估报告第 2、6 节。 + +## 为什么重写 + +原实现里创建先`POST`上游、后落库，删除先删库、后等节点，节点上报异步写完后立刻清队列：上下游任一侧失败都会留下「主站与节点说法不一致」的状态，且没有任何东西能把它们对回来。 + +复现过的具体后果：模拟状态消息发送失败后，完成任务已从节点队列移除、消息却未发出，于是「归档已存在、主站仍显示未完成」。 + +## 不变量 + +1. `gid` 只有一个有效生命周期意图。创建先落唯一 `CREATE` 意图（并写一条可见的占位任务行），靠 `gallery(gid)` 主键唯一把并发创建收敛成一个赢家；输家直接得到业务失败。 +2. 上游副作用只在意图持久化之后发生。顺序固定为：登记意图 → 只读解析（`prepare`，无 POST）→ 持久化 `SUBMITTING` → 真正 `submit`。 +3. `SUBMITTING` 之后绝不自动重发。上游不支持幂等，因此超时/断连/落库失败/重启中断一律落 `UNKNOWN` 等待人工对账处理，而不是再 `POST` 一次。 +4. 任务行与周用量在同一短事务里写入；重复落库不重复计量。数据库失败时任务行与意图保持一致（要么都在，要么明确为 `UNKNOWN`），不允许出现「有任务没计量」。 +5. 删除失败保留记录。先落 `DELETE_PENDING` 与稳定 `deleteOperationId`，只有节点确认（或文件已不存在）后才在同一事务里清收藏、删任务行并留 `DELETED` 墓碑。 +6. 上报按 `revision` 单调推进，按 `operationId` 归属。迟到的旧世代上报被丢弃且不改变当前任务；墓碑（`DELETED`）永久忽略所有上报，已删除任务只能由显式 `CREATE` 重新建立新世代。 +7. 回执只在提交之后发送。数据库失败不回 ACK，节点保留快照重投；重复上报也回 ACK 以停止重放，但不会重复通知。 +8. 通知与 WebSocket 推送在事务提交之后、由独立执行器发送；慢通知不阻塞状态提交。 + +9. 只有确认已提交（`SUBMITTED`）或迁移前确实提交过的旧任务才允许下发给节点。`UNKNOWN`、`RESERVED`、`SUBMITTING`、`DELETE_PENDING`、`DELETED` 一律不下发：上游不支持幂等，把未确认/未计量的任务重投会在节点侧变成「下载完成却 0 额度」。 +10. 启动时先绑定上报处理器与判定解析器、执行恢复（`SUBMITTING → UNKNOWN`、释放 `RESERVED`），完成之后才允许连接与自动重投；恢复失败时只保持连接，禁止自动派发。 + +## 状态表 + +| 层 | 取值 | +| --- | --- | +| `task_operation.state` | `RESERVED` `SUBMITTING` `SUBMITTED` `UNKNOWN` `DELETE_PENDING` `DELETED` | +| 任务展示状态 | `提交中` `已提交` `下载中` `等待压缩` `压缩中` `下载完成` `提交未知` `待删除` `删除失败` | +| 节点上报状态码 | `1` 下载中 `2` 等待压缩 `3` 压缩中 `4` 下载完成（沿用旧码） | + +状态码到中文文案只在 `TaskStatus` 一处映射。 + +## 线协议 + +帧格式不变：`[messageType(1B)][length(4B, 大端)][JSON]`。旧类型 `0..9` 的字段一个都不动（旧节点 Jackson 会拒绝未知字段）。 + +| 类型 | 消息 | 关键字段 | +| --- | --- | --- | +| 10 | `ProtocolHelloMessage` | `protocolVersion=2`, `features=["reliable-task-v2"]` | +| 11 | `TaskCommandMessage` | `operationId`, `action=DOWNLOAD\|DELETE`, `gid`, `name` | +| 12 | `TaskReportMessage` | `reportId`, `revision`, `galleryTasks[]`, `operationIds{gid→op}` | +| 13 | `TaskReportAckMessage` | `reportId`, `result` | + +流程：主站先发旧 `Identity`，再发类型 10；节点回类型 10 声明 v2 后，才允许使用 `11..13`。未协商成功时自动退化为旧的 `DownloadPostMessage`/`DeleteGalleryMessage`（此时不承诺可靠 ACK，属已知兼容边界）。 + +## 部署与迁移 + +1. 用 `scripts/migrate_task_operation_v2.sql` 建表。脚本只有独立语句、不含 `BEGIN/COMMIT`，由外层 wrapper 包事务并在执行前备份。 +2. 该脚本还会在 `gallery(gid)` 上建唯一索引。历史库若已有重复 gid，这条语句会失败：先人工确认重复来源再处理，不要直接删行。 +3. 主库连接需要 busy 等待（`SqlConfiguration` 已为主数据源设置 `PRAGMA busy_timeout=5000`）。单文件 SQLite 并发开短事务时，没有等待窗口会直接抛出 `SQLITE_BUSY`。生产库建议同时启用 WAL。 +4. 旧任务行没有操作记录，首次收到上报时会按显式策略补建一次「世代」（优先采纳上报里的 `operationId`），不会改动其已有状态。 + +## 验证 + +`mvn -o test`：549 项，0 失败/错误/跳过。 + +新增关键用例（`TaskLifecycleIntegrationTest`，真实 SQLite 文件 + 真实事务/触发器/唯一约束，落在 `/tmp` 临时目录）： + +- 意图与占位行先于上游 `POST`；`SUBMITTING` 先于 `submit`。 +- 同 `gid` 6 线程并发创建：上游只被提交一次、任务行不超过一条。 +- 上游超时后落 `UNKNOWN`；再次点击创建也不会第二次 `POST`。 +- 重启恢复把中断的 `SUBMITTING` 判为 `UNKNOWN`（任务行同步显示「提交未知」），安全释放 `RESERVED` 占位，且不改写 `operationId`、不重发。 +- 额度写入失败时落库事务回滚，任务保持可对账的 `提交未知` + `UNKNOWN`。 +- 重投/重试复用同一 `operationId`。 +- 判定为拒绝的任务（`UNKNOWN` 等）绝不下发，也不退化为无 `operationId` 的旧命令。 +- 删除失败保留任务行与意图，重试复用同一删除 ID；成功后清收藏、删行、留墓碑。删除意图 ID 不匹配时不得完成删除。 +- 正在提交（`RESERVED`/`SUBMITTING`）的任务拒绝删除；释放预占只清匹配世代的记录与占位行。 +- 落库严格匹配本次创建世代（expectedOperationId），被删除/换代后不得复活已删 gid。 +- 墓碑永久忽略所有上报（含同一 `operationId`），只允许显式 `CREATE` 用新 `operationId` 重建。 +- 旧节点不带 `operationId` 的连续上报保持可接受；新 v2 已知世代缺归属的上报仍被拒。 +- 过期 `revision`、缺 `operationId`、待删除任务的上报都被丢弃；未知 gid 忽略但回执。 +- 重复上报不重复通知；回执写入失败时 `apply` 返回不可回执结果。 + +`TaskProtocolV2CodecTest` 锁定类型 10..13 的帧头与字段名，并断言旧 `DownloadPostMessage` 没有混入新字段。 + +## 已知限制 + +未做双机真实故障演练。`UNKNOWN` 目前只提供状态与持久化记录：任务行会显示「提交未知」并拒绝自动重投/重试，对账与人工解决的接口由后续批次补齐。 diff --git a/pom.xml b/pom.xml index 037755b..afd0a70 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,11 @@ 21 + + org.yaml + snakeyaml + 2.6 + org.springframework.boot spring-boot-starter-webmvc diff --git a/scripts/migrate_image_index_cache.sh b/scripts/migrate_image_index_cache.sh new file mode 100755 index 0000000..692a2f5 --- /dev/null +++ b/scripts/migrate_image_index_cache.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# 图片索引（cache.db）迁移：为缓存表补唯一约束与查询索引。 +# +# 设计约束： +# * 默认 dry-run，只读检查；必须显式 --apply 才执行写入。 +# * 必须显式 --db 指定数据库文件，不读取任何运行时配置，也不推断生产路径。 +# * 写库用 sqlite3 -cmd '.bail on'，任一语句出错立即以非零码退出，事务不提交。 +# * 只以读写模式打开目标库（文件必须已存在，不会新建）；dry-run 全用 -readonly。 +# * 备份用 SQLite 自身的 .backup（online backup API），而不是 cp： +# cp 会漏掉 WAL 里尚未 checkpoint 的已提交内容。 +# * 备份后立即校验备份可读且含两张缓存表，失败则中止并删除无效备份。 +# * 不健康（不完整/冲突）的 gid 整组索引会被删除，由下一次在线阅读重新抓取。 +# * 唯一约束建在唯一索引上：若清理后仍有重复，建索引会失败并整体回滚。 +# +# 用法： +# scripts/migrate_image_index_cache.sh --db /tmp/cache-copy.db # dry-run（只读） +# scripts/migrate_image_index_cache.sh --db /tmp/cache-copy.db --apply # 执行迁移 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SQL_FILE="$SCRIPT_DIR/migrate_image_index_cache.sql" + +db_path="" +apply="no" + +usage() { + sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --db) db_path="${2:-}"; shift 2 ;; + --apply) apply="yes"; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "未知参数: $1" >&2 + exit 2 ;; + esac +done + +if [[ -z "$db_path" ]]; then + echo "必须显式指定缓存库: --db ；本脚本不会自动推断数据库位置。" >&2 + exit 2 +fi + +if [[ ! -f "$db_path" ]]; then + echo "数据库不存在: $db_path" >&2 + exit 2 +fi + +if [[ ! -f "$SQL_FILE" ]]; then + echo "缺少 SQL: $SQL_FILE" >&2 + exit 2 +fi + +if ! command -v sqlite3 >/dev/null 2>&1; then + echo "缺少 sqlite3 命令行工具" >&2 + exit 2 +fi + +if [[ "$(sqlite3 -readonly "$db_path" \ + "select count(*) from sqlite_master where type='table' and name in ('gidToKey','ImageKeyCache');")" != "2" ]]; then + echo "缓存表缺失：$db_path 中找不到 gidToKey 与 ImageKeyCache。" >&2 + exit 2 +fi + +# 体检统计，与 SQL 里实际清理的范围一致： +# bad_gids 将被整组删除的 gid 个数（去重），覆盖： +# 重复头部 / 行数不符 / 页码去重后不足 / 页码区间异常 / +# 空 imgkey / 空 key / 非法 pages +# bad_gid_rows 这些 gid 占用的头部行数（重复头部的 gid 会多于 1 行） +# duplicate_pages 重复的 (gid,page) 组数，仅用于展示，会随所属 gid 一并清理 +# orphan_gids 有页 key 但无头部记录的 gid 个数 +read_stats() { + sqlite3 -readonly "$db_path" " + select 'header_rows', count(*) from gidToKey; + select 'page_rows', count(*) from ImageKeyCache; + select 'duplicate_gids', count(*) from (select gid from gidToKey group by gid having count(*) > 1); + select 'duplicate_pages', count(*) from (select gid,page from ImageKeyCache group by gid,page having count(*) > 1); + select 'bad_gids', count(distinct g.gid) from gidToKey g + where g.pages is null or g.pages <= 0 + or g.key is null or trim(g.key) = '' + or (select count(*) from ImageKeyCache i where i.gid = g.gid) <> g.pages + or (select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) <> g.pages + or (select min(i.page) from ImageKeyCache i where i.gid = g.gid) <> 1 + or (select max(i.page) from ImageKeyCache i where i.gid = g.gid) <> g.pages + or exists (select 1 from ImageKeyCache i + where i.gid = g.gid and (i.imgkey is null or trim(i.imgkey) = '')); + select 'bad_gid_rows', coalesce(sum(c), 0) from ( + select count(*) as c from gidToKey g where g.gid in ( + select g2.gid from gidToKey g2 where g2.pages is null or g2.pages <= 0 + or g2.key is null or trim(g2.key) = '' + or (select count(*) from ImageKeyCache i where i.gid = g2.gid) <> g2.pages + or (select count(distinct i.page) from ImageKeyCache i where i.gid = g2.gid) <> g2.pages + or (select min(i.page) from ImageKeyCache i where i.gid = g2.gid) <> 1 + or (select max(i.page) from ImageKeyCache i where i.gid = g2.gid) <> g2.pages + or exists (select 1 from ImageKeyCache i + where i.gid = g2.gid and (i.imgkey is null or trim(i.imgkey) = ''))) + group by g.gid); + select 'orphan_gids', count(distinct gid) from ImageKeyCache + where gid not in (select gid from gidToKey); + select 'strict_index', count(*) from sqlite_master + where type='index' and name='ux_ImageKeyCache_gid_page'; + " +} + +mode_label="dry-run（只读）" +if [[ "$apply" == "yes" ]]; then mode_label="apply（读写）"; fi + +echo "=== 缓存库图片索引迁移（$mode_label） ===" +echo "数据库: $db_path" +echo "--- 迁移前体检 ---" +read_stats | while IFS='|' read -r name value; do + printf ' %-16s %s\n' "$name" "$value" +done + +if [[ "$apply" != "yes" ]]; then + echo + echo "dry-run：未做任何写入。确认无误后加 --apply 执行迁移。" + echo "提示：请先在 SQLite 副本上演练，副本应这样生成（含 WAL 内容）：" + echo " sqlite3 -readonly <生产库> \".backup '<副本路径>'\"" + exit 0 +fi + +# ---- 备份：SQLite online backup，包含 WAL 中已提交但未 checkpoint 的内容 ---- +backup="$db_path.before-image-index.$(date -u +%Y%m%dT%H%M%SZ)" +if ! sqlite3 "$db_path" ".backup '$backup'"; then + echo "备份失败，未做任何修改: $backup" >&2 + exit 3 +fi +if [[ ! -s "$backup" ]]; then + echo "备份文件为空，未做任何修改: $backup" >&2 + exit 3 +fi +if [[ "$(sqlite3 -readonly "$backup" \ + "select count(*) from sqlite_master where type='table' and name in ('gidToKey','ImageKeyCache');")" != "2" ]]; then + echo "备份校验失败（缺少缓存表），未做任何修改，已删除无效备份。" >&2 + rm -f -- "$backup" + exit 3 +fi +echo "已备份（SQLite .backup）: $backup" + +# ---- 执行迁移：.bail on 保证任一语句失败即退出，事务不提交 ---- +sqlite3 -cmd '.bail on' "$db_path" < "$SQL_FILE" + +echo "--- 迁移后体检 ---" +read_stats | while IFS='|' read -r name value; do + printf ' %-16s %s\n' "$name" "$value" +done + +echo "图片索引迁移完成: $db_path" diff --git a/scripts/migrate_image_index_cache.sql b/scripts/migrate_image_index_cache.sql new file mode 100644 index 0000000..50e19c2 --- /dev/null +++ b/scripts/migrate_image_index_cache.sql @@ -0,0 +1,67 @@ +-- 图片索引缓存迁移：为 cache.db 的两张索引表补唯一约束与查询索引。 +-- +-- 目标（对应本次重写的三条不变量）： +-- 1. gid 唯一 —— 同一本画廊只能有一条 gidToKey 记录； +-- 2. (gid,page) 唯一 —— 同一页只能有一个 imgkey，同时天然满足按页查询的索引需求； +-- 3. 只保留「完整」的索引 —— 半截或冲突的 gid 整组删除，让下一次在线阅读重新抓取。 +-- +-- 为什么用唯一索引而不是重建表： +-- * 唯一索引对 SQLite 与唯一约束等价，都会拒绝重复写入； +-- * 不需要搬表、改列，旧版本二进制仍可读写同一份数据； +-- * 回滚只需 drop index，最坏情况也只需回到脚本自动生成的备份文件。 +-- +-- 幂等：所有重建索引都带 IF NOT EXISTS；清理语句只按集合删除，重复执行结果相同。 +-- 安全：本脚本只做 DELETE 与 CREATE INDEX，不建表、不改列、不碰 gidToKey/ImageKeyCache +-- 之外的任何对象。必须在事务中整体提交，调用方使用 sqlite3 -bail 保证任一语句 +-- 失败即终止、事务回滚。 +-- +-- 执行前请务必先备份数据库文件（迁移脚本 migrate_image_index_cache.sh --apply 会自动备份）。 + +BEGIN IMMEDIATE; + +-- ---------- 1. 冲突 gid：同一 gid 有多条头部记录 ---------- +-- 无法判断哪一条正确，整组移除，由下次访问重抓。 +CREATE TEMP TABLE __conflict_gid AS +SELECT gid FROM gidToKey GROUP BY gid HAVING COUNT(*) > 1; + +DELETE FROM ImageKeyCache WHERE gid IN (SELECT gid FROM __conflict_gid); +DELETE FROM gidToKey WHERE gid IN (SELECT gid FROM __conflict_gid); + +-- ---------- 2. 半截或非法 gid：头部与页 key 不一致 ---------- +-- 判定同时覆盖：页码数量不符、页码不连续（跳号/重复）、区间不是 1..pages、 +-- 头部字段为空、以及存在空的 imgkey 行。 +CREATE TEMP TABLE __broken_gid AS +SELECT g.gid +FROM gidToKey g +WHERE g.pages IS NULL OR g.pages <= 0 + OR g.key IS NULL OR trim(g.key) = '' + OR (SELECT COUNT(*) FROM ImageKeyCache i WHERE i.gid = g.gid) <> g.pages + OR (SELECT COUNT(DISTINCT i.page) FROM ImageKeyCache i WHERE i.gid = g.gid) <> g.pages + OR (SELECT MIN(i.page) FROM ImageKeyCache i WHERE i.gid = g.gid) <> 1 + OR (SELECT MAX(i.page) FROM ImageKeyCache i WHERE i.gid = g.gid) <> g.pages + OR EXISTS (SELECT 1 FROM ImageKeyCache i + WHERE i.gid = g.gid AND (i.imgkey IS NULL OR trim(i.imgkey) = '')); + +DELETE FROM ImageKeyCache WHERE gid IN (SELECT gid FROM __broken_gid); +DELETE FROM gidToKey WHERE gid IN (SELECT gid FROM __broken_gid); + +-- ---------- 3. 孤儿页 key：没有对应头部记录 ---------- +DELETE FROM ImageKeyCache WHERE gid NOT IN (SELECT gid FROM gidToKey); + +-- ---------- 4. 唯一约束 ---------- +-- 到这里若仍有重复，说明上面的判定有遗漏；CREATE UNIQUE INDEX 会失败， +-- 配合 -bail 让整个事务回滚，不会留下「一半约束」的库。 +CREATE UNIQUE INDEX IF NOT EXISTS ux_gidToKey_gid + ON gidToKey (gid); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_ImageKeyCache_gid_page + ON ImageKeyCache (gid, page); + +-- 按 gid 统计页数会用到 (gid,page) 前缀，按页取 key 也走同一索引， +-- 因此不再单独建 page 索引，避免重复占用写放大。 + +DROP TABLE __conflict_gid; +DROP TABLE __broken_gid; + +COMMIT; + diff --git a/scripts/migrate_task_operation_v2.sql b/scripts/migrate_task_operation_v2.sql new file mode 100644 index 0000000..64de822 --- /dev/null +++ b/scripts/migrate_task_operation_v2.sql @@ -0,0 +1,31 @@ +-- 任务操作记录表（协议 v2：先持久化意图，再产生外部副作用） +-- +-- 这是显式迁移脚本，不会被应用自动执行，也不对生产库运行。 +-- 执行前请备份：sqlite3 LionWebsite.db ".backup LionWebsite.db.before-task-operation" +-- +-- 幂等：可重复执行。 + +create table if not exists task_operation +( + gid integer not null + constraint task_operation_pk + primary key, + operationId varchar(64), + deleteOperationId varchar(64), + state varchar(20) not null, + lastRevision integer not null default 0, + lastReportId varchar(64), + updateTime long not null default 0 +); + +-- 节点上报消费只按 gid 命中主键，这里额外为「按状态找待处理意图」提供查询入口。 +create index if not exists idx_task_operation_state on task_operation (state); + +-- gallery.gid 目前只有 table 级主键约束（部分历史库甚至是自增主键的衍生约束）， +-- 这里显式建立唯一索引，保证同一 gid 不会出现两条任务行： +-- 「同一 gid 只有一个有效意图」是 v2 生命周期成立的前提。 +-- 若该语句因历史重复数据失败，请先人工确认重复行来源再处理，不要直接删除。 +create unique index if not exists idx_gallery_gid_unique on gallery (gid); + +-- 旧任务行没有操作记录：启动/查询时按 gid 惰性补建为 SUBMITTED（已在上游提交过）， +-- 不需要在迁移里批量写入，避免把正在进行的任务判成 UNKNOWN。 diff --git a/scripts/migrate_task_operations.py b/scripts/migrate_task_operations.py new file mode 100644 index 0000000..2a43d4e --- /dev/null +++ b/scripts/migrate_task_operations.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Explicit task-v2 schema migration. Default is read-only; --apply backs up first.""" +import argparse +import datetime +import json +from pathlib import Path +import sqlite3 + + +def migrate(database: Path, apply: bool = False) -> dict: + if not database.is_file(): + raise ValueError("database must already exist") + with sqlite3.connect(database.resolve().as_uri() + ("?mode=rw" if apply else "?mode=ro"), uri=True) as db: + if not db.execute("pragma table_info(gallery)").fetchall(): + raise ValueError("required table missing: gallery") + duplicates = db.execute("select count(*) from (select gid from gallery group by gid having count(*)>1)").fetchone()[0] + if duplicates: + raise ValueError("duplicate gallery gid; reconcile before migration") + present = bool(db.execute("pragma table_info(task_operation)").fetchall()) + result = {"apply": apply, "create_operation_table": not present} + if not apply: + return result + stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + backup = database.with_name(database.name + ".before-task-v2." + stamp) + with sqlite3.connect(backup) as copy: + db.backup(copy) + backup.chmod(0o600) + sql = Path(__file__).with_name("migrate_task_operation_v2.sql").read_text() + try: + db.executescript("BEGIN IMMEDIATE;\n" + sql + "\nCOMMIT;") + except BaseException: + db.rollback() + raise + result["backup"] = str(backup) + result["operation_rows"] = db.execute("select count(*) from task_operation").fetchone()[0] + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("database", type=Path) + parser.add_argument("--apply", action="store_true") + arguments = parser.parse_args() + print(json.dumps(migrate(arguments.database, arguments.apply), ensure_ascii=False)) diff --git a/scripts/migrate_user_links.py b/scripts/migrate_user_links.py new file mode 100644 index 0000000..6485c46 --- /dev/null +++ b/scripts/migrate_user_links.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Add stable user identity to subscription bindings and access records. + +The default invocation only reports schema changes. --apply creates a consistent +SQLite backup before the transaction. Run against an isolated database first. +""" +import argparse +import datetime +import json +from pathlib import Path +import sqlite3 + + +def migrate(db_path: Path, apply: bool = False) -> dict: + if not db_path.is_file(): + raise ValueError("database must already exist") + uri = db_path.resolve().as_uri() + ("?mode=rw" if apply else "?mode=ro") + with sqlite3.connect(uri, uri=True) as db: + changes = [] + for table in ("sub_bind", "sub_update_record"): + columns = {row[1] for row in db.execute(f'PRAGMA table_info("{table}")')} + if not columns: + raise ValueError(f"required table missing: {table}") + if "user_id" not in columns: + changes.append(f"add {table}.user_id") + duplicates = db.execute( + "select count(*) from (select username from User group by username having count(*) > 1)" + ).fetchone()[0] + if duplicates: + raise ValueError("ambiguous duplicate usernames; resolve identities before migration") + result = {"apply": apply, "schema_changes": changes} + if not apply: + return result + stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + backup = db_path.with_name(db_path.name + ".before-user-links." + stamp) + with sqlite3.connect(backup) as snapshot: + db.backup(snapshot) + backup.chmod(0o600) + db.execute("BEGIN IMMEDIATE") + try: + for change in changes: + table = change.split()[1].split(".")[0] + db.execute(f'ALTER TABLE "{table}" ADD COLUMN user_id INTEGER REFERENCES User(id)') + for table in ("sub_bind", "sub_update_record"): + db.execute(f'UPDATE "{table}" SET user_id=(SELECT id FROM User WHERE username="{table}".user) WHERE user_id IS NULL') + db.execute("CREATE UNIQUE INDEX IF NOT EXISTS User_username_uindex ON User(username)") + db.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_sub_bind_user_id ON sub_bind(user_id) WHERE user_id IS NOT NULL") + db.execute("CREATE INDEX IF NOT EXISTS idx_sub_record_user_id ON sub_update_record(user_id, id)") + db.commit() + except BaseException: + db.rollback() + raise + result["backup"] = str(backup) + result["unresolved_bindings"] = db.execute("select count(*) from sub_bind where user_id is null").fetchone()[0] + result["unresolved_records"] = db.execute("select count(*) from sub_update_record where user_id is null").fetchone()[0] + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("database", type=Path) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + print(json.dumps(migrate(args.database, args.apply), ensure_ascii=False)) diff --git a/scripts/test_user_links.py b/scripts/test_user_links.py new file mode 100644 index 0000000..5e1deef --- /dev/null +++ b/scripts/test_user_links.py @@ -0,0 +1,55 @@ +import importlib.util +from pathlib import Path +import sqlite3 +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("migration", Path(__file__).with_name("migrate_user_links.py")) +migration = importlib.util.module_from_spec(spec) +spec.loader.exec_module(migration) + + +class MigrationTest(unittest.TestCase): + def test_dry_run_idempotent_migration_and_rename(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "test.db" + with sqlite3.connect(path) as db: + db.executescript(""" + CREATE TABLE User(id INTEGER PRIMARY KEY, username TEXT); + CREATE TABLE sub_bind(key TEXT PRIMARY KEY, user TEXT, subscription_account_id INTEGER); + CREATE TABLE sub_update_record(id INTEGER PRIMARY KEY, user TEXT, location TEXT); + INSERT INTO User VALUES(1,'old-name'); + INSERT INTO sub_bind VALUES('example-key','old-name',2); + INSERT INTO sub_update_record VALUES(10,'old-name','unknown'); + INSERT INTO sub_update_record VALUES(11,'old-name','unknown'); + """) + before = path.read_bytes() + self.assertEqual(2, len(migration.migrate(path)["schema_changes"])) + self.assertEqual(before, path.read_bytes()) + self.assertEqual(0, migration.migrate(path, True)["unresolved_bindings"]) + self.assertEqual([], migration.migrate(path, True)["schema_changes"]) + with sqlite3.connect(path) as db: + db.execute("update User set username='new-name' where id=1") + self.assertEqual((1,), db.execute("select user_id from sub_bind").fetchone()) + # A slow first lookup must not overwrite the second request's row. + db.execute("update sub_update_record set location='first-location' where id=10") + self.assertEqual(("unknown",), db.execute("select location from sub_update_record where id=11").fetchone()) + + def test_ambiguous_names_fail_without_modifying_database(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "test.db" + with sqlite3.connect(path) as db: + db.executescript(""" + CREATE TABLE User(id INTEGER PRIMARY KEY, username TEXT); + CREATE TABLE sub_bind(user TEXT); + CREATE TABLE sub_update_record(user TEXT); + INSERT INTO User VALUES(1,'same'),(2,'same'); + """) + before = path.read_bytes() + with self.assertRaises(ValueError): + migration.migrate(path, True) + self.assertEqual(before, path.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/main/java/com/lion/lionwebsite/Configuration/CustomBean.java b/src/main/java/com/lion/lionwebsite/Configuration/CustomBean.java index c227574..e47e8cf 100644 --- a/src/main/java/com/lion/lionwebsite/Configuration/CustomBean.java +++ b/src/main/java/com/lion/lionwebsite/Configuration/CustomBean.java @@ -27,6 +27,9 @@ import java.util.Random; Chat.class, MessageEntity.class, AbstractMethodError.class, DeleteGalleryMessage.class, DownloadPostMessage.class, DownloadStatusMessage.class, IdentityMessage.class, MaintainMessage.class, ResponseMessage.class, AvailableCheckMessage.class, + // 协议 v2 的消息与任务操作实体：原生镜像下 Jackson/MyBatis 都要反射构造它们。 + ProtocolHelloMessage.class, TaskCommandMessage.class, TaskReportMessage.class, TaskReportAckMessage.class, + TaskOperation.class, SubscriptionSnapshotMessage.class, SubscriptionSnapshotPayload.class, SubscriptionAccountSnapshot.class, SubscriptionBindingSnapshot.class, LinkPreviewOptions.class}) public class CustomBean { diff --git a/src/main/java/com/lion/lionwebsite/Configuration/GalleryIndexConfiguration.java b/src/main/java/com/lion/lionwebsite/Configuration/GalleryIndexConfiguration.java new file mode 100644 index 0000000..4e08d76 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Configuration/GalleryIndexConfiguration.java @@ -0,0 +1,83 @@ +package com.lion.lionwebsite.Configuration; + +import com.lion.lionwebsite.Dao.cache.ImageCacheMapper; +import com.lion.lionwebsite.Domain.ImageKeyCache; +import com.lion.lionwebsite.Service.GalleryIndexService; +import com.lion.lionwebsite.Util.GalleryUtil; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.sql.DataSource; +import java.util.List; + +/** + * 图片索引服务的事务装配。 + * + *

单独成文件是为了不触碰既有的 SqlConfiguration：那里的两个 SqlSessionFactory + * 保持原样，这里只额外声明缓存库的事务管理器与模板。 + * + *

事务管理器必须绑定 datasource-cache —— 索引只写缓存库， + * 若误用主库事务管理器，整组替换就失去原子性（删除与写入落在不同连接上）。 + */ +@Configuration +public class GalleryIndexConfiguration { + + @Bean + public PlatformTransactionManager cacheTransactionManager( + @Qualifier("datasource-cache") DataSource cacheDataSource) { + return new DataSourceTransactionManager(cacheDataSource); + } + + /** 只用于图片索引的短事务发布，超时留给上层请求超时控制，这里显式设为 30 秒。 */ + @Bean + public TransactionTemplate cacheTransactionTemplate( + @Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) { + TransactionTemplate template = new TransactionTemplate(cacheTransactionManager); + template.setTimeout(30); + return template; + } + + /** + * 只读快照事务，用于一次性读取头部与页码。 + * + *

读事务不写库，因此只读连接可以与写入并发；它的作用是让「读头部 + 读页码」 + * 落在同一个快照上，避免恰好读到整组替换的中间状态。 + */ + @Bean + public TransactionTemplate cacheReadTransactionTemplate( + @Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) { + TransactionTemplate template = new TransactionTemplate(cacheTransactionManager); + template.setTimeout(10); + return template; + } + + /** + * 这里显式不设置 {@code readOnly}：sqlite-jdbc 不允许在已建立的连接上改只读标志 + * （实测抛 SQLException: Cannot change read-only flag after establishing a connection）， + * 而事务模板拿到的是连接池里的既有连接，设置只读会让每次读取都直接失败。 + * 快照一致性由「同一个事务」保证，不需要额外的只读标志。 + */ + + @Bean + public GalleryIndexService galleryIndexService( + ImageCacheMapper imageCacheMapper, + @Qualifier("cacheTransactionTemplate") TransactionTemplate cacheTransactionTemplate, + @Qualifier("cacheReadTransactionTemplate") TransactionTemplate cacheReadTransactionTemplate) { + return new GalleryIndexService(imageCacheMapper, cacheTransactionTemplate, + cacheReadTransactionTemplate, + GalleryIndexConfiguration::fetchPageKeys); + } + + /** + * 生产环境的上游实现：沿用 GalleryUtil.parseImageKeys（HTTP + JS 解析）。 + * + *

放在这里而不是服务内部，是为了让服务在测试中可以脱离静态工具类。 + */ + static List fetchPageKeys(String url) throws Exception { + return GalleryUtil.parseImageKeys(url); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Configuration/SqlConfiguration.java b/src/main/java/com/lion/lionwebsite/Configuration/SqlConfiguration.java index 831ec50..e5c5297 100644 --- a/src/main/java/com/lion/lionwebsite/Configuration/SqlConfiguration.java +++ b/src/main/java/com/lion/lionwebsite/Configuration/SqlConfiguration.java @@ -8,6 +8,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import com.zaxxer.hikari.HikariDataSource; import javax.sql.DataSource; @@ -16,7 +21,9 @@ public class SqlConfiguration { @Bean(name = "datasource-main") @ConfigurationProperties(prefix = "spring.datasource-main") public DataSource dataSourceMain() { - return DataSourceBuilder.create().build(); + DataSource dataSource = DataSourceBuilder.create().build(); + applyBusyTimeout(dataSource); + return dataSource; } @Bean(name = "datasource-cache") @@ -25,6 +32,18 @@ public class SqlConfiguration { return DataSourceBuilder.create().build(); } + /** + * 给主库连接打开 busy 等待。 + * + *

主库是单文件 SQLite，任务生命周期会并发开启短事务（创建、状态回执、删除）。 + * 默认没有 busy 超时，第二个写事务会立刻拿到 SQLITE_BUSY 并被当作 500； + * 打开等待后，写冲突会按序排队重试而不是失败。 + */ + private void applyBusyTimeout(DataSource dataSource) { + if (dataSource instanceof HikariDataSource hikari) + hikari.setConnectionInitSql("PRAGMA busy_timeout=5000"); + } + @Bean(name = "sqlSessionFactory-main") public SqlSessionFactory sqlSessionFactoryMain(@Qualifier("datasource-main") DataSource dataSource) throws Exception { SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean(); @@ -48,4 +67,30 @@ public class SqlConfiguration { public SqlSessionTemplate sqlSessionTemplateCache(@Qualifier("sqlSessionFactory-cache") SqlSessionFactory sqlSessionFactory) { return new SqlSessionTemplate(sqlSessionFactory); } + + /** + * 主库事务管理器。 + * + *

必须显式绑定到 datasource-main：应用里有两个 DataSource，Spring Boot 的自动配置 + * 会退避，若没有这个 bean 就没有任何事务管理器，@Transactional 会静默失效。 + * @Primary 同时让「不指定名字就注入 TransactionManager」的旧代码继续可用。 + */ + @Bean(name = "mainTransactionManager") + @Primary + public DataSourceTransactionManager mainTransactionManager(@Qualifier("datasource-main") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + /** + * 主库编程式事务模板。 + * + *

任务生命周期需要把「短事务」放在网络调用之外，并且要求「回调里捕获异常后不再抛出」 + * 不能错误提交，因此调用方必须用 {@code execute} 的返回状态或再次抛出决定回滚， + * 而不是在回调里吞掉异常（见 GalleryTaskLifecycleService）。 + */ + @Bean(name = "mainTransactionTemplate") + @Primary + public TransactionTemplate mainTransactionTemplate(@Qualifier("mainTransactionManager") DataSourceTransactionManager transactionManager) { + return new TransactionTemplate(transactionManager); + } } diff --git a/src/main/java/com/lion/lionwebsite/Configuration/TaskLifecycleWiring.java b/src/main/java/com/lion/lionwebsite/Configuration/TaskLifecycleWiring.java new file mode 100644 index 0000000..e147cfd --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Configuration/TaskLifecycleWiring.java @@ -0,0 +1,57 @@ +package com.lion.lionwebsite.Configuration; + +import com.lion.lionwebsite.Domain.TaskOperation; +import com.lion.lionwebsite.Service.GalleryTaskLifecycleService; +import com.lion.lionwebsite.Service.RemoteService; +import com.lion.lionwebsite.Service.TaskReportHandler; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +/** + * 装配协议 v2 的双向依赖，并做启动恢复。 + * + *

RemoteService 消费上报时需要生命周期服务，而生命周期服务下发命令时需要 RemoteService。 + * 直接互相注入构造参数会形成环，所以这里在两端都构造完成之后用 setter 接上； + * 同时也把「启动时把中断的 SUBMITTING 判为 UNKNOWN、释放 RESERVED 占位」这一步接到应用就绪事件上。 + * + *

顺序是不可交换的： + *

    + *
  1. 先绑定上报处理器、下发判定解析器与 operationId 解析器；
    2. + *
  2. 再执行启动恢复（SUBMITTING → UNKNOWN、释放 RESERVED）；
    3. + *
  3. 最后才允许建立连接与自动重投。连接由本类显式触发，不再由 RemoteService 的 + * PostConstruct 在装配前发起，否则会把状态不明的任务盲目重投给节点。
    4. + *
+ * 恢复失败时仍然允许连接（保持节点在线/心跳），但明确禁止自动派发。 + */ +@Component +@Slf4j +@RequiredArgsConstructor +public class TaskLifecycleWiring { + + final RemoteService remoteService; + + final GalleryTaskLifecycleService lifecycleService; + + @EventListener(ApplicationReadyEvent.class) + public void wire() { + TaskReportHandler handler = lifecycleService; + remoteService.setTaskReportHandler(handler); + remoteService.setDownloadOperationIdResolver(lifecycleService::currentOperationId); + remoteService.setDispatchResolver(lifecycleService::resolveDispatch); + boolean recoverySafe = false; + try { + int recovered = lifecycleService.recoverInterruptedSubmissions(); + recoverySafe = true; + if (recovered > 0) + log.warn("启动恢复：{} 个中断的创建/预占任务已处理（提交中的标为提交未知）", recovered); + } catch (RuntimeException e) { + // 恢复失败说明库中任务真实状态未知：可以保持连接，但绝不能自动重投/派发。 + log.error("启动恢复失败，本次启动将拒绝自动重投未完成任务", e); + } + // 绑定与恢复完成之后才连接：连接成功后的自动重投受 lifecycleReady/recoverySafe 双重约束。 + remoteService.startAfterWiring(recoverySafe); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Controller/PublicController.java b/src/main/java/com/lion/lionwebsite/Controller/PublicController.java index 502b813..f16378a 100644 --- a/src/main/java/com/lion/lionwebsite/Controller/PublicController.java +++ b/src/main/java/com/lion/lionwebsite/Controller/PublicController.java @@ -62,16 +62,14 @@ public class PublicController { public String validate(String AuthCode){ Response response = Response.generateResponse(); User user = publicService.getUserId(AuthCode); - // 用 ObjectMapper 组装内层 JSON：手工 String.format 拼用户名时， - // 名字里带引号或反斜杠会直接产出非法 JSON，前端 JSON.parse 随即失败。 - // 这里仍以「JSON 文本」形式放进 data（历史契约，前端按字符串再解析一次）。 + // 身份字段直接作为结构化 data 返回；共享前端解码器兼容旧版本字符串。 ObjectNode identity = CustomUtil.objectMapper.createObjectNode(); identity.put("userId", user.getId()); identity.put("username", user.getUsername()); identity.put("isAvailable", !remoteService.isDead()); // 管理员标记随登录一起下发，前端据此决定是否显示下载人信息与筛选。 identity.put("isAdmin", user.getId() == UserService.ADMIN_USER_ID); - response.success(identity.toString()); + response.success(identity); return response.toJSONString(); } diff --git a/src/main/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapper.java b/src/main/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapper.java index 3130885..52e8b50 100644 --- a/src/main/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapper.java +++ b/src/main/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapper.java @@ -8,6 +8,8 @@ import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Select; +import java.util.List; + @Mapper public interface ImageCacheMapper { @Insert("insert into ImageKeyCache values (#{gid}, #{imgkey}, #{page})") @@ -31,4 +33,32 @@ public interface ImageCacheMapper { @Delete("delete from gidToKey where gid=#{gid}") void deleteGidToKey(String gid); + + /** + * 某个 gid 已落库的页码列表（升序）。 + * + *

只按行数判断完整性会把「页码 1 和 3、共 2 行」当成完整索引， + * 因此完整性判定改为按页码集合比对——先取回页码，再校验它是否恰好是 1..pages。 + */ + @Select("select page from ImageKeyCache where gid=#{gid} order by page") + List selectPageNumbersByGid(String gid); + + /** 该 gid 下 imgkey 为空的脏行数；任何一行脏行都说明索引不可用。 */ + @Select("select count(*) from ImageKeyCache where gid=#{gid} and (imgkey is null or trim(imgkey)='')") + int countInvalidImageKeysByGid(String gid); + + /** + * 批量写入页 key（单条多值 INSERT）。 + * + *

整本画廊的索引可能上千页，逐行 INSERT 会拉长写事务并放大锁竞争； + * 调用方按固定大小分片后，一个写事务里只需少量语句即可写完。 + */ + // MyBatis 只在注解值「以 ") + int insertImageKeyCacheBatch(@Param("pages") List pages); } diff --git a/src/main/java/com/lion/lionwebsite/Dao/normal/CollectMapper.java b/src/main/java/com/lion/lionwebsite/Dao/normal/CollectMapper.java index 2d565e2..abac0c9 100644 --- a/src/main/java/com/lion/lionwebsite/Dao/normal/CollectMapper.java +++ b/src/main/java/com/lion/lionwebsite/Dao/normal/CollectMapper.java @@ -20,4 +20,8 @@ public interface CollectMapper { @Select("select collector from collect where gid=#{gid}") ArrayList selectCollectorByGid(int gid); + + /** 任务删除完成后清理该 gid 的全部收藏记录（与删任务行同一事务）。 */ + @Delete("delete from collect where gid=#{gid}") + void deleteByGid(@Param("gid") int gid); } diff --git a/src/main/java/com/lion/lionwebsite/Dao/normal/GalleryMapper.java b/src/main/java/com/lion/lionwebsite/Dao/normal/GalleryMapper.java index fce56c9..1b7575d 100644 --- a/src/main/java/com/lion/lionwebsite/Dao/normal/GalleryMapper.java +++ b/src/main/java/com/lion/lionwebsite/Dao/normal/GalleryMapper.java @@ -36,9 +36,10 @@ public interface GalleryMapper { @Update(""" - update gallery set name=#{name}, link=#{link}, language=#{language}, + update gallery set name=#{name}, link=#{link}, language=#{language}, downloader=#{downloader}, pages=#{pages}, status=#{status}, fileSize=#{fileSize}, createTime=#{createTime}, - proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize}, is_download=#{is_download} + proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize}, + thumb_link=#{thumb_link}, is_download=#{is_download} where gid=#{gid}""") void updateGallery(Gallery gallery); @@ -47,4 +48,8 @@ public interface GalleryMapper { @Delete("delete from gallery where gid=#{gid}") void deleteGalleryByGid(Integer gid); + + /** 只改状态列，用于「待删除」这类不牵动其他字段的意图标记。 */ + @Update("update gallery set status=#{status} where gid=#{gid}") + int updateGalleryStatus(@Param("gid") int gid, @Param("status") String status); } diff --git a/src/main/java/com/lion/lionwebsite/Dao/normal/SubMapper.java b/src/main/java/com/lion/lionwebsite/Dao/normal/SubMapper.java index 0ee524f..6c19ba2 100644 --- a/src/main/java/com/lion/lionwebsite/Dao/normal/SubMapper.java +++ b/src/main/java/com/lion/lionwebsite/Dao/normal/SubMapper.java @@ -43,49 +43,57 @@ public interface SubMapper { @Delete("delete from subscription_account where id=#{id}") void deleteSubscriptionAccount(Integer id); - @Insert("insert into sub_bind (key, user, subscription_account_id) values (#{key}, #{user}, #{subscriptionAccountId})") + @Insert("insert into sub_bind (key, user, subscription_account_id, user_id) values (#{key}, #{user}, #{subscriptionAccountId}, (select id from User where username=#{user}))") void insertSubBind(SubBind subBind); - @Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user") + @Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user") ArrayList selectAllSubBind(); - @Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}") + @Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}") SubBind selectSubBind(String key); @Select("select count(key) from sub_bind where key=#{key}") boolean selectSubBindExist(String key); - @Select("select count(*) from sub_bind where user=#{user}") + @Select("select count(*) from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") int countSubBindByUser(String user); - @Update("update sub_bind set subscription_account_id=#{accountId} where user=#{user}") + @Update("update sub_bind set subscription_account_id=#{accountId} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") int updateSubBindAccount(@Param("user") String user, @Param("accountId") Integer accountId); - @Update("update sub_bind set key=#{key} where user=#{user}") + @Update("update sub_bind set key=#{key} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") int updateSubBindKey(@Param("user") String user, @Param("key") String key); - @Delete("delete from sub_bind where user=#{user}") + @Delete("delete from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") void deleteSubBind(String user); - @Insert("insert into sub_update_record (user, ip, UA, time, location) values (#{user}, #{ip}, #{UA}, #{time}, #{location})") + @Insert("insert into sub_update_record (user, ip, UA, time, location, user_id) values (#{user}, #{ip}, #{UA}, #{time}, #{location}, coalesce(#{userId}, (select id from User where username=#{user})))") + @Options(useGeneratedKeys = true, keyProperty = "id") void insertSubUpdateRecord(SubUpdateRecord subUpdateRecord); - @Select("select * from sub_update_record order by time desc") + @Select("select id,user,ip,UA,time,location,user_id as userId from sub_update_record order by time desc") ArrayList selectAllSubUpdateRecord(); - @Delete("delete from sub_update_record where user=#{user}") + @Delete("delete from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") void deleteSubUpdateRecord(String user); - @Select("select count(user) from sub_update_record where user=#{user}") + @Select("select count(user) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") Integer selectUpdateRecordCount(String user); - @Select("select min(id) from sub_update_record where user=#{user}") + @Select("select min(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") Integer selectMinUpdateRecordId(String user); /** 后台补齐归属地：只更新该用户最近一条记录，避免覆盖更早的历史记录。 */ - @Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user=#{user})") + @Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user}))") void updateLatestSubUpdateRecordLocation(@Param("user") String user, @Param("location") String location); + @Update("update sub_update_record set location=#{location} where id=#{id}") + void updateSubUpdateRecordLocation(@Param("id") int id, @Param("location") String location); + + @Update("update sub_bind set user=#{newName} where user_id=#{userId} or (user_id is null and user=#{oldName})") + void renameBoundUser(@Param("userId") int userId, @Param("oldName") String oldName, + @Param("newName") String newName); + @Delete("delete from sub_update_record where id=#{id}") void deleteSubUpdateRecordById(int id); } diff --git a/src/main/java/com/lion/lionwebsite/Dao/normal/TaskOperationRepository.java b/src/main/java/com/lion/lionwebsite/Dao/normal/TaskOperationRepository.java new file mode 100644 index 0000000..764aaee --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Dao/normal/TaskOperationRepository.java @@ -0,0 +1,107 @@ +package com.lion.lionwebsite.Dao.normal; + +import com.lion.lionwebsite.Domain.TaskOperation; +import org.apache.ibatis.annotations.*; + +/** + * 任务操作记录的持久化入口（主库）。 + * + *

所有写操作都在调用方的事务里执行；这里的注解只描述 SQL，不包含业务判断。 + * 主键为 gid，保证「同一 gid 只能有一条有效操作意图」。 + */ +@Mapper +public interface TaskOperationRepository { + + @Select("select * from task_operation where gid=#{gid}") + TaskOperation selectByGid(int gid); + + /** 生成该 gid 的稳定下载 operationId：只有还没有记录时才写入。 */ + @Insert("insert or ignore into task_operation (gid, operationId, deleteOperationId, state, lastRevision," + + " lastReportId, updateTime) values (#{gid}, #{operationId}, null, #{state}, 0, null, #{updateTime})") + int insertIfAbsent(@Param("gid") int gid, @Param("operationId") String operationId, + @Param("state") String state, @Param("updateTime") long updateTime); + + @Insert("insert into task_operation (gid, operationId, deleteOperationId, state, lastRevision, lastReportId, updateTime)" + + " values (#{gid}, #{operationId}, #{deleteOperationId}, #{state}, #{lastRevision}, #{lastReportId}, #{updateTime})") + void insert(TaskOperation operation); + + /** 推进创建意图的状态（RESERVED → SUBMITTING → SUBMITTED/UNKNOWN）。 */ + @Update("update task_operation set state=#{state}, updateTime=#{updateTime} where gid=#{gid}") + int updateState(@Param("gid") int gid, @Param("state") String state, @Param("updateTime") long updateTime); + + /** + * 带前置条件的创建状态推进：只有当前状态与 operationId 都匹配时才写入。 + * + *

这是「提交前先确认这一代意图仍然有效」的 CAS：若期间被删除/重建， + * 返回值 0，调用方必须停下，绝不能再产生上游副作用。 + */ + @Update("update task_operation set state=#{state}, updateTime=#{updateTime}" + + " where gid=#{gid} and operationId=#{operationId} and state=#{expectedState}") + int compareAndSetCreateState(@Param("gid") int gid, @Param("operationId") String operationId, + @Param("expectedState") String expectedState, @Param("state") String state, + @Param("updateTime") long updateTime); + + /** + * 只按当前删除意图 ID 匹配的删除完成：期间若已被删除/重建，则不改动新世代记录。 + */ + @Update("update task_operation set deleteOperationId=null, state=#{state}, updateTime=#{updateTime}" + + " where gid=#{gid} and deleteOperationId=#{deleteOperationId}") + int completeDeleteIfSameGeneration(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId, + @Param("state") String state, @Param("updateTime") long updateTime); + + /** + * 删除「本次创建意图」的预占记录：必须同时匹配 gid 与 operationId，且状态仍是 + * RESERVED/SUBMITTING。 + * + *

三重匹配保证释放流程只能清掉自己那一代：期间被删除/重建（新 operationId 或墓碑） + * 的记录一律不动，避免把新世代抹掉。 + */ + @Delete("delete from task_operation where gid=#{gid} and operationId=#{operationId}" + + " and state in ('RESERVED','SUBMITTING')") + int deleteReservationIfSameGeneration(@Param("gid") int gid, @Param("operationId") String operationId); + + /** 记录/替换当前下载 operationId 并推进状态。 */ + @Update("update task_operation set operationId=#{operationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}") + int updateOperationId(@Param("gid") int gid, @Param("operationId") String operationId, + @Param("state") String state, @Param("updateTime") long updateTime); + + /** + * 给「有记录但缺 operationId」的历史任务补一个稳定 operationId。 + * 只在确实缺失时写入，避免覆盖并发创建出来的新世代。 + */ + @Update("update task_operation set operationId=#{operationId}, updateTime=#{updateTime}" + + " where gid=#{gid} and operationId is null") + int assignOperationIdIfMissing(@Param("gid") int gid, @Param("operationId") String operationId, + @Param("updateTime") long updateTime); + + /** + * 墓碑复活：删除完成后的 gid 允许用全新 operationId 重新创建。 + * revision 与回执一并清零，旧操作的迟到上报会因 operationId 不匹配被丢弃。 + */ + @Update("update task_operation set operationId=#{operationId}, deleteOperationId=null, state=#{state}," + + " lastRevision=0, lastReportId=null, updateTime=#{updateTime} where gid=#{gid}") + int reactivate(@Param("gid") int gid, @Param("operationId") String operationId, + @Param("state") String state, @Param("updateTime") long updateTime); + + /** 记录删除意图与其稳定的删除操作 ID（重试必须复用同一个 ID）。 */ + @Update("update task_operation set deleteOperationId=#{deleteOperationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}") + int updateDeleteIntent(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId, + @Param("state") String state, @Param("updateTime") long updateTime); + + /** + * 写入状态上报回执（revision + reportId）。 + * + *

回执与任务行更新在同一个事务里提交，提交成功之后才回 ACK； + * 因此节点重复投递同一份上报时，只能看到已经提交过的回执，不会重复落库或重复通知。 + */ + @Update("update task_operation set lastRevision=#{revision}, lastReportId=#{reportId}, updateTime=#{updateTime}" + + " where gid=#{gid}") + int updateReportReceipt(@Param("gid") int gid, @Param("revision") long revision, + @Param("reportId") String reportId, @Param("updateTime") long updateTime); + + @Select("select * from task_operation where state=#{state}") + java.util.List selectByState(String state); + + @Delete("delete from task_operation where gid=#{gid}") + int deleteByGid(int gid); +} diff --git a/src/main/java/com/lion/lionwebsite/Domain/SubBind.java b/src/main/java/com/lion/lionwebsite/Domain/SubBind.java index 2f1f42e..c640f5e 100644 --- a/src/main/java/com/lion/lionwebsite/Domain/SubBind.java +++ b/src/main/java/com/lion/lionwebsite/Domain/SubBind.java @@ -14,4 +14,10 @@ public class SubBind { String subscriptionAccountName; boolean subscriptionAccountEnabled; boolean filterHighMultiplier; + Integer userId; + + public SubBind(String key, String user, Integer accountId, String accountName, + boolean accountEnabled, boolean filterHighMultiplier) { + this(key, user, accountId, accountName, accountEnabled, filterHighMultiplier, null); + } } diff --git a/src/main/java/com/lion/lionwebsite/Domain/SubUpdateRecord.java b/src/main/java/com/lion/lionwebsite/Domain/SubUpdateRecord.java index dbea5ad..0e9bbfd 100644 --- a/src/main/java/com/lion/lionwebsite/Domain/SubUpdateRecord.java +++ b/src/main/java/com/lion/lionwebsite/Domain/SubUpdateRecord.java @@ -16,4 +16,9 @@ public class SubUpdateRecord { String UA; Date time; String location; + Integer userId; + + public SubUpdateRecord(int id, String user, String ip, String UA, Date time, String location) { + this(id, user, ip, UA, time, location, null); + } } diff --git a/src/main/java/com/lion/lionwebsite/Domain/TaskOperation.java b/src/main/java/com/lion/lionwebsite/Domain/TaskOperation.java new file mode 100644 index 0000000..c760e8d --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Domain/TaskOperation.java @@ -0,0 +1,53 @@ +package com.lion.lionwebsite.Domain; + +import lombok.Data; + +/** + * 单个 gid 的下载生命周期操作记录（协议 v2 的持久化意图）。 + * + *

这张表是「先持久化意图、再产生外部副作用」的落点： + *

    + *
  • CREATE 意图先于上游 POST 落库，用来保证同一 gid 只有一个未完成意图；
    • + *
  • 提交结果不确定时只能停在 {@link #STATE_UNKNOWN}，绝不自动重复上游 POST；
    • + *
  • DELETE 意图先于节点删除落库，节点确认后（或文件已不存在）才在同一事务里删任务行并留墓碑。
    • + *
+ * + *

旧任务行没有对应操作记录，查询时按 gid 惰性补建（见 GalleryTaskLifecycleService）。 + */ +@Data +public class TaskOperation { + + /** 已登记创建意图，尚未向上游提交。 */ + public static final String STATE_RESERVED = "RESERVED"; + /** 正在向上游提交；此时不可再发第二次 POST。 */ + public static final String STATE_SUBMITTING = "SUBMITTING"; + /** 上游确认已接受下载请求。 */ + public static final String STATE_SUBMITTED = "SUBMITTED"; + /** + * 提交结果未知：超时、断连或在 SUBMITTING 期间重启。 + * 上游不支持幂等，因此这里只能等对账（reconcile），不能自动重发。 + */ + public static final String STATE_UNKNOWN = "UNKNOWN"; + /** 已登记删除意图，等待节点确认。 */ + public static final String STATE_DELETE_PENDING = "DELETE_PENDING"; + /** 节点已确认删除，任务行也随之删除；本行作为墓碑保留，用于停止迟到状态的重放。 */ + public static final String STATE_DELETED = "DELETED"; + + private int gid; + + /** 当前有效的下载操作 ID；同一 gid 的 DOWNLOAD 重试必须复用它。 */ + private String operationId; + + /** 独立于下载操作 ID 的删除操作 ID，删除重试复用它。 */ + private String deleteOperationId; + + private String state; + + /** 最近一次应用的状态上报 revision；用于丢弃乱序/重放的旧上报。 */ + private long lastRevision; + + /** 节点已确认收到的最新上报 ID；用于重复 ACK 的幂等判断。 */ + private String lastReportId; + + private long updateTime; +} diff --git a/src/main/java/com/lion/lionwebsite/Exception/GalleryIndexException.java b/src/main/java/com/lion/lionwebsite/Exception/GalleryIndexException.java new file mode 100644 index 0000000..a15f6a2 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Exception/GalleryIndexException.java @@ -0,0 +1,44 @@ +package com.lion.lionwebsite.Exception; + +/** + * 图片索引构建失败。 + * + *

调用方需要区分三类结果，才能给出正确的用户可见响应： + * 链接本身有问题（{@link Reason#INVALID_LINK}，属于请求错误）、 + * 画廊已不存在（{@link Reason#UNAVAILABLE}，重试无意义）、 + * 上游或数据库暂时不可用（{@link Reason#TRANSIENT}，可以重试）。 + */ +public class GalleryIndexException extends Exception { + + public enum Reason { + /** 链接无法解析出 gid 或 key。 */ + INVALID_LINK, + /** 上游明确表示画廊不存在（下架/删除/解析不到页 key）。 */ + UNAVAILABLE, + /** 上游返回的页 key 无法自洽（gid 不匹配、页码重复/跳号、imgkey 为空）。 */ + MALFORMED, + /** 上游波动或索引写入失败，重试可能成功。 */ + TRANSIENT + } + + private final Reason reason; + + public GalleryIndexException(Reason reason, String message) { + super(message); + this.reason = reason; + } + + public GalleryIndexException(Reason reason, String message, Throwable cause) { + super(message, cause); + this.reason = reason; + } + + public Reason getReason() { + return reason; + } + + /** 重试没有意义（链接错误或画廊不可用）；上游数据畸形与会话级故障仍可重试。 */ + public boolean isPermanent() { + return reason == Reason.INVALID_LINK || reason == Reason.UNAVAILABLE; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Maintenance/GalleryIndexMaintenance.java b/src/main/java/com/lion/lionwebsite/Maintenance/GalleryIndexMaintenance.java new file mode 100644 index 0000000..251e571 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Maintenance/GalleryIndexMaintenance.java @@ -0,0 +1,319 @@ +package com.lion.lionwebsite.Maintenance; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 缓存库图片索引的离线体检与修复工具。 + * + *

它是独立可执行类（只用 JDBC，不依赖 Spring / MyBatis），因此可以在服务停止后 + * 直接对数据库文件运行，也可以先对生产库的副本演练。

+ * + * 
+ *   java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db
+ *   java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db --apply
+ *
+ * + *

它只做两类写入：删除不完整或有冲突的 gid 整组索引（gidToKey 头部加该 gid 的全部 + * 页 key），以及删除没有头部记录的孤儿页 key。被删掉的 gid 由下一次在线阅读重新抓取， + * 因此不需要猜测保留哪一组 key。工具不建表、不改列、不碰其它业务表。

+ */ +public final class GalleryIndexMaintenance { + + /** 生产运行时数据库所在目录；默认拒绝直接写入，必须显式加 --allow-production。 */ + private static final String PRODUCTION_DIR = "/home/lionwebsite"; + + private static final String PLAIN_REPAIR_NOTE = + "这些画廊将在下次被访问时重新抓取。"; + + public static void main(String[] args) { + Map options = parseOptions(args); + if (options.containsKey("help") || !options.containsKey("db")) { + printUsage(); + System.exit(options.containsKey("help") ? 0 : 2); + return; + } + + Path database = Path.of(options.get("db")).toAbsolutePath().normalize(); + boolean apply = options.containsKey("apply"); + if (!Files.isRegularFile(database)) { + System.err.println("数据库不存在: " + database); + System.exit(2); + return; + } + if (apply && isProductionPath(database) && !options.containsKey("allow-production")) { + System.err.println("拒绝写入生产库路径 " + database + + "。请先在副本上演练；确需在本机生产库执行时显式加 --allow-production，" + + "并确保服务已停止。"); + System.exit(3); + return; + } + + try (Connection connection = open(database)) { + Report before = inspect(connection); + System.out.println(before.render(apply ? "修复前" : "体检结果")); + + if (!apply) { + System.out.println(); + System.out.println("dry-run：未做任何写入。确认无误后加 --apply 执行修复。"); + System.exit(before.isHealthy() ? 0 : 1); + return; + } + + if (before.isHealthy()) { + System.out.println(); + System.out.println("索引已健康，无需修复。"); + return; + } + + connection.setAutoCommit(false); + int removed = repair(connection, before.badGids, before.orphanGids); + connection.commit(); + System.out.println(); + System.out.println("已删除 " + removed + " 个 gid 的索引整组，" + PLAIN_REPAIR_NOTE); + + Report after = inspect(connection); + System.out.println(); + System.out.println(after.render("修复后")); + if (!after.isHealthy()) { + System.err.println("修复后仍未达到健康状态，请人工检查。"); + System.exit(4); + } + } catch (SQLException e) { + System.err.println("执行失败: " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(5); + } + } + + private static Connection open(Path database) throws SQLException { + Connection connection = DriverManager.getConnection("jdbc:sqlite:" + database); + try (Statement statement = connection.createStatement()) { + statement.execute("pragma busy_timeout=5000"); + statement.execute("pragma foreign_keys=off"); + } + return connection; + } + + /** 是否位于生产运行时目录；工具据此拒绝在未显式确认时写入生产库。 */ + static boolean isProductionPath(Path database) { + return database.startsWith(Path.of(PRODUCTION_DIR).toAbsolutePath().normalize()); + } + + // ---------- 体检 ---------- + + static Report inspect(Connection connection) throws SQLException { + Report report = new Report(); + report.hasHeaderTable = tableExists(connection, "gidToKey"); + report.hasPageTable = tableExists(connection, "ImageKeyCache"); + if (!report.hasHeaderTable || !report.hasPageTable) + return report; + + report.headerRows = count(connection, "select count(*) from gidToKey"); + report.pageRows = count(connection, "select count(*) from ImageKeyCache"); + report.duplicateHeaderGids = count(connection, + "select count(*) from (select gid from gidToKey group by gid having count(*) > 1)"); + report.duplicatePageGroups = count(connection, + "select count(*) from (select gid, page from ImageKeyCache group by gid, page having count(*) > 1)"); + + try (PreparedStatement statement = connection.prepareStatement(BAD_GID_QUERY); + ResultSet rows = statement.executeQuery()) { + while (rows.next()) { + String gid = rows.getString(1); + report.badGids.add(gid); + report.badReasons.put(gid, describe(rows)); + } + } + + try (PreparedStatement statement = connection.prepareStatement( + "select distinct gid from ImageKeyCache where gid not in (select gid from gidToKey)"); + ResultSet rows = statement.executeQuery()) { + while (rows.next()) + report.orphanGids.add(rows.getString(1)); + } + return report; + } + + /** + * 一个 gid 触发的全部不健康条件集中在一句里检查，避免多次全表扫描。 + * + *

必须按 gid 分组后再判断：重复 gid（同一 gid 有多条头部记录）本身就是要清除的对象， + * 而重复行里的某一条可能恰好与页 key 数一致，逐行判断会把它当成健康记录漏掉。 + */ + private static final String BAD_GID_QUERY = """ + select g.gid as gid, + count(*) as header_rows, + max(g.pages) as pages, + min(g.key) as key, + (select count(*) from ImageKeyCache i where i.gid = g.gid) as page_rows, + (select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) as distinct_pages, + (select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as min_page, + (select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as max_page, + (select count(*) from ImageKeyCache i where i.gid = g.gid + and (i.imgkey is null or trim(i.imgkey) = '')) as dirty_keys + from gidToKey g + group by g.gid + having count(*) > 1 + or max(g.pages) is null or max(g.pages) <= 0 + or min(g.key) is null or trim(min(g.key)) = '' + or (select count(*) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages) + or (select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages) + or (select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> 1 + or (select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages) + or (select count(*) from ImageKeyCache i where i.gid = g.gid + and (i.imgkey is null or trim(i.imgkey) = '')) > 0 + order by g.gid + """; + + private static String describe(ResultSet row) throws SQLException { + List reasons = new ArrayList<>(); + int headerRows = row.getInt("header_rows"); + int pages = row.getInt("pages"); + String key = row.getString("key"); + int pageRows = row.getInt("page_rows"); + int distinctPages = row.getInt("distinct_pages"); + int minPage = row.getInt("min_page"); + int maxPage = row.getInt("max_page"); + int dirtyKeys = row.getInt("dirty_keys"); + if (headerRows > 1) reasons.add("重复头部记录 " + headerRows + " 条"); + if (key == null || key.isBlank()) reasons.add("key 为空"); + if (pages <= 0) reasons.add("pages 非法(" + pages + ")"); + if (pageRows != pages) reasons.add("页 key 行数 " + pageRows + " 与 pages " + pages + " 不符"); + if (distinctPages != pages) reasons.add("页码去重后 " + distinctPages + " 个"); + if (minPage != 1 || maxPage != pages) reasons.add("页码区间 " + minPage + "-" + maxPage); + if (dirtyKeys > 0) reasons.add("imgkey 为空 " + dirtyKeys + " 行"); + return String.join("、", reasons); + } + + // ---------- 修复 ---------- + + /** 删除不健康 gid 的整组索引与孤儿页 key，返回被移除索引的 gid 个数。 */ + static int repair(Connection connection, List badGids, List orphanGids) + throws SQLException { + int touched = 0; + try (PreparedStatement pages = connection.prepareStatement("delete from ImageKeyCache where gid = ?"); + PreparedStatement header = connection.prepareStatement("delete from gidToKey where gid = ?")) { + for (String gid : badGids) { + pages.setString(1, gid); + pages.executeUpdate(); + header.setString(1, gid); + header.executeUpdate(); + touched++; + } + for (String gid : orphanGids) { + pages.setString(1, gid); + if (pages.executeUpdate() > 0) + touched++; + } + } + return touched; + } + + // ---------- 基础设施 ---------- + + private static boolean tableExists(Connection connection, String table) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "select count(*) from sqlite_master where type='table' and name=?")) { + statement.setString(1, table); + try (ResultSet rows = statement.executeQuery()) { + return rows.next() && rows.getInt(1) > 0; + } + } + } + + private static int count(Connection connection, String sql) throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery(sql)) { + return rows.next() ? rows.getInt(1) : 0; + } + } + + private static Map parseOptions(String[] args) { + Map options = new LinkedHashMap<>(); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (!arg.startsWith("--")) + continue; + String name = arg.substring(2); + if (name.equals("apply") || name.equals("help") || name.equals("allow-production")) { + options.put(name, "true"); + continue; + } + if (i + 1 >= args.length) { + System.err.println("缺少参数值: " + arg); + System.exit(2); + } + options.put(name, args[++i]); + } + return options; + } + + private static void printUsage() { + System.out.println(""" + 用法: java -cp GalleryIndexMaintenance.java --db [--apply] [--allow-production] + + --db 必填，缓存库文件 + --apply 执行修复；不加则只做 dry-run 体检 + --allow-production 允许写入 /home/lionwebsite 下的数据库（默认拒绝） + + 退出码: 0 健康或修复成功 / 1 存在待修复项(dry-run) / 2 参数错误 / + 3 拒绝写生产路径 / 4 修复后仍不健康 / 5 执行异常 + """); + } + + private GalleryIndexMaintenance() { } + + /** 一次体检结果。 */ + static final class Report { + boolean hasHeaderTable; + boolean hasPageTable; + int headerRows; + int pageRows; + int duplicateHeaderGids; + int duplicatePageGroups; + final List badGids = new ArrayList<>(); + final List orphanGids = new ArrayList<>(); + final Map badReasons = new LinkedHashMap<>(); + + boolean isHealthy() { + return hasHeaderTable && hasPageTable && badGids.isEmpty() && orphanGids.isEmpty(); + } + + String render(String title) { + StringBuilder out = new StringBuilder(); + out.append("=== 图片索引").append(title).append("（").append(Instant.now()).append("）===\n"); + if (!hasHeaderTable || !hasPageTable) { + out.append("缺少缓存表: gidToKey=").append(hasHeaderTable) + .append(" ImageKeyCache=").append(hasPageTable).append('\n'); + return out.toString(); + } + out.append("gidToKey 行数: ").append(headerRows).append('\n'); + out.append("ImageKeyCache 行数: ").append(pageRows).append('\n'); + out.append("重复 gid 组: ").append(duplicateHeaderGids).append('\n'); + out.append("重复 (gid,page) 组: ").append(duplicatePageGroups).append('\n'); + out.append("不完整或冲突 gid: ").append(badGids.size()).append('\n'); + int limit = 20; + for (int i = 0; i < badGids.size() && i < limit; i++) { + String gid = badGids.get(i); + out.append(" - gid=").append(gid).append(' ') + .append(badReasons.getOrDefault(gid, "")).append('\n'); + } + if (badGids.size() > limit) + out.append(" ... 其余 ").append(badGids.size() - limit).append(" 个略\n"); + out.append("孤儿页 key 的 gid: ").append(orphanGids.size()).append('\n'); + out.append(isHealthy() ? "结论: 健康\n" : "结论: 需要修复\n"); + return out.toString(); + } + } +} diff --git a/src/main/java/com/lion/lionwebsite/Message/AbstractMessage.java b/src/main/java/com/lion/lionwebsite/Message/AbstractMessage.java index a47cc91..eeff19d 100644 --- a/src/main/java/com/lion/lionwebsite/Message/AbstractMessage.java +++ b/src/main/java/com/lion/lionwebsite/Message/AbstractMessage.java @@ -21,6 +21,20 @@ public class AbstractMessage { public static final byte SUBSCRIPTION_SNAPSHOT_MESSAGE = 9; + // ---- 协议 v2（可靠任务）。旧类型 0..9 的字段保持不变，不能新增字段。 ---- + + /** 能力协商：{@code {protocolVersion: 2, features: ["reliable-task-v2"]}}。 */ + public static final byte PROTOCOL_HELLO_MESSAGE = 10; + + /** 带稳定 operationId 的任务命令（DOWNLOAD/DELETE）。 */ + public static final byte TASK_COMMAND_MESSAGE = 11; + + /** 带 reportId + revision 的状态上报，替代旧的无确认上报。 */ + public static final byte TASK_REPORT_MESSAGE = 12; + + /** 主站提交事务成功后的上报回执，节点据此清理快照。 */ + public static final byte TASK_REPORT_ACK_MESSAGE = 13; + public byte messageType; public int messageId; diff --git a/src/main/java/com/lion/lionwebsite/Message/MessageCodec.java b/src/main/java/com/lion/lionwebsite/Message/MessageCodec.java index b0bd3ba..4215970 100644 --- a/src/main/java/com/lion/lionwebsite/Message/MessageCodec.java +++ b/src/main/java/com/lion/lionwebsite/Message/MessageCodec.java @@ -45,6 +45,10 @@ public class MessageCodec extends ByteToMessageCodec { case AbstractMessage.MAINTAIN_MESSAGE -> objectMapper.readValue(metadata, MaintainMessage.class); case AbstractMessage.AVAILABLE_CHECK_MESSAGE -> objectMapper.readValue(metadata, AvailableCheckMessage.class); case AbstractMessage.SUBSCRIPTION_SNAPSHOT_MESSAGE -> objectMapper.readValue(metadata, SubscriptionSnapshotMessage.class); + case AbstractMessage.PROTOCOL_HELLO_MESSAGE -> objectMapper.readValue(metadata, ProtocolHelloMessage.class); + case AbstractMessage.TASK_COMMAND_MESSAGE -> objectMapper.readValue(metadata, TaskCommandMessage.class); + case AbstractMessage.TASK_REPORT_MESSAGE -> objectMapper.readValue(metadata, TaskReportMessage.class); + case AbstractMessage.TASK_REPORT_ACK_MESSAGE -> objectMapper.readValue(metadata, TaskReportAckMessage.class); default -> null; }; diff --git a/src/main/java/com/lion/lionwebsite/Message/ProtocolHelloMessage.java b/src/main/java/com/lion/lionwebsite/Message/ProtocolHelloMessage.java new file mode 100644 index 0000000..4ecc9eb --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Message/ProtocolHelloMessage.java @@ -0,0 +1,30 @@ +package com.lion.lionwebsite.Message; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +/** 协议 v2 能力协商（类型 10）。双方都以对方声明了 version>=2 为前提才启用新消息。 */ +@Data +@NoArgsConstructor +public class ProtocolHelloMessage extends AbstractMessage { + { + messageType = PROTOCOL_HELLO_MESSAGE; + } + + public static final int PROTOCOL_V2 = 2; + + public static final String FEATURE_RELIABLE_TASK_V2 = "reliable-task-v2"; + + private int protocolVersion = PROTOCOL_V2; + + private List features = new ArrayList<>(List.of(FEATURE_RELIABLE_TASK_V2)); + + /** 对端是否声明了 v2 可靠任务能力。 */ + public boolean supportsReliableTasks() { + return protocolVersion >= PROTOCOL_V2 + && features != null && features.contains(FEATURE_RELIABLE_TASK_V2); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Message/TaskCommandMessage.java b/src/main/java/com/lion/lionwebsite/Message/TaskCommandMessage.java new file mode 100644 index 0000000..2210875 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Message/TaskCommandMessage.java @@ -0,0 +1,29 @@ +package com.lion.lionwebsite.Message; + +import lombok.Data; + +/** + * 任务命令（类型 11）。 + * + *

只有确认对端支持 v2 之后才发送。同一逻辑操作在重试/重连后必须复用同一个 + * {@link #operationId}，节点据此返回原结果而不重复执行副作用。 + */ +@Data +public class TaskCommandMessage extends AbstractMessage { + { + messageType = TASK_COMMAND_MESSAGE; + } + + public static final String ACTION_DOWNLOAD = "DOWNLOAD"; + + public static final String ACTION_DELETE = "DELETE"; + + /** 稳定操作 ID：DOWNLOAD 与 DELETE 各自独立。 */ + private String operationId; + + private String action; + + private int gid; + + private String name; +} diff --git a/src/main/java/com/lion/lionwebsite/Message/TaskReportAckMessage.java b/src/main/java/com/lion/lionwebsite/Message/TaskReportAckMessage.java new file mode 100644 index 0000000..11186e3 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Message/TaskReportAckMessage.java @@ -0,0 +1,29 @@ +package com.lion.lionwebsite.Message; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 上报回执（类型 13）。 + * + *

只在任务行与回执已经提交之后才发送；重复上报（同一 reportId）也会得到回执， + * 让节点停止重放。{@link #result} 0 表示已落库。 + */ +@Data +@NoArgsConstructor +public class TaskReportAckMessage extends AbstractMessage { + { + messageType = TASK_REPORT_ACK_MESSAGE; + } + + public static final byte RESULT_SUCCESS = 0; + + private String reportId; + + private byte result; + + public TaskReportAckMessage(String reportId, byte result) { + this.reportId = reportId; + this.result = result; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Message/TaskReportMessage.java b/src/main/java/com/lion/lionwebsite/Message/TaskReportMessage.java new file mode 100644 index 0000000..0ecae55 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Message/TaskReportMessage.java @@ -0,0 +1,32 @@ +package com.lion.lionwebsite.Message; + +import com.lion.lionwebsite.Domain.GalleryTask; +import lombok.Data; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 节点状态上报（类型 12）。 + * + *

相比旧的无确认上报，这里带上单调递增的 {@link #revision} 与唯一 {@link #reportId}： + * 主站据此丢弃乱序/重放的旧快照，并在事务提交后回 {@link TaskReportAckMessage}， + * 节点只有在收到匹配回执后才清理自己的快照。 + */ +@Data +public class TaskReportMessage extends AbstractMessage { + { + messageType = TASK_REPORT_MESSAGE; + } + + /** 本次上报的唯一 ID，重复投递同一份快照时保持不变。 */ + private String reportId; + + /** 节点侧单调递增版本号；主站只接受比已记录更大的 revision。 */ + private long revision; + + private GalleryTask[] galleryTasks; + + /** gid → 产生该状态的下载 operationId，用于丢弃属于旧操作的迟到上报。 */ + private Map operationIds = new LinkedHashMap<>(); +} diff --git a/src/main/java/com/lion/lionwebsite/Service/GalleryIndexService.java b/src/main/java/com/lion/lionwebsite/Service/GalleryIndexService.java new file mode 100644 index 0000000..9a2808b --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/GalleryIndexService.java @@ -0,0 +1,262 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.cache.ImageCacheMapper; +import com.lion.lionwebsite.Domain.GidToKey; +import com.lion.lionwebsite.Domain.ImageKeyCache; +import com.lion.lionwebsite.Exception.GalleryIndexException; +import com.lion.lionwebsite.Util.GalleryUtil; +import com.lion.lionwebsite.Util.SingleFlight; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.support.TransactionOperations; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * 图片索引（gid 到每页 imgkey）的唯一读写入口。 + * + *

这个类替换掉原先「读缓存 / 写缓存 / 回滚」分散在业务服务里、并且逐页 INSERT + * 的写法。三件事在这里被固定下来： + * + *

    + *
  1. 同 gid 合并：并发请求同一本画廊时只有一个线程真的去抓上游， + * 其余线程等待同一个结果（含失败），因此不会出现两个构建者互相删除对方写入的索引。
    2. + *
  2. 抓取在事务外：上游 HTTP 可能耗时十几秒，绝不能占着 SQLite 写事务， + * 否则整个缓存库的写入都会排队。
    3. + *
  3. 整组短事务发布：一个事务里先删该 gid 的旧索引，再批量写入新索引。 + * 失败整体回滚，磁盘上仍保留旧的完整索引；只有写入成功这一个状态是对外可见的。
    4. + *
+ * + *

依赖全部由构造函数注入：mapper 负责落库，operations 负责事务边界， + * upstream 负责把 mpv 页解析成页 key 列表。测试可以直接给假的实现， + * 不需要起 Spring 上下文，也不需要网络。 + */ +public class GalleryIndexService { + + private static final Logger log = LoggerFactory.getLogger(GalleryIndexService.class); + + /** 单条 INSERT 携带的最大行数，避免 SQL 过长。 */ + static final int BATCH_SIZE = 200; + + private final ImageCacheMapper mapper; + private final TransactionOperations operations; + private final TransactionOperations readOperations; + private final GalleryKeyUpstream upstream; + private final SingleFlight builds = new SingleFlight<>(); + + public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations, + GalleryKeyUpstream upstream) { + this(mapper, operations, operations, upstream); + } + + /** + * @param operations 发布索引的写事务（删除加批量写入必须整体生效） + * @param readOperations 读取索引的一致性快照事务；传 null 表示不额外开启读事务 + */ + public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations, + TransactionOperations readOperations, GalleryKeyUpstream upstream) { + if (mapper == null || operations == null || upstream == null) + throw new IllegalArgumentException("mapper/operations/upstream 不能为空"); + this.mapper = mapper; + this.operations = operations; + this.readOperations = readOperations; + this.upstream = upstream; + } + + /** + * 保证该画廊的索引可用，返回 gid 到 key 的映射。 + * + * @param url 画廊链接，形如 https://exhentai.org/g/1596929/f08534d87d/ + * @throws GalleryIndexException 链接错误 / 画廊不可用 / 暂时失败，见 GalleryIndexException.Reason + */ + public GidToKey ensureIndex(String url) throws GalleryIndexException { + return ensureIndex(url, this::buildAndPublish); + } + + /** + * 同 ensureIndex，但允许调用方提供构建动作。 + * + *

离线重建需要「先检查再决定是否重建」的不同策略，因此把构建步骤作为参数暴露出来； + * 生产调用使用默认实现即可。 + */ + GidToKey ensureIndex(String url, IndexBuilder builder) throws GalleryIndexException { + GalleryLink link = GalleryLink.parse(url); + GidToKey cached = lookup(link); + if (cached != null) + return cached; + try { + return builds.run(link.gid(), () -> { + // 进入临界区后必须再看一眼：等待期间别的线程可能已经把索引建好了。 + GidToKey existing = lookup(link); + return existing != null ? existing : builder.build(url); + }); + } catch (GalleryIndexException e) { + throw e; + } catch (Exception e) { + throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT, + "构建图片索引失败 gid=" + link.gid(), e); + } + } + + /** + * 已缓存且完整则返回索引，否则返回 null。 + * + *

完整性由页码集合判定，而不是行数：历史重复行与半截写入都会让行数与页数不符。 + */ + GidToKey lookup(GalleryLink link) { + if (readOperations != null) + return readOperations.execute(status -> lookupInTransaction(link)); + return lookupInTransaction(link); + } + + /** + * 在同一个读事务里读头部与页码，得到一个一致的快照。 + * + *

两个独立的 SELECT 之间可能夹着另一个线程的整组替换，于是头部是旧的、页码是新的 + * （或相反），完整性判断会得出错误结论。SQLite 的读事务能看到固定快照，避免这种交错。 + */ + private GidToKey lookupInTransaction(GalleryLink link) { + GidToKey header = mapper.selectKeyByGid(link.gid()); + if (header == null) + return null; + if (header.getPages() <= 0 || header.getKey() == null || header.getKey().isBlank()) + return null; + if (!isComplete(header)) + return null; + return header; + } + + private boolean isComplete(GidToKey header) { + List pages = mapper.selectPageNumbersByGid(header.getGid()); + if (pages == null || pages.size() != header.getPages()) + return false; + if (mapper.countInvalidImageKeysByGid(header.getGid()) != 0) + return false; + // 页码必须是严格递增的 1..pages，重复或跳号都视为不完整。 + for (int i = 0; i < pages.size(); i++) { + Integer page = pages.get(i); + if (page == null || page != i + 1) + return false; + } + return true; + } + + /** 抓上游（事务外）后短事务整组替换。 */ + private GidToKey buildAndPublish(String url) throws GalleryIndexException { + GalleryLink link = GalleryLink.parse(url); + List pageKeys; + try { + // 抓取必须发生在事务之外：上游可能耗时十几秒，占着 SQLite 写事务会拖垮整个缓存库。 + pageKeys = upstream.fetchPageKeys(url); + } catch (Exception e) { + throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT, + "抓取图片索引失败 gid=" + link.gid(), e); + } + if (pageKeys == null || pageKeys.isEmpty()) + throw new GalleryIndexException(GalleryIndexException.Reason.UNAVAILABLE, + "该图片已下架或已被删除 gid=" + link.gid()); + + // 落库前先校验上游数据自洽：整组替换一旦开始就没有回头路， + // 与其把畸形页码写进唯一约束再回滚，不如在此之前就判为失败。 + validatePageKeys(link, pageKeys); + + GidToKey header = new GidToKey(); + header.setGid(link.gid()); + header.setKey(link.key()); + header.setPages(pageKeys.size()); + + publish(header, pageKeys); + return header; + } + + /** + * 校验上游返回的页 key 是否自洽。 + * + *

四条要求缺一不可，否则索引会以「看起来完整」的样子写进库： + * gid 属于请求的那本画廊、页码恰好是 1..N、页码不重复、imgkey 非空。 + * 任一不满足都抛出 {@link GalleryIndexException.Reason#MALFORMED}，此时尚未开启写事务， + * 库中旧索引保持原样。 + */ + void validatePageKeys(GalleryLink link, List pageKeys) throws GalleryIndexException { + int expected = pageKeys.size(); + Set seen = new HashSet<>(expected * 2); + for (ImageKeyCache row : pageKeys) { + if (row == null || row.getGid() == null || !row.getGid().equals(link.gid())) + throw malformed(link, "上游页 key 的 gid 与请求不一致"); + if (row.getImgkey() == null || row.getImgkey().isBlank()) + throw malformed(link, "上游页 key 的 imgkey 为空"); + int page = row.getPage(); + if (page < 1 || page > expected) + throw malformed(link, "上游页码越界: " + page + "（共 " + expected + " 页）"); + if (!seen.add(page)) + throw malformed(link, "上游页码重复: " + page); + } + if (seen.size() != expected) + throw malformed(link, "上游页码不连续，去重后 " + seen.size() + " 个，期望 " + expected + " 个"); + } + + private GalleryIndexException malformed(GalleryLink link, String detail) { + log.warn("上游图片索引非法 gid={} detail={}", link.gid(), detail); + return new GalleryIndexException(GalleryIndexException.Reason.MALFORMED, + detail + " gid=" + link.gid()); + } + + /** + * 在单个事务里整组替换索引。 + * + *

失败时事务整体回滚：要么看到旧的完整索引，要么看到新的完整索引， + * 不存在「头部已更新、页 key 缺失」的中间态。 + */ + void publish(GidToKey header, List pageKeys) { + operations.executeWithoutResult(status -> { + mapper.deleteImageKeyCacheByGid(header.getGid()); + mapper.deleteGidToKey(header.getGid()); + mapper.insertGidToKey(header); + writeBatches(pageKeys); + }); + log.info("图片索引已整体替换 gid={} pages={}", header.getGid(), header.getPages()); + } + + private void writeBatches(List pageKeys) { + for (int start = 0; start < pageKeys.size(); start += BATCH_SIZE) { + int end = Math.min(start + BATCH_SIZE, pageKeys.size()); + mapper.insertImageKeyCacheBatch(new ArrayList<>(pageKeys.subList(start, end))); + } + } + + /** 便于日志与错误信息使用的解析结果。 */ + record GalleryLink(String gid, String key) { + + static GalleryLink parse(String url) throws GalleryIndexException { + String[] segments = url == null ? null : url.split("/"); + Integer gid = GalleryUtil.parseGid(url); + if (gid == null || segments == null || segments.length <= 5 || segments[5].isBlank()) + throw new GalleryIndexException(GalleryIndexException.Reason.INVALID_LINK, + "链接错误"); + return new GalleryLink(String.valueOf(gid), segments[5].strip()); + } + } + + /** + * 上游取页 key 的端口。 + * + *

生产实现只是把 URL 转交给 GalleryUtil.parseImageKeys； + * 单独抽出来是为了让本服务在测试里完全脱离网络与静态工具类。 + */ + @FunctionalInterface + public interface GalleryKeyUpstream { + List fetchPageKeys(String url) throws Exception; + } + + /** + * 构建索引的动作。允许抛出业务异常，因此不用 java.util.function.Function + * （后者不能抛受检异常，会让调用方被迫包装成运行时异常）。 + */ + @FunctionalInterface + interface IndexBuilder { + GidToKey build(String url) throws GalleryIndexException; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java b/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java index bbece46..0b03ca0 100644 --- a/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java +++ b/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java @@ -46,6 +46,16 @@ public class GalleryManageService { final PushService pushService; + /** + * 任务生命周期（协议 v2）。创建/删除都先走它做持久化意图，再产生外部副作用。 + * 用字段注入是为了兼容现有构造式 mock 测试与旧调用点。 + */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + GalleryTaskLifecycleService lifecycleService; + + @org.springframework.beans.factory.annotation.Autowired + GalleryIndexService galleryIndexService; + /** * 创建任务 * @param link 任务链接 @@ -79,6 +89,20 @@ public class GalleryManageService { pushService.taskCreateReport(user.getUsername(), taskName, response); return response.toJSONString(); } + + // 协议 v2 生命周期：gid 唯一意图先落库（占位行「提交中」），提交前持久化 SUBMITTING， + // 上游结果不确定时报 UNKNOWN 而不是自动重发；差额额度与任务行在同一短事务里写入。 + if (lifecycleService != null) { + response = lifecycleService.createTask(gid, taskName, link, targetResolution, user.getId()); + Gallery current = galleryMapper.selectGalleryByGid(gid); + taskName = current == null || current.getName() == null ? taskName : current.getName(); + // 业务结果由生命周期服务给出，data 用结构化对象下发（前端解码器兼容旧字符串）。 + if (response.isSuccess() && current != null) + response.success(objectMapper.valueToTree(current)); + pushService.taskCreateReport(user.getUsername(), taskName, response); + return response.toJSONString(); + } + Gallery gallery; //判断数据库中是否有这个任务，有则返回状态 @@ -124,7 +148,7 @@ public class GalleryManageService { // Do not overwrite an immediate node status with the original submitted state. Gallery current = galleryMapper.selectGalleryByGid(gallery.getGid()); - response.success((current == null ? gallery : current).toString()); + response.success(objectMapper.valueToTree(current == null ? gallery : current)); pushService.taskCreateReport(user.getUsername(), taskName, response); return response.toJSONString(); @@ -184,7 +208,7 @@ public class GalleryManageService { try { gallery = GalleryUtil.parse(link, false, null); if (gallery != null) - response.success(objectMapper.valueToTree(gallery).toString()); + response.success(objectMapper.valueToTree(gallery)); else response.failure("查询失败"); } catch (Exception e) { @@ -193,7 +217,7 @@ public class GalleryManageService { } else - response.success(objectMapper.valueToTree(gallery).toString()); + response.success(objectMapper.valueToTree(gallery)); return response.toJSONString(); } @@ -212,7 +236,7 @@ public class GalleryManageService { response.failure("未找到该图片，请使用链接"); else - response.success(gallery.toString()); + response.success(objectMapper.valueToTree(gallery)); return response.toJSONString(); } @@ -245,7 +269,7 @@ public class GalleryManageService { if (userId == UserService.ADMIN_USER_ID) fillDownloaderNames(galleries); - response.success(objectMapper.valueToTree(galleries).toString()); + response.success(objectMapper.valueToTree(galleries)); return response.toJSONString(); } @@ -302,7 +326,7 @@ public class GalleryManageService { Gallery gallery = galleryMapper.selectGalleryByName("%" + name + "%"); if (gallery != null) - response.success(objectMapper.valueToTree(gallery).toString()); + response.success(objectMapper.valueToTree(gallery)); else response.failure("没有找到该名字的图片"); @@ -320,7 +344,7 @@ public class GalleryManageService { Gallery[] galleries = galleryMapper.selectGalleryByDownloader(userMapper.selectUserByAuthCode(AuthCode).getId()); if (galleries.length > 0) - response.success(objectMapper.valueToTree(galleries).toString()); + response.success(objectMapper.valueToTree(galleries)); else response.failure("您未下载图片"); @@ -358,8 +382,46 @@ public class GalleryManageService { return response.toJSONString(); } - // 通过授权后才落库并通知节点，避免被拒请求仍删除节点文件。 + // 通过授权后才产生外部副作用。协议 v2 的删除顺序： + // 先落「待删除」意图 → 节点确认（或文件已不存在）→ 才在同一事务删任务行并留墓碑。 + // 失败时任务行与意图都保留，可沿原记录重试并复用同一个删除 operationId。 log.info("删除图片{}", gallery.getName()); + if (lifecycleService != null) { + // 被拒绝（例如任务仍在提交中）时必须回业务失败： + // 绝不能退避到裸删除，否则会与并发的创建 POST 同时执行并写回已删/重建的 gid。 + GalleryTaskLifecycleService.DeleteIntent intent = lifecycleService.beginDelete(gallery.getGid()); + if (!intent.accepted()) { + response.failure(intent.message()); + log.info("拒绝删除 gid={}：{}", gid, intent.message()); + return response.toJSONString(); + } + byte result = remoteService.sendDeleteCommand(intent.deleteOperationId(), + gallery.getGid(), gallery.getName()); + switch (result) { + case 0 -> { + lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId()); + response.success(); + } + // 文件已不存在等于目标达成：同样完成本地清理，避免留下永远删不掉的任务行。 + case ErrorCode.FILE_NOT_FOUND -> { + lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId()); + response.success(); + } + case ErrorCode.IO_ERROR -> { + lifecycleService.failDelete(gallery.getGid()); + response.failure("图片:" + gallery.getName() + "删除失败，IO错误"); + } + // 节点无响应/超时：保留记录与待删除意图，可重试。 + default -> { + lifecycleService.failDelete(gallery.getGid()); + response.failure("图片:" + gallery.getName() + "删除失败，节点无响应"); + } + } + if (!response.isSuccess()) + log.info(response.getData()); + return response.toJSONString(); + } + galleryMapper.deleteGalleryByGid(gallery.getGid()); switch (remoteService.deleteGallery(gallery)) { case ErrorCode.IO_ERROR -> response.failure("图片:" + gallery.getName() + "删除失败，IO错误"); @@ -399,76 +461,25 @@ public class GalleryManageService { lastResetAmountTime == null || lastResetAmountTime.getValue() == null ? "" : lastResetAmountTime.getValue()); - response.success(objectMapper.valueToTree(data).toString()); + response.success(objectMapper.valueToTree(data)); return response.toJSONString(); } public String cacheImagesKey(String url) { - Response response = Response.generateResponse(); - - // 畸形链接在这里就拦掉：段数不足会让 url.split("/")[5] 抛 ArrayIndexOutOfBoundsException， - // 而 parseGid 也会对无 /g/ 的链接返回 null，两者都不该变成 500。 - String[] segments = url == null ? null : url.split("/"); - Integer parsedGid = parseGidFromLink(url); - if (parsedGid == null || segments == null || segments.length <= 5) { - response.failure("链接错误"); - return response.toJSONString(); - } - - String gid = String.valueOf(parsedGid); - GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid); - //已缓存过且页 key 完整，直接返回 - if(gidToKey != null && isIndexComplete(gidToKey)) { - return response.success(objectMapper.valueToTree(gidToKey)).toJSONString(); - } - // 半截缓存（历史崩溃或旧版本写入中断）先清干净再整体重建，否则残留的页 key 会重复累积。 - if (gidToKey != null) { - log.warn("图片索引缓存不完整，重建 gid={} 记录页数={}", gid, gidToKey.getPages()); - rollbackImageIndex(gid); - } - - boolean inserted = false; try { - gidToKey = new GidToKey(); - gidToKey.setGid(gid); - gidToKey.setKey(segments[5].strip()); - ArrayList imageKeyCaches = GalleryUtil.parseImageKeys(url); - if(imageKeyCaches == null) - return response.failure("该图片已下架或已被删除").toJSONString(); - gidToKey.setPages(imageKeyCaches.size()); - imageCacheMapper.insertGidToKey(gidToKey); - inserted = true; - for (ImageKeyCache imageKeyCache : imageKeyCaches) - imageCacheMapper.insertImageKeyCache(imageKeyCache); - response.success(objectMapper.valueToTree(gidToKey)); - }catch (Exception e){ - // 索引必须整体生效：先写 gidToKey 再逐页写 key，中途失败会留下「gidToKey 命中、 - // 页 key 缺失」的半截缓存，后续请求直接返回已缓存而永远取不到图。 - // 因此任何异常都回滚已写入的部分再回业务失败。Jackson 3 的解析异常继承 - // RuntimeException 而非 IOException，只用 IOException 会漏掉，这里统一兜住。 - if (inserted) - rollbackImageIndex(gid); - log.warn("缓存图片索引失败 gid={} errorType={}", gid, e.getClass().getSimpleName()); - response.failure("网络波动或其他异常"); - } - return response.toJSONString(); - } - - /** 页 key 行数与 gidToKey 记录不一致时视为半截缓存，需要重建。 */ - private boolean isIndexComplete(GidToKey gidToKey) { - Integer pages = gidToKey.getPages(); - if (pages == null || pages <= 0) - return false; - return imageCacheMapper.countImageKeyCacheByGid(gidToKey.getGid()) == pages; - } - - /** 删除某个 gid 已写入的索引（页 key 与 gidToKey），用于回滚或重建。 */ - private void rollbackImageIndex(String gid) { - try { - imageCacheMapper.deleteImageKeyCacheByGid(gid); - imageCacheMapper.deleteGidToKey(gid); - } catch (Exception e) { - log.warn("回滚图片索引缓存失败 gid={}", gid, e); + GidToKey index = galleryIndexService.ensureIndex(url); + return Response.generateResponse().success(objectMapper.valueToTree(index)).toJSONString(); + } catch (com.lion.lionwebsite.Exception.GalleryIndexException error) { + String message = switch (error.getReason()) { + case INVALID_LINK -> "链接错误"; + case UNAVAILABLE -> "该图片已下架或已被删除"; + case MALFORMED, TRANSIENT -> "网络波动或其他异常"; + }; + log.warn("缓存图片索引失败 reason={}", error.getReason()); + return Response._failure(message); + } catch (RuntimeException error) { + log.warn("缓存图片索引失败 errorType={}", error.getClass().getSimpleName()); + return Response._failure("网络波动或其他异常"); } } @@ -517,10 +528,24 @@ public class GalleryManageService { Gallery[] galleries = galleryMapper.selectUnDoneGalleries(); if(galleries != null && galleries.length != 0) { - log.info("发送未下载完成图片至节点,{}本", galleries.length); - for (Gallery gallery : galleries) + // 只重投「确实已提交上游」的任务：提交未知/预占中/待删除的任务重投会变成 + // 未计量副作用（节点侧显示下载完成但没有任何额度记录）。 + int sent = 0; + int skipped = 0; + for (Gallery gallery : galleries) { + if (lifecycleService != null + && !lifecycleService.resolveDispatch(gallery.getGid()).dispatchable()) { + skipped++; + continue; + } remoteService.addGalleryToQueue(gallery); - response.success(String.format("发送未下载完成图片至节点,%s本", galleries.length)); + sent++; + } + log.info("发送未下载完成图片至节点,{}本（跳过需要对账的{}本）", sent, skipped); + if (sent == 0) + response.failure("没有可安全重投的任务，需先人工对账提交未知的任务"); + else + response.success(String.format("发送未下载完成图片至节点,%s本", sent)); }else{ response.failure("当前没有未下载完成的图片"); } @@ -536,6 +561,13 @@ public class GalleryManageService { return Response._success("下载完成"); if(remoteService.isDead()) return Response._failure("节点不在线，无法重试"); + // 提交结果未知（UNKNOWN）或正在删除的任务不能重试：上游不支持幂等，重发会让 + // 「可能已受理但未计量」的任务在节点侧变成下载完成却 0 额度。这类任务需人工对账。 + if (lifecycleService != null) { + var decision = lifecycleService.resolveDispatch(gid); + if (decision.kind() == GalleryTaskLifecycleService.DispatchDecision.Kind.REFUSE) + return Response._failure("该任务提交结果未知或正在删除，请先人工对账后再重试"); + } RemoteService.RetryResult retryResult = remoteService.retryGallery(gallery); if(retryResult.success()) diff --git a/src/main/java/com/lion/lionwebsite/Service/GalleryTaskLifecycleService.java b/src/main/java/com/lion/lionwebsite/Service/GalleryTaskLifecycleService.java new file mode 100644 index 0000000..59f511b --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/GalleryTaskLifecycleService.java @@ -0,0 +1,783 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.normal.*; +import com.lion.lionwebsite.Domain.*; +import com.lion.lionwebsite.Exception.ResolutionNotMatchException; +import com.lion.lionwebsite.Message.TaskCommandMessage; +import com.lion.lionwebsite.Message.TaskReportMessage; +import com.lion.lionwebsite.Util.PreparedGallery; +import com.lion.lionwebsite.Util.Response; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static com.lion.lionwebsite.Util.CustomUtil.objectMapper; + +/** + * 下载任务的持久化生命周期（协议 v2）。 + * + *

核心约束： + *

    + *
  1. 先保存意图，再产生外部副作用：同一 gid 的唯一 CREATE 意图先落库（占位任务行 + RESERVED 操作记录）， + * 通过 gid 的主键唯一约束把并发创建收敛成「一个赢家」；
    2. + *
  2. SUBMITTING 之后绝不自动重发：上游不支持幂等，提交结果不确定时只能落 UNKNOWN，等显式对账/解决；
    3. + *
  3. 提交结果与额度用一个短事务写入：Gallery 与 weekUsedAmount 要么一起成功，要么一起回滚， + * 事务提交之后才把 DOWNLOAD 命令下发给节点；
    4. + *
  4. 删除失败保留记录：先落待删除意图，节点确认（或文件已不存在）后才在同一事务里删任务行并留墓碑； + * 失败则保留 Gallery 与 DELETE_PENDING 意图，可沿原记录重试且复用同一删除 operationId。
    5. + *
+ * + *

所有网络调用都发生在事务之外；事务回调内部不做 I/O，也不吞异常（吞掉会错误提交）。 + */ +@Service +@Slf4j +public class GalleryTaskLifecycleService implements TaskReportHandler { + + final GalleryMapper galleryMapper; + + final CollectMapper collectMapper; + + final CustomConfigurationMapper configurationMapper; + + final UserMapper userMapper; + + final TaskOperationRepository taskOperationRepository; + + final UpstreamGateway upstreamGateway; + + final RemoteService remoteService; + + /** + * 短全局创建预占锁。 + * + *

只有「登记唯一 CREATE 意图」这一小段数据库写入在锁内，网络调用一律在锁外。 + * 单进程内它把并发同 gid 创建直接收敛成一个赢家；跨进程的正确性仍由 + * {@code task_operation}/{@code gallery} 的主键唯一约束兜底。 + */ + private final Object reservationLock = new Object(); + + /** + * 测试钩子：进入上游只读解析之前执行。 + * 用来让并发删除发生在预占之后、SUBMITTING 之前。生产环境是空实现。 + */ + volatile Runnable beforePrepare = () -> { }; + + /** + * 测试钩子：SUBMITTING 已持久化之后、真正 POST 上游之前执行。 + * 用来制造「提交瞬间任务被删除」的 late generation 场景。生产环境是空实现。 + */ + volatile Runnable afterMarkSubmitting = () -> { }; + + /** 主库事务模板；测试可注入内存 sqlite 的事务模板。 */ + TransactionTemplate transactions; + + public GalleryTaskLifecycleService(GalleryMapper galleryMapper, + CollectMapper collectMapper, + CustomConfigurationMapper configurationMapper, + UserMapper userMapper, + TaskOperationRepository taskOperationRepository, + UpstreamGateway upstreamGateway, + RemoteService remoteService, + @Qualifier("mainTransactionTemplate") TransactionTemplate transactions) { + this.galleryMapper = galleryMapper; + this.collectMapper = collectMapper; + this.configurationMapper = configurationMapper; + this.userMapper = userMapper; + this.taskOperationRepository = taskOperationRepository; + this.upstreamGateway = upstreamGateway; + this.remoteService = remoteService; + this.transactions = transactions; + } + + /** 测试用：替换事务模板或在没有 Spring 事务时退化为直连执行。 */ + void setTransactions(TransactionTemplate transactions) { + this.transactions = transactions; + } + + /** + * 一次「登记创建意图」的结果。 + * + *

只有赢家能继续做外部副作用；输家（已存在有效意图 / 数据库竞争失败） + * 一律转成业务失败，绝不再 POST 上游。 + */ + private enum ReserveOutcome { + /** 已成功登记唯一 CREATE 意图，可以做只读解析与提交。 */ + RESERVED, + /** 同一 gid 已存在有效意图（或任务行）。 */ + DUPLICATE, + /** 登记时的数据库竞争/失败：没有产生任何副作用，必须转业务失败。 */ + CONTENDED + } + + /** 一次「登记创建意图」的返回：结果 + 赢家持有的操作记录。 */ + private record Reservation(ReserveOutcome outcome, TaskOperation operation) { + } + + /** + * 单次下发决策。 + * + *

把「能不能把这个任务交给节点」从「下载是否已提交」里拆出来： + * 未确认上游提交结果（UNKNOWN / 预占 / 待删除 / 删除墓碑）绝不能下发， + * 否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。 + */ + public static final class DispatchDecision { + + public enum Kind { + /** 当前 operation 已是 SUBMITTED：可以按稳定 operationId 下发。 */ + READY, + /** 迁移前的旧任务行：没有操作记录但历史上确实提交过下载。 */ + LEGACY, + /** 不可下发：状态不明或被删除，下发会产生未计量副作用。 */ + REFUSE + } + + private final Kind kind; + + private final String operationId; + + private DispatchDecision(Kind kind, String operationId) { + this.kind = kind; + this.operationId = operationId; + } + + public Kind kind() { + return kind; + } + + public String operationId() { + return operationId; + } + + /** 只有 READY/LEGACY 允许真正下发到节点。 */ + public boolean dispatchable() { + return kind != Kind.REFUSE && operationId != null; + } + + /** 拒绝下发（供调用方与测试构造）。 */ + public static DispatchDecision refuse() { + return REFUSE; + } + + /** 可直接下发（供测试构造）。 */ + public static DispatchDecision ready(String operationId) { + return new DispatchDecision(Kind.READY, operationId); + } + } + + private static final DispatchDecision REFUSE = new DispatchDecision(DispatchDecision.Kind.REFUSE, null); + + /** + * 一次删除登记的返回。 + * + *

用显式结果替代「null 即退避到裸删除」的旧语义： + * 被拒绝时调用方必须回业务失败，不能绕过待删除意图直接删节点文件。 + */ + public record DeleteIntent(boolean accepted, String deleteOperationId, String message) { + + static DeleteIntent refused(String message) { + return new DeleteIntent(false, null, message); + } + + static DeleteIntent accepted(String deleteOperationId) { + return new DeleteIntent(true, deleteOperationId, null); + } + } + + /** 在事务里执行；没有配置事务模板时候直接执行（仅便于构造 mock 测试）。 */ + private T inTransaction(java.util.function.Supplier body) { + TransactionTemplate template = transactions; + if (template == null) + return body.get(); + return template.execute(status -> { + try { + return body.get(); + } catch (RuntimeException e) { + // 回调里必须把异常继续抛出，否则 Spring 会把只读了一半的事务当成正常提交。 + status.setRollbackOnly(); + throw e; + } + }); + } + + /** + * 创建任务的持久化前置：在同一个事务里登记唯一 CREATE 意图。 + * + *

同时写入可见的占位任务行（状态「提交中」），这样提交期间任务列表就能解析该 gid， + * 不会出现「上游已受理但本地查不到」的空窗。 + * + * @return 已登记意图（赢家）返回它；同一 gid 已有有效意图返回 {@code null}（输家）。 + */ + private Reservation reserveCreateIntent(int gid, String placeholderName, int downloaderId) { + // 锁内只做数据库写入：把并发同 gid 创建串行化，网络调用留在锁外。 + synchronized (reservationLock) { + try { + TaskOperation operation = inTransaction(() -> { + TaskOperation existingOperation = taskOperationRepository.selectByGid(gid); + Gallery existingGallery = galleryMapper.selectGalleryByGid(gid); + if (existingOperation != null) { + // 只有「删除已完成」的墓碑才允许用全新 operationId 重新创建（显式重建）。 + if (!TaskOperation.STATE_DELETED.equals(existingOperation.getState())) + return null; + if (existingGallery != null) + return null; + String operationId = TaskProtocol.newDownloadOperationId(); + taskOperationRepository.reactivate(gid, operationId, TaskOperation.STATE_RESERVED, now()); + insertPlaceholder(gid, placeholderName, downloaderId); + return taskOperationRepository.selectByGid(gid); + } + if (existingGallery != null) + return null; + TaskOperation created = new TaskOperation(); + created.setGid(gid); + created.setOperationId(TaskProtocol.newDownloadOperationId()); + created.setState(TaskOperation.STATE_RESERVED); + created.setUpdateTime(now()); + taskOperationRepository.insert(created); + insertPlaceholder(gid, placeholderName, downloaderId); + return created; + }); + return operation == null + ? new Reservation(ReserveOutcome.DUPLICATE, null) + : new Reservation(ReserveOutcome.RESERVED, operation); + } catch (RuntimeException e) { + // 数据库竞争/失败发生在任何外部副作用之前，绝不能穿透成 500，也不能继续提交。 + log.warn("登记创建意图失败，按业务失败处理 gid={} error={}", gid, e.getClass().getSimpleName()); + return new Reservation(ReserveOutcome.CONTENDED, null); + } + } + } + + /** 写入占位任务行：让提交期间的任务在列表里可见。 */ + private void insertPlaceholder(int gid, String name, int downloaderId) { + Gallery placeholder = new Gallery(); + placeholder.setGid(gid); + placeholder.setName(name); + placeholder.setStatus(TaskStatus.PENDING_SUBMIT); + placeholder.setDownloader(downloaderId); + placeholder.setCreateTime(System.currentTimeMillis() / 1000); + placeholder.setProceeding(0); + galleryMapper.insertGallery(placeholder); + } + + /** + * 标记为正在提交，并用 CAS 确认这一代意图仍然有效。 + * + *

只有当前状态与 operationId 都匹配（RESERVED → SUBMITTING）才返回 true； + * 期间被删除或重建返回 false，调用方必须停下，绝不能再 POST 上游。 + * + * @return true 表示已持久化 SUBMITTING，可以安全产生上游副作用 + */ + boolean markSubmitting(String operationId, int gid) { + return inTransaction(() -> taskOperationRepository.compareAndSetCreateState(gid, operationId, + TaskOperation.STATE_RESERVED, TaskOperation.STATE_SUBMITTING, now()) == 1); + } + + /** + * 提交结果未知（超时/断连/中断）。保留意图，等待人工对账，不自动重发。 + * + *

TaskOperation 与 Gallery 状态在同一个短事务里更新： + * 操作落 UNKNOWN 而任务行仍显示「提交中」会让运营侧误以为还能自动重试， + * 因此两边必须一致地表达「不确定，需要人工对账」。 + */ + void markUnknown(String operationId, int gid) { + inTransaction(() -> { + int advanced = taskOperationRepository.compareAndSetCreateState(gid, operationId, + TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now()); + // 只有本次这一代确实从 SUBMITTING 推进到 UNKNOWN 时才同步任务行： + // 若期间已被删除/重建（CAS 返回 0），绝不能再改那条记录。 + if (advanced != 1) + return null; + Gallery gallery = galleryMapper.selectGalleryByGid(gid); + if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus())) + galleryMapper.updateGalleryStatus(gid, TaskStatus.UNKNOWN); + return null; + }); + } + + /** + * 提交成功后的「一次性短事务」：写入 Gallery 行、累计周用量、把操作推进为 SUBMITTED。 + * 三者同一个事务，任何一个失败都一起回滚，避免出现「任务存在但用量未计」这类不一致。 + */ + /** + * 提交成功后的落库（带世代校验）。 + * + * @param expectedOperationId 本次提交所归属的 operationId；只有它仍是当前有效意图时才能落库， + * 否则（被删除/重建）必须抛错，避免复活已删 gid 或污染新世代 + */ + int persistSubmitted(Gallery gallery, int downloaderId, String expectedOperationId) { + return inTransaction(() -> { + TaskOperation operation = taskOperationRepository.selectByGid(gallery.getGid()); + // 严格匹配本次世代：任务被删除、被重建或换了 operationId 时都不能落库。 + if (operation == null || TaskOperation.STATE_DELETED.equals(operation.getState()) + || TaskOperation.STATE_DELETE_PENDING.equals(operation.getState())) + throw new IllegalStateException("任务已删除或没有有效创建意图，禁止落库复活 gid=" + gallery.getGid()); + if (expectedOperationId == null || !expectedOperationId.equals(operation.getOperationId())) + throw new IllegalStateException("创建意图世代不匹配，禁止落库 gid=" + gallery.getGid()); + // 幂等：同一意图重复落库（例如上游已提交后重跑）不得把用量累计两次。 + boolean alreadyCounted = TaskOperation.STATE_SUBMITTED.equals(operation.getState()); + gallery.setDownloader(downloaderId); + gallery.set_download(true); + if (!TaskStatus.SUBMITTED.equals(gallery.getStatus())) + gallery.setStatus(TaskStatus.SUBMITTED); + Gallery existing = galleryMapper.selectGalleryByGid(gallery.getGid()); + if (existing == null) + galleryMapper.insertGallery(gallery); + else + galleryMapper.updateGallery(gallery); + if (!alreadyCounted) + configurationMapper.incrementConfiguration(CustomConfiguration.WEEK_USED_AMOUNT, gallery.getFileSize()); + taskOperationRepository.updateState(gallery.getGid(), TaskOperation.STATE_SUBMITTED, now()); + return 1; + }); + } + + /** + * 预占式创建：登记 gid 唯一意图（提交任务行占位）→ 只读解析上游 → 标 SUBMITTING → 真正提交。 + * + *

上游提交发生在事务之外，且只在 SUBMITTING 已持久化之后执行一次。网络异常一律落 UNKNOWN。 + */ + public Response createTask(int gid, String placeholderName, String link, String targetResolution, int downloaderId) { + Response response = Response.generateResponse(); + Reservation reservation = reserveCreateIntent(gid, placeholderName, downloaderId); + if (reservation.outcome() == ReserveOutcome.DUPLICATE) { + response.failure("任务队列已有此任务，请点击查找任务"); + return response; + } + if (reservation.outcome() == ReserveOutcome.CONTENDED) { + // 预占阶段的数据库竞争不产生任何副作用；转业务失败，绝不下发上游、绝不穿透成 500。 + response.failure("任务提交失败，请稍后重试"); + return response; + } + + // 解析上游是只读步骤；真正的 POST 只在 SUBMITTING 持久化之后发生。 + // 生命周期约束：RESERVED 阶段被中断是安全的（可释放重来），SUBMITTING 之后一律按 UNKNOWN 处理。 + return submitPreparedIntent(gid, reservation.operation(), link, targetResolution, downloaderId); + } + + private Response submitPreparedIntent(int gid, TaskOperation operation, String link, String targetResolution, + int downloaderId) { + Response response = Response.generateResponse(); + beforePrepare.run(); + PreparedGallery prepared; + try { + prepared = upstreamGateway.prepare(link); + } catch (IOException e) { + log.warn("解析任务链接失败 gid={}", gid, e); + releaseReservation(gid, operation.getOperationId()); + response.failure("IO错误，可能是网络波动"); + return response; + } catch (RuntimeException e) { + log.warn("解析任务链接异常 gid={}", gid, e); + releaseReservation(gid, operation.getOperationId()); + response.failure("提交任务失败，未知原因，请检查链接是否正常"); + return response; + } + if (prepared == null || prepared.gallery() == null) { + releaseReservation(gid, operation.getOperationId()); + response.failure("提交任务失败，未知原因，请检查链接是否正常"); + return response; + } + + Gallery gallery = prepared.gallery(); + gallery.setGid(gid); + + // CAS 确认这一代意图仍然有效（未被并发删除/重建），并把状态持久化为 SUBMITTING。 + // 只有成功之后才允许真正 POST；失败说明任务已不在当前生命周期里，直接放弃。 + if (!markSubmitting(operation.getOperationId(), gid)) { + log.warn("创建意图已失效（可能被并发删除），放弃提交 gid={} operationId={}", + gid, operation.getOperationId()); + response.failure("任务已被删除或状态已变化，请刷新任务列表"); + return response; + } + afterMarkSubmitting.run(); + + Gallery submitted; + try { + submitted = upstreamGateway.submit(prepared, targetResolution); + } catch (ResolutionNotMatchException e) { + releaseReservation(gid, operation.getOperationId()); + response.failure("提交失败，分辨率不存在"); + return response; + } catch (IOException | RuntimeException e) { + // 超时或断连时上游可能已经受理，无法确认：只能记 UNKNOWN，绝不自动重发。 + log.warn("上游提交结果未知 gid={} operationId={}", gid, operation.getOperationId(), e); + markUnknown(operation.getOperationId(), gid); + response.failure("上游提交结果未知，请稍后在任务列表中确认，不要重复提交"); + return response; + } + if (submitted == null || !TaskStatus.SUBMITTED.equals(submitted.getStatus())) { + markUnknown(operation.getOperationId(), gid); + response.failure("上游提交结果未知，请稍后在任务列表中确认，不要重复提交"); + return response; + } + submitted.setGid(gid); + try { + persistSubmitted(submitted, downloaderId, operation.getOperationId()); + } catch (RuntimeException e) { + // 上游已受理但本地落库失败：保留 SUBMITTING 意图并标 UNKNOWN，任务行仍在列表里可对账。 + log.error("提交成功后落库失败 gid={} operationId={}", gid, operation.getOperationId(), e); + markUnknown(operation.getOperationId(), gid); + response.failure("任务已提交，但本地保存失败，请在任务列表中确认"); + return response; + } + // 只有事务提交之后才向节点下发命令；重复下发由节点按 operationId 去重。 + byte dispatchResult = dispatchDownload(gid, submitted.getName()); + Gallery current = galleryMapper.selectGalleryByGid(gid); + Gallery payload = current == null ? submitted : current; + if (dispatchResult == 0) + response.success(objectMapper.valueToTree(payload)); + else + response.failure("任务已保存，但节点未确认接收；请刷新任务列表后重试"); + return response; + } + + /** + * 提交成功后的落库 + 下发。落库（含用量）在一个短事务里完成，事务提交后才向节点发命令。 + * + *

先解析当前世代再落库：世代不存在（未提交过 / 已删除）时不落库也不下发。 + * + * @return 已持久化的 Gallery；数据库失败或世代无效时抛出异常，调用方回业务失败且不下发节点。 + */ + public Gallery persistAndDispatch(Gallery gallery, int downloaderId) { + DispatchDecision decision = resolveDispatch(gallery.getGid()); + if (!decision.dispatchable()) + throw new IllegalStateException("任务没有可下发的有效世代 gid=" + gallery.getGid()); + persistSubmitted(gallery, downloaderId, decision.operationId()); + dispatchDownload(gallery.getGid(), gallery.getName()); + return gallery; + } + + /** 向节点下发 DOWNLOAD 命令（复用当前有效 operationId），返回节点结果码。 */ + public byte dispatchDownload(int gid, String name) { + DispatchDecision decision = resolveDispatch(gid); + if (!decision.dispatchable()) + return -1; + return remoteService.sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(), gid, name); + } + + /** + * 判定一个 gid 能否下发给节点，并为迁移前的旧任务补齐稳定 operationId。 + * + *

规则： + *

    + *
  • v2 任务只有 {@code SUBMITTED} 可下发；{@code UNKNOWN} 必须人工对账， + * 否则未计量的未知任务会在节点侧变成「下载完成却 0 额度」；
    • + *
  • 预占中的 {@code RESERVED}/{@code SUBMITTING}、待删除与删除墓碑都不下发；
    • + *
  • 没有任何操作记录、但任务行确实存在且状态已是「已提交」及之后的历史任务， + * 补建一个稳定 operationId 后再下发（这是唯一的 legacy 兼容路径）。
    • + *
+ */ + public DispatchDecision resolveDispatch(int gid) { + TaskOperation operation = taskOperationRepository.selectByGid(gid); + if (operation != null && TaskOperation.STATE_SUBMITTED.equals(operation.getState()) + && operation.getOperationId() != null) + return new DispatchDecision(DispatchDecision.Kind.READY, operation.getOperationId()); + // 没有操作记录，或只有「旧世代」记录（缺 operationId）：必须先确认任务行确实已提交过， + // 再补一个稳定 operationId；UNKNOWN/预占/待删除一律拒绝。 + if (operation != null && !TaskOperation.STATE_SUBMITTED.equals(operation.getState())) + return REFUSE; + if (!isLegacySubmitted(gid)) + return REFUSE; + String freshId = ensureStableOperationId(gid); + return freshId == null ? REFUSE : new DispatchDecision(DispatchDecision.Kind.LEGACY, freshId); + } + + /** 迁移前的旧任务行：没有操作记录，但状态表明上游确实已受理过下载，可安全重投。 */ + private boolean isLegacySubmitted(int gid) { + Gallery gallery = galleryMapper.selectGalleryByGid(gid); + if (gallery == null || gallery.getStatus() == null) + return false; + return switch (gallery.getStatus()) { + case TaskStatus.SUBMITTED, TaskStatus.DOWNLOADING, TaskStatus.WAITING_COMPRESS, + TaskStatus.COMPRESSING -> true; + // 提交未知 / 提交中 / 待删除 / 删除失败 一律不算已提交，不能下发。 + default -> false; + }; + } + + /** + * 为一个已经真正提交过的旧任务持久化稳定 operationId。 + * + *

必须是短事务写入、先落库后返回，之后重投/重连才复用同一个 ID； + * 用 {@code insert or ignore} + 回读，避免与并发创建互相覆盖。 + * + * @return 稳定 operationId；无法安全建立时返回 null（调用方不下发） + */ + String ensureStableOperationId(int gid) { + // 与创建预占共用同一把短锁（锁内只有数据库写），并发为同一 gid 补 ID 时只能有一个赢家。 + synchronized (reservationLock) { + String candidate = TaskProtocol.newDownloadOperationId(); + return inTransaction(() -> { + if (taskOperationRepository.selectByGid(gid) == null) + taskOperationRepository.insertIfAbsent(gid, candidate, TaskOperation.STATE_SUBMITTED, now()); + else + taskOperationRepository.assignOperationIdIfMissing(gid, candidate, now()); + TaskOperation current = taskOperationRepository.selectByGid(gid); + if (current == null || current.getOperationId() == null) + return null; + // 只有确实处于 SUBMITTED 的世代才可下发；并发写入的其他状态一律拒绝。 + return TaskOperation.STATE_SUBMITTED.equals(current.getState()) ? current.getOperationId() : null; + }); + } + } + + /** + * 取消预占（解析失败、分辨率不存在等：还没有产生任何上游副作用）。 + * + *

只删除与本次创建 intent 匹配的 RESERVED 记录：若期间该 gid 已被删除并重建 + * （新的 operationId / 墓碑），这里绝不能把新世代记录抹掉。占位行同理只删「提交中」。 + */ + private void releaseReservation(int gid, String operationId) { + inTransaction(() -> { + // 只删与本代 operationId 匹配且仍停在 RESERVED/SUBMITTING 的记录。 + int removed = taskOperationRepository.deleteReservationIfSameGeneration(gid, operationId); + // 只有确实删掉了本代预占记录，才能删对应的占位行： + // 若期间已出现新的 RESERVED 代，删除返回 0，此时绝不能误删新占位。 + if (removed == 1) { + Gallery existing = galleryMapper.selectGalleryByGid(gid); + if (existing != null && TaskStatus.PENDING_SUBMIT.equals(existing.getStatus())) + galleryMapper.deleteGalleryByGid(gid); + } + return null; + }); + } + + /** + * 删除的持久化部分。 + * + *

正在创建意图（RESERVED/SUBMITTING）的任务不能删除：此刻 DELETE 与创建 POST 会同时发生， + * 随后创建落库还会把已删/重建的 gid 写回。这里直接拒绝，由用户在提交结果明确后再操作。 + * + * @return 显式结果：被拒时调用方必须回业务失败，不能退避到裸删除 + */ + public DeleteIntent beginDelete(int gid) { + return inTransaction(() -> { + Gallery gallery = galleryMapper.selectGalleryByGid(gid); + if (gallery == null) + return DeleteIntent.refused("删除失败，该图片不存在"); + TaskOperation operation = taskOperationRepository.selectByGid(gid); + if (operation != null && (TaskOperation.STATE_RESERVED.equals(operation.getState()) + || TaskOperation.STATE_SUBMITTING.equals(operation.getState()))) + return DeleteIntent.refused("任务正在提交中，请等待提交结果明确后再删除"); + if (operation == null) { + // 历史任务行没有操作记录：直接落一条 DELETE_PENDING 意图，避免中间态缺字段。 + TaskOperation fresh = new TaskOperation(); + fresh.setGid(gid); + fresh.setOperationId(TaskProtocol.newDownloadOperationId()); + fresh.setDeleteOperationId(TaskProtocol.newDeleteOperationId()); + fresh.setState(TaskOperation.STATE_DELETE_PENDING); + fresh.setUpdateTime(now()); + taskOperationRepository.insert(fresh); + galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING); + return DeleteIntent.accepted(fresh.getDeleteOperationId()); + } + if (operation.getDeleteOperationId() == null) { + operation.setDeleteOperationId(TaskProtocol.newDeleteOperationId()); + operation.setUpdateTime(now()); + } + taskOperationRepository.updateDeleteIntent(gid, operation.getDeleteOperationId(), + TaskOperation.STATE_DELETE_PENDING, now()); + galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING); + return DeleteIntent.accepted(operation.getDeleteOperationId()); + }); + } + + /** + * 删除的完成部分：节点已确认（或文件已不存在）后，在一个短事务里删任务行、收藏并留墓碑。 + * + *

事务提交前若失败，任务行与 DELETE_PENDING 意图都保留，可以继续重试。 + * 删除意图 ID 必须匹配（防止并发删除/重建时抹掉别的一代），随后只把 DELETE_PENDING + * 世代推进为 DELETED；墓碑（DELETED 且无删除意图）不允许在这里被重新激活。 + * + * @return true 表示本次删除意图已完成并留下墓碑 + */ + public boolean completeDelete(int gid, String deleteOperationId) { + return inTransaction(() -> { + TaskOperation operation = taskOperationRepository.selectByGid(gid); + if (operation == null || operation.getDeleteOperationId() == null + || !operation.getDeleteOperationId().equals(deleteOperationId)) + return false; + // 先清收藏再删任务行：收藏一侧在部分历史库上带有指向 gallery 的外键。 + collectMapper.deleteByGid(gid); + galleryMapper.deleteGalleryByGid(gid); + int updated = taskOperationRepository.completeDeleteIfSameGeneration(gid, deleteOperationId, + TaskOperation.STATE_DELETED, now()); + return updated == 1; + }); + } + + /** 删除失败：保留任务行并记一条可重试的失败状态。 */ + public void failDelete(int gid) { + inTransaction(() -> { + Gallery gallery = galleryMapper.selectGalleryByGid(gid); + if (gallery == null) + return null; + gallery.setStatus(TaskStatus.DELETE_FAILED); + galleryMapper.updateGallery(gallery); + return null; + }); + } + + /** + * 启动恢复：把上次中断在 SUBMITTING 的意图判为 UNKNOWN，并安全释放还停在 RESERVED 的预占。 + * + *

这段恢复必须发生在任何自动重投之前： + *

    + *
  • {@code SUBMITTING}：上游可能已经受理但本地无法确认，只能落 UNKNOWN（操作与任务行同时 + * 更新为「提交未知」），交人工对账，绝不自动再 POST；
    • + *
  • {@code RESERVED}：还没有产生任何上游副作用，重启后可以直接释放预占 + * （删掉匹配的 RESERVED 记录与「提交中」占位行），让用户重新提交。
    • + *
+ * + * @return 本次恢复（判未知 + 释放预占）处理的任务数 + */ + public int recoverInterruptedSubmissions() { + return inTransaction(() -> { + int recovered = 0; + for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_SUBMITTING)) { + int advanced = taskOperationRepository.compareAndSetCreateState(operation.getGid(), operation.getOperationId(), + TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now()); + if (advanced != 1) + continue; + Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid()); + if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus())) + galleryMapper.updateGalleryStatus(operation.getGid(), TaskStatus.UNKNOWN); + recovered++; + } + for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_RESERVED)) { + // 还没 POST：释放匹配的预占记录与占位行，允许重启后重新提交。 + // 只有确实删掉了这条本代 RESERVED 记录，才删其占位行；否则不动（可能已换代）。 + int removed = taskOperationRepository.deleteReservationIfSameGeneration(operation.getGid(), + operation.getOperationId()); + if (removed == 1) { + Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid()); + if (gallery != null && TaskStatus.PENDING_SUBMIT.equals(gallery.getStatus())) + galleryMapper.deleteGalleryByGid(operation.getGid()); + } + recovered++; + } + return recovered; + }); + } + + /** 已完成状态上报的处理：丢弃旧 revision、旧 operation、待删除任务与重复上报。 */ + @Override + public CommittedReport apply(TaskReportMessage report) { + if (report == null || report.getReportId() == null || report.getGalleryTasks() == null) + return null; + Applied applied; + try { + applied = inTransaction(() -> applyInTransaction(report)); + } catch (RuntimeException e) { + // 数据库失败：不返回可回执结果，节点会保留快照并稍后重投；失败写入绝不清任务。 + log.warn("应用节点上报失败，未回执 reportId={}", report.getReportId(), e); + return null; + } + if (applied == null) + return null; + return new CommittedReport(report.getReportId(), applied.newlyCompleted(), applied.acceptedTasks()); + } + + /** 事务内应用上报的结果：新完成的任务与真正被接受的进度更新。 */ + private record Applied(List newlyCompleted, List acceptedTasks) { + } + + /** + * 在一个事务里应用上报。 + * + * @return 非 null 表示已经提交（可以回 ACK，重复上报也会返回结果以停止重放）。 + */ + private Applied applyInTransaction(TaskReportMessage report) { + long revision = report.getRevision(); + List newlyCompleted = new ArrayList<>(); + List acceptedTasks = new ArrayList<>(); + for (GalleryTask task : report.getGalleryTasks()) { + if (task == null) + continue; + TaskOperation operation = taskOperationRepository.selectByGid(task.getGid()); + Gallery gallery = galleryMapper.selectGalleryByGid(task.getGid()); + if (gallery == null) { + // 未知 gid（任务已删除）：忽略但 ACK，让节点停止重放。 + log.debug("忽略节点上报的未知任务 gid={}", task.getGid()); + continue; + } + String expectedOperationId = report.getOperationIds() == null + ? null : report.getOperationIds().get(task.getGid()); + if (operation == null) { + // 旧任务行（迁移前创建）没有操作记录：按显式生成策略补建并采纳上报里的 operationId。 + operation = adoptGeneration(task.getGid(), expectedOperationId); + } else if (TaskOperation.STATE_DELETED.equals(operation.getState())) { + // 墓碑必须永久忽略旧上报：删除已完成的任务只能由显式 CREATE 重新建立新世代。 + // （gallery 已被删除时上面的 null 分支已经提前忽略，这里防御性兜底。） + log.debug("忽略已删除 gid 的迟到上报 gid={}", task.getGid()); + continue; + } else if (TaskOperation.STATE_DELETE_PENDING.equals(operation.getState())) { + log.info("忽略待删除任务的迟到上报 gid={}", task.getGid()); + continue; + } else if (operation.getLastRevision() > 0 && revision <= operation.getLastRevision()) { + log.debug("丢弃过期上报 gid={} revision={} lastRevision={}", + task.getGid(), revision, operation.getLastRevision()); + continue; + } else if (operation.getOperationId() != null) { + // v2 任务必须有归属：缺 operationId 或与当前世代不符都不能改当前任务， + // 否则「删除后重建」的新世代会被旧上报污染。 + if (expectedOperationId == null || !operation.getOperationId().equals(expectedOperationId)) { + log.info("丢弃无法归属/旧世代的迟到上报 gid={} 上报={} 当前={}", + task.getGid(), expectedOperationId, operation.getOperationId()); + continue; + } + } + + String status = TaskStatus.fromReport(task.getStatus()); + if (status == null) { + log.warn("未知上报状态码 gid={} status={}", task.getGid(), task.getStatus()); + continue; + } + gallery.setProceeding(task.getProceeding()); + if (task.getName() != null) + gallery.setName(task.getName()); + boolean justCompleted = TaskStatus.COMPLETE.equals(status) + && !TaskStatus.COMPLETE.equals(gallery.getStatus()); + gallery.setStatus(status); + galleryMapper.updateGallery(gallery); + taskOperationRepository.updateReportReceipt(task.getGid(), revision, report.getReportId(), now()); + acceptedTasks.add(task); + if (justCompleted) + newlyCompleted.add(gallery); + } + return new Applied(newlyCompleted, acceptedTasks); + } + + /** + * 为迁移前创建、尚无操作记录的历史任务建立一次「世代」：优先采纳上报里的 operationId； + * 旧节点不带 operationId 时保留 null，让后续同世代的进度上报仍能被接受。 + * + *

不在这里凭空生成随机 ID：那会让第一份进度上报之后的所有旧上报都因「无法归属」被丢弃。 + * 真正的稳定 operationId 会在首次自动下发时用 {@code ensureStableOperationId} 落库， + * 之后该世代即可被严格匹配。不会覆盖任何已有任务状态。 + */ + private TaskOperation adoptGeneration(int gid, String reportedOperationId) { + TaskOperation operation = new TaskOperation(); + operation.setGid(gid); + operation.setOperationId(reportedOperationId); + operation.setState(TaskOperation.STATE_SUBMITTED); + operation.setUpdateTime(now()); + taskOperationRepository.insert(operation); + return operation; + } + + private static long now() { + return System.currentTimeMillis() / 1000; + } + + /** 供装配层解析当前有效 operationId（避免装配代码访问包私有字段）。 */ + public String currentOperationId(int gid) { + TaskOperation operation = taskOperationRepository.selectByGid(gid); + return operation == null ? null : operation.getOperationId(); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/QueryService.java b/src/main/java/com/lion/lionwebsite/Service/QueryService.java index a71f4ae..43c5529 100644 --- a/src/main/java/com/lion/lionwebsite/Service/QueryService.java +++ b/src/main/java/com/lion/lionwebsite/Service/QueryService.java @@ -54,35 +54,16 @@ public class QueryService { } Document parse = Jsoup.parse(result); - Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr"); - - ArrayList galleries = new ArrayList<>(); - ListIterator elementListIterator = elements.listIterator(); + java.util.List galleries; try { - elementListIterator.next(); //skip firstlink - }catch (NullPointerException | NoSuchElementException e){ - response.failure("没有搜索到结果"); - return response.toJSONString(); + galleries = com.lion.lionwebsite.Util.GallerySearchParser.parse(parse); + } catch (IOException | RuntimeException error) { + log.warn("上游搜索页面格式不支持: {}", error.getClass().getSimpleName()); + return response.failure("上游搜索页面格式不支持").toJSONString(); } + if (galleries.isEmpty()) return response.failure("没有搜索到结果").toJSONString(); - while (elementListIterator.hasNext()){ - Element element = elementListIterator.next(); - GalleryForQuery gallery = new GalleryForQuery(); - gallery.setType(element.child(0).text()); //type - String src = element.child(1).select("img").attr("src"); - if (src.startsWith("data")) - src = element.child(1).select("img").attr("data-src"); - src = src.replace("https://s.exhentai.org", ""); - gallery.setThumbnailUrl(src); //thumbnailSrc - gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime - gallery.setLink(element.child(2).child(0).attr("href")); //link - gallery.setGid(GalleryUtil.parseGid(gallery.getLink()) + ""); //gid - gallery.setName(element.child(2).child(0).child(0).text()); //name - gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page - galleries.add(gallery); - } - - response.success(objectMapper.valueToTree(galleries).toString()); + response.success(objectMapper.valueToTree(galleries)); Elements nextLink = parse.select("#unext"); if(nextLink.hasAttr("href")) response.set("next", nextLink.attr("href")); diff --git a/src/main/java/com/lion/lionwebsite/Service/RemoteService.java b/src/main/java/com/lion/lionwebsite/Service/RemoteService.java index f0a5c67..975cf17 100644 --- a/src/main/java/com/lion/lionwebsite/Service/RemoteService.java +++ b/src/main/java/com/lion/lionwebsite/Service/RemoteService.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.List; @Service @Data @@ -76,12 +77,131 @@ public class RemoteService { final SubscriptionStandbySnapshotService subscriptionStandbySnapshotService; + /** + * 协议 v2 上报处理器。用 setter 注入而不是构造参数，避免 RemoteService 与 + * GalleryTaskLifecycleService 之间的依赖环（后者需要 RemoteService 下发命令）。 + */ + volatile TaskReportHandler taskReportHandler; + + /** + * gid → 当前有效的下载 operationId。由生命周期服务在装配期注册（保持依赖方向单向）。 + * 未注册或返回 null 时，重投/重试退化为旧的无 operationId 命令。 + */ + volatile java.util.function.IntFunction downloadOperationIdResolver; + + /** + * gid → 下发判定（是否允许下发 + 用哪个稳定 operationId）。 + * + *

替代「能解析出 operationId 就下发」的旧语义：UNKNOWN / 预占中 / 待删除的任务 + * 必须被明确拒绝，否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。 + */ + volatile java.util.function.IntFunction dispatchResolver; + + /** + * 生命周期装配是否已经完成（handler/resolver 已绑定 + 启动恢复已执行）。 + * + *

在它为 true 之前，绝不自动重投未完成任务：启动早期还不知道哪些任务是 + * SUBMITTED、哪些是 UNKNOWN，盲目下发会制造未计量副作用。 + */ + volatile boolean lifecycleReady; + + /** + * 启动恢复是否安全完成。恢复失败时拒绝自动重投，避免在状态不明的库上继续派发。 + */ + volatile boolean recoverySafe; + + /** + * 节点是否声明了 v2 可靠任务能力。只有协商成功之后才发送类型 11..13， + * 否则继续走旧的无确认上报，保持向后兼容。 + */ + volatile boolean v2Negotiated; + + /** 本次连接的 Hello 协商结果；收到节点 Hello（或超时）后完成。 */ + volatile CompletableFuture helloNegotiation; + + /** 明确「未下发」的结果码：与 -1（发送失败/超时）区分开。 */ + public static final byte REFUSED = -3; + + /** 事务提交之后的通知/推送执行器：外部延迟（Telegram、慢浏览器）不得阻塞状态提交。 */ + final ExecutorService notificationExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "gallery-notify"); + thread.setDaemon(true); + return thread; + }); + + /** + * 协议握手的专用线程。hello 等待响应是阻塞调用，绝不能放在 Netty 事件循环上， + * 否则响应本身要由同一个事件循环投递，会直接死锁。 + */ + final ExecutorService handshakeExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "node-protocol-handshake"); + thread.setDaemon(true); + return thread; + }); + final ExecutorService subscriptionSyncExecutor = Executors.newSingleThreadExecutor(r -> { Thread thread = new Thread(r, "subscription-standby-sync"); thread.setDaemon(true); return thread; }); + /** 测试/装配用：绑定上报处理器。 */ + public void setTaskReportHandler(TaskReportHandler taskReportHandler) { + this.taskReportHandler = taskReportHandler; + } + + public void setDownloadOperationIdResolver(java.util.function.IntFunction resolver) { + this.downloadOperationIdResolver = resolver; + } + + public void setDispatchResolver(java.util.function.IntFunction resolver) { + this.dispatchResolver = resolver; + } + + /** + * 由装配层在「绑定 handler/resolver + 完成启动恢复」之后调用：标记可以安全派发并建立连接。 + * + *

初始连接不再放在 {@code @PostConstruct}：那时 handler/resolver 都还没绑定， + * 连接成功会把尚未确认的 UNKNOWN/预占任务当作普通任务重投。 + * + * @param recoverySafe 启动恢复是否成功；false 时只允许连接，不允许自动重投 + */ + public void startAfterWiring(boolean recoverySafe) { + this.recoverySafe = recoverySafe; + this.lifecycleReady = true; + initChannel(); + } + + /** 把 gid 解析成下发判定；未注册解析器或解析失败时按「拒绝下发」处理。 */ + private GalleryTaskLifecycleService.DispatchDecision resolveDispatchDecision(int gid) { + java.util.function.IntFunction resolver = dispatchResolver; + if (resolver == null) + return null; + try { + return resolver.apply(gid); + } catch (Exception e) { + log.warn("解析任务下发判定失败 gid={}", gid, e); + return null; + } + } + + private String resolveDownloadOperationId(int gid) { + java.util.function.IntFunction resolver = downloadOperationIdResolver; + if (resolver == null) + return null; + try { + return resolver.apply(gid); + } catch (Exception e) { + log.warn("解析任务 operationId 失败 gid={}", gid, e); + return null; + } + } + + /** 节点是否已完成 v2 能力协商（供健康检查/测试观察）。 */ + public boolean isReliableProtocolReady() { + return v2Negotiated; + } + /** * 处理节点上报的任务状态。原先这些 JDBC 读写直接跑在 Netty 的 IO 线程上， * N 条上报会阻塞该 IO 线程，连带拖慢心跳与请求响应；改为单线程顺序执行， @@ -120,10 +240,14 @@ public class RemoteService { @Value("${subscription.standby.sync-enabled:false}") boolean subscriptionSyncEnabled; - @PostConstruct - void init() { - initChannel(); - } + /** + * 这里刻意不再自动连接。 + * + *

原实现在 PostConstruct 里立即 initChannel()，而那一刻 TaskLifecycleWiring + * 还没绑定上报处理器、operationId 解析器，也没执行启动恢复：连接成功后会马上把 + * 未完成任务（含 UNKNOWN/预占）当作普通任务重投给节点。 + * 现在改为由装配层在完成绑定与恢复后调用 startAfterWiring(boolean)。 + */ public boolean initChannel(){ if (stopping || !connecting.compareAndSet(false, true)) @@ -175,12 +299,32 @@ public class RemoteService { channel = channelFuture.channel(); closeMonitorSocket(); channel.writeAndFlush(new IdentityMessage("lionwebsite")); - - //子节点上线时，发送未完成的任务 - resetUndone(); - // 节点刚上线时可能没有任何快照（例如刚重启），必须推一次； - // 这同时会刷新节点「最近收到快照」的时刻，避免内容未变时被判过期。 - requestSubscriptionSync(); + // 协议 v2：先发旧的 Identity，再发 hello 做能力协商。 + // 协商完成之前不允许下发任何任务，也不允许自动重投未完成任务。 + CompletableFuture negotiation = queueHelloHandshake(); + // 在协商线程上等待结果，再决定重投；绝不阻塞 Netty 事件循环。 + handshakeExecutor.execute(() -> { + boolean reliable = awaitNegotiation(negotiation); + // 订阅快照与任务重投都要求生命周期装配已完成；装配前只保持连接。 + requestSubscriptionSync(); + if (!reliable) { + log.info("节点未确认 v2 能力，跳过自动重投未完成任务"); + return; + } + if (!lifecycleReady) { + log.info("生命周期装配未就绪，跳过自动重投未完成任务"); + return; + } + if (!recoverySafe) { + log.error("启动恢复未安全完成，拒绝自动重投未完成任务"); + return; + } + try { + resetUndone(); + } catch (RuntimeException e) { + log.warn("重投未完成任务失败", e); + } + }); return true; }catch (Exception e){ log.error("connect node failed, wait for node back online", e); @@ -297,6 +441,8 @@ public class RemoteService { failPendingRequests(); if (channel != null) channel.close(); statusApplyExecutor.shutdownNow(); + notificationExecutor.shutdownNow(); + handshakeExecutor.shutdownNow(); subscriptionSyncExecutor.shutdownNow(); networkGroup.shutdownGracefully(); eventLoopGroup.shutdownGracefully(); @@ -315,7 +461,11 @@ public class RemoteService { } public void resetUndone(){ - if (channelFuture.channel() == null || !channelFuture.channel().isActive()) + // 只有「连接在线 + v2 已协商 + 生命周期装配完成 + 启动恢复安全完成」才允许批量重投； + // 任何一项不满足都说明还不知道任务真实状态，盲目下发会制造未计量副作用。 + if (channelFuture == null || channelFuture.channel() == null || !channelFuture.channel().isActive()) + return; + if (!v2Negotiated || !lifecycleReady || !recoverySafe) return; Gallery[] galleries = galleryMapper.selectUnDoneGalleries(); @@ -328,15 +478,135 @@ public class RemoteService { } public byte addGalleryToQueue(Gallery gallery){ + GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid()); + if (decision != null) { + // 明确拒绝（UNKNOWN/预占/待删除等）：绝不能退化为无 operationId 的旧命令。 + if (!decision.dispatchable()) + return REFUSED; + return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(), + gallery.getGid(), gallery.getName()); + } + // 兼容：装配层未注册判定解析器时（如单元测试），退回只按 operationId 解析的旧行为。 + String operationId = resolveDownloadOperationId(gallery.getGid()); + if (operationId != null) + return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, operationId, gallery.getGid(), gallery.getName()); + return legacyDownloadCommand(gallery.getGid(), gallery.getName()); + } + + /** 旧的无 operationId 下载命令，供未启用 v2 或无法解析 operationId 时退避使用。 */ + private byte legacyDownloadCommand(int gid, String name) { GalleryTask galleryTask = new GalleryTask(); - galleryTask.setGid(gallery.getGid()); - galleryTask.setName(gallery.getName()); + galleryTask.setGid(gid); + galleryTask.setName(name); DownloadPostMessage message = new DownloadPostMessage(); message.setGalleryTask(galleryTask); return sendRequest(message, 10, TimeUnit.SECONDS); } + /** + * 在专用线程上执行 hello 协商，并返回一个在「收到节点 Hello」或「等待超时」后完成的 future。 + * + *

节点的应答是 Hello 而不是 ResponseMessage，因此绝不能走 sendRequest 的响应等待器 + * （那会白等 10 秒超时，排在后面的自动重投也只能 10 秒后才开始）。 + * 这里直接 writeAndFlush，并让 channelRead 的 Hello 分支完成 future； + * 超时用 completeOnTimeout 处理，写失败立刻完成。 + */ + private CompletableFuture queueHelloHandshake() { + // 10 秒内没等到节点 Hello 就按「未协商」完成，避免后续重投被无限期挂起。 + CompletableFuture result = new CompletableFuture() + .completeOnTimeout(false, 10, TimeUnit.SECONDS); + helloNegotiation = result; + if (stopping) { + result.complete(false); + return result; + } + try { + handshakeExecutor.execute(() -> { + v2Negotiated = false; + try { + Channel target = channel; + if (target == null || !target.isActive()) { + result.complete(false); + return; + } + // Hello 不写入 promiseHashMap：节点回的是 Hello，没有 ResponseMessage 等待者。 + target.writeAndFlush(new ProtocolHelloMessage()).addListener(future -> { + if (!future.isSuccess()) { + if (!stopping) + log.warn("协议协商发送失败: {}", future.cause() == null + ? "unknown" : future.cause().getClass().getSimpleName()); + result.complete(false); + } + }); + } catch (RuntimeException e) { + log.warn("协议协商失败", e); + result.complete(false); + } + }); + } catch (java.util.concurrent.RejectedExecutionException e) { + if (!stopping) log.warn("安排协议协商失败", e); + result.complete(false); + } + return result; + } + + /** 等待协商结果；超时或中断都按「未协商」处理（宁可不下发，也不盲目重投）。 */ + boolean awaitNegotiation(CompletableFuture negotiation) { + if (negotiation == null) + return v2Negotiated; + try { + // future 自身已在 10 秒时按「未协商」完成，这里留少量余量即可。 + return negotiation.get(12, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (Exception e) { + return false; + } + } + + /** + * 下发一条带稳定 operationId 的任务命令；未协商 v2 时退化为旧的 DownloadPostMessage， + * 保证新旧节点组合仍可用（旧节点上无法保证可靠 ACK，这是已知的兼容边界）。 + */ + public byte sendTaskCommand(String action, String operationId, int gid, String name) { + if (!v2Negotiated) + return TaskCommandMessage.ACTION_DELETE.equals(action) + ? legacyDeleteCommand(gid, name) + : legacyDownloadCommand(gid, name); + TaskCommandMessage command = new TaskCommandMessage(); + command.setOperationId(operationId); + command.setAction(action); + command.setGid(gid); + command.setName(name); + return sendRequest(command, 10, TimeUnit.SECONDS); + } + + /** 删除命令：带独立稳定 operationId，重试复用同一个 ID。 */ + public byte sendDeleteCommand(String operationId, int gid, String name) { + if (!v2Negotiated) + return legacyDeleteCommand(gid, name); + TaskCommandMessage command = new TaskCommandMessage(); + command.setOperationId(operationId); + command.setAction(TaskCommandMessage.ACTION_DELETE); + command.setGid(gid); + command.setName(name); + return sendRequest(command, 10, TimeUnit.SECONDS); + } + + /** 旧的无 operationId 删除命令，供未协商 v2 时退避使用。 */ + private byte legacyDeleteCommand(int gid, String name) { + Gallery gallery = new Gallery(); + gallery.setGid(gid); + gallery.setName(name); + return deleteGallery(gallery); + } + public RetryResult retryGallery(Gallery gallery){ + // 提交结果未知或被删除的任务不能重试：节点可能已经把未计量任务当成新下载。 + GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid()); + if (decision != null && !decision.dispatchable()) + return new RetryResult(false, "该任务提交结果未知或正在删除，需要人工对账后再重试"); CompletableFuture statusFuture = new CompletableFuture<>(); retryStatusWaiters.computeIfAbsent(gallery.getGid(), ignored -> new CopyOnWriteArrayList<>()) .add(statusFuture); @@ -447,6 +717,18 @@ public class RemoteService { } }); } + else if(msg instanceof ProtocolHelloMessage hello) { + // 节点声明了 v2 可靠任务能力；此后才允许使用类型 11..13。 + v2Negotiated = hello.supportsReliableTasks(); + log.info("节点协议协商完成 v2={} version={}", v2Negotiated, hello.getProtocolVersion()); + CompletableFuture negotiation = helloNegotiation; + if (negotiation != null) + negotiation.complete(v2Negotiated); + } + else if(msg instanceof TaskReportMessage report) { + // 事务提交与 ACK 都放在单线程消费队列里顺序执行，避免并发写库。 + statusApplyExecutor.execute(() -> consumeV2Report(report)); + } else if(msg instanceof ResponseMessage rsm) { Promise promise = promiseHashMap.remove(rsm.messageId); if(promise != null) @@ -456,6 +738,55 @@ public class RemoteService { } } + /** + * 消费协议 v2 上报：短事务提交 → 事务成功后回 ACK → 再用独立执行器做通知/推送。 + * + *

顺序很关键：数据库失败时既不能回 ACK，也不能清任务；慢通知不能阻塞这里， + * 否则会拖住后续上报的提交。 + */ + private void consumeV2Report(TaskReportMessage report) { + try { + TaskReportHandler handler = taskReportHandler; + if (handler == null) { + log.warn("收到 v2 上报但未装配处理器，暂不回执 reportId={}", report.getReportId()); + return; + } + TaskReportHandler.CommittedReport committed = handler.apply(report); + if (committed == null) { + // 提交失败：不回 ACK，节点保留快照后重投。 + log.warn("v2 上报未提交，等待节点重投 reportId={}", report.getReportId()); + return; + } + // 提交成功之后才回执，让节点清理快照。 + Channel target = channel; + if (target != null && target.isActive()) + target.writeAndFlush(new TaskReportAckMessage(committed.reportId(), TaskReportAckMessage.RESULT_SUCCESS)); + dispatchNotifications(report, committed); + } catch (Exception e) { + log.warn("处理 v2 上报失败 reportId={}", report.getReportId(), e); + } + } + + /** 通知/推送在独立执行器里执行，外部延迟不会阻塞状态提交。 */ + private void dispatchNotifications(TaskReportMessage report, TaskReportHandler.CommittedReport committed) { + // 只推送事务里真正被接受的更新（含下载中/压缩中等中间状态）； + // 过期、旧世代、待删除任务已被过滤掉，不会把它们带到前端。 + GalleryTask[] accepted = committed.acceptedTasks() == null + ? new GalleryTask[0] + : committed.acceptedTasks().toArray(new GalleryTask[0]); + List newlyCompleted = committed.newlyCompleted(); + notificationExecutor.execute(() -> { + try { + for (Gallery gallery : newlyCompleted) + pushService.downloadComplete(gallery); + if (accepted.length > 0) + webSocketService.updateTaskProcessing(accepted); + } catch (Exception e) { + log.warn("任务完成通知/推送失败", e); + } + }); + } + /** 把节点上报的任务状态写回库里并推送给前端，按上报顺序逐个处理。 */ private void applyReportedStatus(GalleryTask[] galleryTasks) { for (GalleryTask galleryTask : galleryTasks) { diff --git a/src/main/java/com/lion/lionwebsite/Service/SubService.java b/src/main/java/com/lion/lionwebsite/Service/SubService.java index aaefcfd..ae01413 100644 --- a/src/main/java/com/lion/lionwebsite/Service/SubService.java +++ b/src/main/java/com/lion/lionwebsite/Service/SubService.java @@ -17,6 +17,10 @@ import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.select.Elements; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.transaction.support.TransactionTemplate; +import jakarta.annotation.PreDestroy; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -39,6 +43,10 @@ import java.util.function.Supplier; final RemoteService remoteService; final SubscriptionStateCoordinator stateCoordinator; + @Autowired + @Qualifier("mainTransactionTemplate") + TransactionTemplate transactions; + /** * 归属地查询是同步外呼（最长十几秒），且失败只影响记录里的一列展示。 * 若放在订阅分发路径上，用户取一次订阅就得先等它完成，因此改为后台补齐。 @@ -48,11 +56,12 @@ import java.util.function.Supplier; Executor locationExecutor = newFixedLocationExecutor(); private static ExecutorService newFixedLocationExecutor() { - return Executors.newFixedThreadPool(2, r -> { + return new java.util.concurrent.ThreadPoolExecutor(2, 2, 0, java.util.concurrent.TimeUnit.MILLISECONDS, + new java.util.concurrent.ArrayBlockingQueue<>(128), r -> { Thread thread = new Thread(r, "sub-location-resolver"); thread.setDaemon(true); return thread; - }); + }, new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()); } public String insertSubscriptionAccount(String name, String upstreamKey, boolean filterHighMultiplier, boolean enabled) { @@ -211,6 +220,16 @@ import java.util.function.Supplier; } public void updateSub(HttpServletResponse response, HttpServletRequest request, String client, String key) { + Lock lock = stateCoordinator.readLock(); + lock.lock(); + try { + distributeSubscription(response, request, client, key); + } finally { + lock.unlock(); + } + } + + private void distributeSubscription(HttpServletResponse response, HttpServletRequest request, String client, String key) { if (key == null || client == null) return; SubBind subBind = subMapper.selectSubBind(key); @@ -221,7 +240,7 @@ import java.util.function.Supplier; String ip = resolveClientIp(request); String ua = request.getHeader("User-Agent"); if (ua == null) return; - recordUpdate(subBind.getUser(), ip, ua); + recordUpdate(subBind, ip, ua); if (!"v2".equals(client) && !"cat".equals(client)) { sendStatus(response, HttpServletResponse.SC_BAD_REQUEST, "client error"); return; @@ -259,24 +278,34 @@ import java.util.function.Supplier; || "0:0:0:0:0:0:0:1".equals(address); } - private void recordUpdate(String user, String ip, String ua) { + private void recordUpdate(SubBind binding, String ip, String ua) { + String user = binding.getUser(); // 先落一条 unknown 记录并立即返回，归属地随后在后台补齐； // 这样取订阅的延迟不再取决于第三方定位站点的响应时间。 - subMapper.insertSubUpdateRecord(new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown")); + SubUpdateRecord record = new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown"); + record.setUserId(binding.getUserId()); + subMapper.insertSubUpdateRecord(record); if (subMapper.selectUpdateRecordCount(user) > 10) subMapper.deleteSubUpdateRecordById(subMapper.selectMinUpdateRecordId(user)); - locationExecutor.execute(() -> enrichLocation(user, ip)); + // Capture the generated row ID: another request may arrive before the lookup finishes. + if (record.getId() > 0) { + try { + locationExecutor.execute(() -> enrichLocation(record.getId(), ip)); + } catch (java.util.concurrent.RejectedExecutionException busy) { + log.debug("位置查询队列已满，保留 unknown recordId={}", record.getId()); + } + } } - /** 后台补齐最近一条记录的归属地；任何失败都只记日志，不影响已落库的记录。 */ - private void enrichLocation(String user, String ip) { + /** 后台按记录 ID 补齐归属地；任何失败都只记日志，不影响已落库的记录。 */ + private void enrichLocation(int recordId, String ip) { try { String location = resolveLocation(ip); if (location == null || location.isBlank() || "unknown".equals(location)) return; - subMapper.updateLatestSubUpdateRecordLocation(user, location); + subMapper.updateSubUpdateRecordLocation(recordId, location); } catch (Exception e) { - log.debug("补齐订阅更新记录归属地失败 user={}", user, e); + log.debug("补齐订阅更新记录归属地失败 recordId={}", recordId, e); } } @@ -310,20 +339,26 @@ import java.util.function.Supplier; Lock stateLock = stateCoordinator.writeLock(); stateLock.lock(); try { - return action.get(); + return transactions == null ? action.get() : transactions.execute(ignored -> action.get()); } finally { stateLock.unlock(); } } + @PreDestroy + void shutdownLocationResolver() { + if (locationExecutor instanceof ExecutorService service) + service.shutdownNow(); + } + private SubscriptionAccount firstEnabledAccount() { return subMapper.selectAllSubscriptionAccounts().stream().filter(SubscriptionAccount::isEnabled).findFirst().orElse(null); } - private String accountJson(SubscriptionAccount account) { + private tools.jackson.databind.JsonNode accountJson(SubscriptionAccount account) { account.setV2Url(refreshService.v2Url(account)); account.setClashUrl(refreshService.clashUrl(account)); - return CustomUtil.objectMapper.valueToTree(account).toString(); + return CustomUtil.objectMapper.valueToTree(account); } private static void sendStatus(HttpServletResponse response, int status, String message) { diff --git a/src/main/java/com/lion/lionwebsite/Service/SubscriptionRefreshService.java b/src/main/java/com/lion/lionwebsite/Service/SubscriptionRefreshService.java index 2cb1bae..af6195b 100644 --- a/src/main/java/com/lion/lionwebsite/Service/SubscriptionRefreshService.java +++ b/src/main/java/com/lion/lionwebsite/Service/SubscriptionRefreshService.java @@ -2,7 +2,9 @@ package com.lion.lionwebsite.Service; import com.lion.lionwebsite.Dao.normal.SubMapper; import com.lion.lionwebsite.Domain.SubscriptionAccount; +import com.lion.lionwebsite.Util.ClashSubscriptionParser; import com.lion.lionwebsite.Util.SubscriptionClientProfile; +import com.lion.lionwebsite.Util.SubscriptionCacheStore; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.hc.client5.http.classic.methods.HttpGet; @@ -57,6 +59,12 @@ public class SubscriptionRefreshService { /** 时间源，测试可替换以固定成功时刻。 */ Clock clock = Clock.systemDefaultZone(); + /** + * 分代缓存存储。默认按 {@link #cacheRoot} 懒构造；测试可覆写为受控子类来模拟写入失败。 + * 保持为普通字段而不是构造参数，是为了不改动既有两参构造器（Spring 与老测试都在用）。 + */ + SubscriptionCacheStore cacheStore; + private long refreshSequence; // Accessed only while holding the coordinator write lock. private final Map latestRefresh = new HashMap<>(); @@ -78,18 +86,19 @@ public class SubscriptionRefreshService { // Network access and parsing never hold the shared subscription lock. try { + boolean filter = account.isFilterHighMultiplier(); String v2 = processV2(firstLine(download(v2Url(account), SubscriptionClientProfile.forAccount(accountId, false))), - account.isFilterHighMultiplier(), highMultiplierThreshold); - List clash = processClash(download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)), - account.isFilterHighMultiplier(), highMultiplierThreshold); + filter, highMultiplierThreshold); + // 按 YAML 语义过滤高倍节点并同步清理 proxy-groups 引用，不再靠行缩进猜测。 + String clash = ClashSubscriptionParser.transform( + download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)), + filter, highMultiplierThreshold); stateLock.lock(); try { if (!isCurrent(account, version)) return false; - Path dir = Paths.get(cacheRoot, String.valueOf(accountId)); - Files.createDirectories(dir); - atomicWrite(dir.resolve("v2ray.txt"), v2.getBytes(StandardCharsets.UTF_8)); - atomicWrite(dir.resolve("clash.yaml"), String.join("\n", clash).concat("\n").getBytes(StandardCharsets.UTF_8)); + // 两份内容写进同一分代，校验通过后指针最后发布：任一步失败都保留上次完整版本。 + store().publish(accountId, v2.getBytes(StandardCharsets.UTF_8), clash.getBytes(StandardCharsets.UTF_8)); subMapper.markSubscriptionRefreshSuccess(accountId, clock.millis()); return true; } finally { @@ -162,56 +171,6 @@ public class SubscriptionRefreshService { return URLDecoder.decode(node.substring(hash + 1), StandardCharsets.UTF_8); } - private List processClash(List source, boolean filter, double threshold) { - Set removed = new HashSet<>(); - List result = new ArrayList<>(); - boolean inProxies = false; - boolean skipNode = false; - for (String line : source) { - if (line.equals("proxies:")) { - inProxies = true; - skipNode = false; - result.add(line); - continue; - } - if (line.equals("proxy-groups:")) { - inProxies = false; - skipNode = false; - result.add(line); - continue; - } - if (inProxies && line.matches("^\\s{2}-\\s+name:.*")) { - String name = clashName(line); - skipNode = filter && isHigh(name, threshold); - if (skipNode) - removed.add(name); - else - result.add(line); - continue; - } - if (skipNode) - continue; - if (!inProxies && !removed.isEmpty() && line.trim().startsWith("- ")) { - String ref = line.trim().substring(2).trim(); - if (removed.contains(unquote(ref))) - continue; - } - result.add(line); - } - return result; - } - - private static String clashName(String line) { - int index = line.indexOf("name:"); - return unquote(line.substring(index + 5).trim()); - } - - private static String unquote(String value) { - if (value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")))) - return value.substring(1, value.length() - 1); - return value; - } - private static boolean isHigh(String name, double threshold) { Matcher matcher = MULTIPLIER.matcher(name); return matcher.find() && Double.parseDouble(matcher.group(1)) > threshold; @@ -251,22 +210,17 @@ public class SubscriptionRefreshService { } } - private static void atomicWrite(Path target, byte[] data) throws IOException { - Path temp = target.resolveSibling(target.getFileName() + ".tmp"); - Files.write(temp, data); - try { - Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException e) { - Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING); - } - } - + /** + * 当前应分发的缓存文件。只在一整个版本有效时返回真实路径；否则返回一个保证不存在的 + * 哨兵路径，使分发端按「订阅暂不可用」处理，而不是发出半份旧缓存。 + */ public Path cachedPath(Integer accountId, String client) { - return Paths.get(cacheRoot, String.valueOf(accountId), client.equals("v2") ? "v2ray.txt" : "clash.yaml"); + SubscriptionCacheStore store = store(); + return store.resolve(accountId, client).orElseGet(() -> store.unavailablePath(accountId, client)); } public boolean hasCompleteCache(Integer accountId) { - return Files.isRegularFile(cachedPath(accountId, "v2")) && Files.isRegularFile(cachedPath(accountId, "cat")); + return store().hasComplete(accountId); } public void invalidateCache(Integer accountId) { @@ -274,12 +228,15 @@ public class SubscriptionRefreshService { stateLock.lock(); try { latestRefresh.remove(accountId); // In-flight responses must not restore invalidated content. - Files.deleteIfExists(cachedPath(accountId, "v2")); - Files.deleteIfExists(cachedPath(accountId, "cat")); - } catch (IOException e) { - log.warn("清理失效订阅缓存失败 accountId={}", accountId, e); + store().invalidate(accountId); } finally { stateLock.unlock(); } } + + private synchronized SubscriptionCacheStore store() { + if (cacheStore == null) + cacheStore = new SubscriptionCacheStore(cacheRoot); + return cacheStore; + } } diff --git a/src/main/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotService.java b/src/main/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotService.java index cc71d4d..0843b0c 100644 --- a/src/main/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotService.java +++ b/src/main/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotService.java @@ -6,6 +6,7 @@ import com.lion.lionwebsite.Domain.SubBind; import com.lion.lionwebsite.Domain.SubscriptionAccount; import com.lion.lionwebsite.Message.*; import com.lion.lionwebsite.Util.CustomUtil; +import com.lion.lionwebsite.Util.SubscriptionCacheStore; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -16,8 +17,6 @@ import javax.crypto.spec.SecretKeySpec; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.security.MessageDigest; import java.util.*; import java.util.zip.GZIPOutputStream; @@ -39,6 +38,12 @@ public class SubscriptionStandbySnapshotService { @Value("${subscription.cache-root:sub/accounts}") String cacheRoot; + /** + * 与 {@link SubscriptionRefreshService} 共用的分代缓存解析器。 + * 测试可直接覆写；未设置时按 {@link #cacheRoot} 懒构造。 + */ + SubscriptionCacheStore cacheStore; + @PostConstruct void validateConfiguration() { if (syncEnabled && (syncSecret == null || syncSecret.isBlank())) @@ -51,12 +56,14 @@ public class SubscriptionStandbySnapshotService { try { Map accountMap = new HashMap<>(); for (SubscriptionAccount account : subMapper.selectAllSubscriptionAccounts()) { - Path v2Path = cachedPath(account.getId(), "v2"); - Path clashPath = cachedPath(account.getId(), "cat"); - if (!account.isEnabled() || !Files.isRegularFile(v2Path) || !Files.isRegularFile(clashPath)) + if (!account.isEnabled()) continue; - byte[] v2 = Files.readAllBytes(v2Path); - byte[] clash = Files.readAllBytes(clashPath); + // 从同一分代读出成对内容，保证快照里两份文件同版本；无完整版本则跳过该账号。 + Optional pair = store().read(account.getId()); + if (pair.isEmpty()) + continue; + byte[] v2 = pair.get().v2(); + byte[] clash = pair.get().clash(); SubscriptionAccountSnapshot snapshot = new SubscriptionAccountSnapshot(); snapshot.setAccountId(account.getId()); snapshot.setEnabled(true); @@ -132,7 +139,9 @@ public class SubscriptionStandbySnapshotService { private static String hex(byte[] input) { return HexFormat.of().formatHex(input); } - private Path cachedPath(Integer accountId, String client) { - return Path.of(cacheRoot, String.valueOf(accountId), "v2".equals(client) ? "v2ray.txt" : "clash.yaml"); + private synchronized SubscriptionCacheStore store() { + if (cacheStore == null) + cacheStore = new SubscriptionCacheStore(cacheRoot); + return cacheStore; } } diff --git a/src/main/java/com/lion/lionwebsite/Service/TaskProtocol.java b/src/main/java/com/lion/lionwebsite/Service/TaskProtocol.java new file mode 100644 index 0000000..4953230 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/TaskProtocol.java @@ -0,0 +1,23 @@ +package com.lion.lionwebsite.Service; + +import java.util.UUID; + +/** 协议 v2 的操作 ID 生成与解析。稳定 ID 是「节点重发/重试不重复执行副作用」的依据。 */ +public final class TaskProtocol { + + private TaskProtocol() { + } + + /** 生成一个新的稳定操作 ID（下载或删除），形如 {@code op-}。 */ + public static String newOperationId(String action) { + return action + "-" + UUID.randomUUID(); + } + + public static String newDownloadOperationId() { + return newOperationId("dl"); + } + + public static String newDeleteOperationId() { + return newOperationId("del"); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/TaskReportHandler.java b/src/main/java/com/lion/lionwebsite/Service/TaskReportHandler.java new file mode 100644 index 0000000..835b1f8 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/TaskReportHandler.java @@ -0,0 +1,37 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Domain.Gallery; +import com.lion.lionwebsite.Domain.GalleryTask; +import com.lion.lionwebsite.Message.TaskReportMessage; + +import java.util.List; + +/** + * 协议 v2 状态上报的消费入口。 + * + *

定义成接口是为了断开依赖环：RemoteService 需要把上报交给生命周期服务， + * 而生命周期服务需要有 RemoteService 才能下发命令。RemoteService 通过可选的 setter + * 注入本接口（见 {@code RemoteService#setTaskReportHandler}），因此构造期没有环。 + */ +public interface TaskReportHandler { + + /** + * 提交一份上报。 + * + * @return 已成功提交（可回 ACK）时返回提交结果；数据库失败等「不可回 ACK」的情况返回 null。 + * 返回 null 时节点会保留快照并稍后重投，这正是「失败写入不得清任务」的要求。 + */ + CommittedReport apply(TaskReportMessage report); + + /** + * 已提交的结果。 + * + * @param reportId 本次上报 ID（回执用） + * @param newlyCompleted 本次「首次」完成的任务，通知与 WebSocket 由独立执行器在事务之后处理， + * 因此慢通知不会阻塞数据库消费线程；重复上报不会重复出现在这里 + * @param acceptedTasks 事务内实际被接受的进度更新（含下载中/压缩中等中间状态，已排除过期、 + * 旧世代与待删除任务），WebSocket 只推这一批，避免前端看到被丢弃的迟到状态 + */ + record CommittedReport(String reportId, List newlyCompleted, List acceptedTasks) { + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/TaskStatus.java b/src/main/java/com/lion/lionwebsite/Service/TaskStatus.java new file mode 100644 index 0000000..816f408 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/TaskStatus.java @@ -0,0 +1,45 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Domain.GalleryTask; + +/** + * 任务业务状态码到展示文案的唯一映射点。 + * + *

状态以稳定代码表达，中文文案只出现在展示层（WebSocket/列表）， + * 避免把「失败/待重试/结果未知」塞进成功失败提示字符串。 + */ +public final class TaskStatus { + + public static final String SUBMITTED = "已提交"; + /** 已登记创建意图、尚未确认上游提交；用于让任务列表在提交期间也能看到占位任务。 */ + public static final String PENDING_SUBMIT = "提交中"; + public static final String DOWNLOADING = "下载中"; + public static final String WAITING_COMPRESS = "等待压缩"; + public static final String COMPRESSING = "压缩中"; + public static final String COMPLETE = "下载完成"; + public static final String UNKNOWN = "提交未知"; + public static final String DELETE_PENDING = "待删除"; + public static final String DELETE_FAILED = "删除失败"; + + private TaskStatus() { + } + + /** + * 节点上报的进度状态映射为业务文案；未知状态码返回 null，由调用方忽略。 + * + *

GalleryTask 里的状态码是可变 static byte，不是编译期常量，因此不能用 switch case； + * 这里用整数比较，避免为改常量引入风险。 + */ + public static String fromReport(byte reportStatus) { + int status = reportStatus; + if (status == GalleryTask.DOWNLOADING) + return DOWNLOADING; + if (status == GalleryTask.DOWNLOAD_COMPLETE) + return WAITING_COMPRESS; + if (status == GalleryTask.COMPRESSING) + return COMPRESSING; + if (status == GalleryTask.COMPRESS_COMPLETE) + return COMPLETE; + return null; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/UpstreamGateway.java b/src/main/java/com/lion/lionwebsite/Service/UpstreamGateway.java new file mode 100644 index 0000000..c28da49 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Service/UpstreamGateway.java @@ -0,0 +1,47 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Domain.Gallery; +import com.lion.lionwebsite.Exception.ResolutionNotMatchException; +import com.lion.lionwebsite.Util.GalleryUtil; +import com.lion.lionwebsite.Util.PreparedGallery; +import org.springframework.stereotype.Component; + +import java.io.IOException; + +/** + * 上游图站的读写边界。 + * + *

分成两步是协议 v2 的核心约束： + *

    + *
  1. {@link #prepare} 只做读取（页面、分辨率），绝不产生下载副作用；
    2. + *
  2. 调用方必须在持久化 SUBMITTING 之后才调用 {@link #submit}，因为那一步才真正 POST。
    3. + *
+ * + *

拆成接口是为了让生命周期服务可以在测试里注入假实现，也方便后续把抓取/解析/提交 + * 进一步分开；默认实现 {@link GalleryUtilAdapter} 直接转发到现有 GalleryUtil。 + */ +public interface UpstreamGateway { + + /** 只读解析：失败返回 null 表示链接无效或页面不可用，不产生任何副作用。 */ + PreparedGallery prepare(String link) throws IOException; + + /** 真正提交下载请求。抛异常表示结果未知，由调用方落 UNKNOWN，不做自动重发。 */ + Gallery submit(PreparedGallery prepared, String targetResolution) + throws IOException, ResolutionNotMatchException; + + /** 默认实现：转发现有 GalleryUtil 的拆分方法。 */ + @Component + class GalleryUtilAdapter implements UpstreamGateway { + + @Override + public PreparedGallery prepare(String link) throws IOException { + return GalleryUtil.prepare(link); + } + + @Override + public Gallery submit(PreparedGallery prepared, String targetResolution) + throws IOException, ResolutionNotMatchException { + return GalleryUtil.submit(prepared, targetResolution); + } + } +} diff --git a/src/main/java/com/lion/lionwebsite/Service/UserService.java b/src/main/java/com/lion/lionwebsite/Service/UserService.java index f403b2a..fe369b9 100644 --- a/src/main/java/com/lion/lionwebsite/Service/UserService.java +++ b/src/main/java/com/lion/lionwebsite/Service/UserService.java @@ -3,6 +3,7 @@ package com.lion.lionwebsite.Service; import com.lion.lionwebsite.Dao.normal.CollectMapper; import com.lion.lionwebsite.Dao.normal.GalleryMapper; import com.lion.lionwebsite.Dao.normal.UserMapper; +import com.lion.lionwebsite.Dao.normal.SubMapper; import com.lion.lionwebsite.Domain.User; import com.lion.lionwebsite.Interceptor.TaskHandlerInterceptor; import com.lion.lionwebsite.Util.CustomUtil; @@ -10,8 +11,13 @@ import com.lion.lionwebsite.Util.Response; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.transaction.support.TransactionTemplate; import java.util.ArrayList; +import java.util.concurrent.locks.Lock; +import java.util.function.Supplier; import static com.lion.lionwebsite.Util.CustomUtil.objectMapper; @@ -28,6 +34,11 @@ public class UserService{ final TaskHandlerInterceptor taskHandlerInterceptor; + @Autowired SubMapper subMapper; + @Autowired SubscriptionStateCoordinator subscriptionState; + @Autowired RemoteService remoteService; + @Autowired @Qualifier("mainTransactionTemplate") TransactionTemplate transactions; + public String addAuthCode(String targetAuthCode, String people) { Response response = Response.generateResponse(); @@ -66,12 +77,22 @@ public class UserService{ Response response = Response.generateResponse(); try{ - if(userMapper.isExist(targetAuthCode) != 0){ - userMapper.updateUsername(targetAuthCode, newUsername); - response.success("修改成功"); - }else { - response.failure("授权码不存在"); - } + return coordinated(() -> { + if (userMapper.isExist(targetAuthCode) == 0) + return response.failure("授权码不存在").toJSONString(); + if (newUsername == null || newUsername.isBlank()) + return response.failure("用户名不能为空").toJSONString(); + String name = newUsername.trim(); + User current = userMapper.selectUserByAuthCode(targetAuthCode); + User existing = userMapper.selectUserByUsername(name); + if (existing != null && (current == null || existing.getId() != current.getId())) + return response.failure("用户名已存在").toJSONString(); + userMapper.updateUsername(targetAuthCode, name); + if (subMapper != null && current != null) + subMapper.renameBoundUser(current.getId(), current.getUsername(), name); + if (remoteService != null) remoteService.requestSubscriptionSync(); + return response.success("修改成功").toJSONString(); + }); }catch (Exception e){ log.error("修改用户名失败", e); response.failure("修改失败"); @@ -85,17 +106,22 @@ public class UserService{ if(userMapper.isExist(targetAuthCode) != 0) try{ - User user = userMapper.selectUserByAuthCode(targetAuthCode); - - //修改下载人以及取消收藏 - ArrayList galleries = collectMapper.selectGidByCollector(user.getId()); - for(Integer gallery: galleries) - collectMapper.disCollect(gallery, user.getId()); - galleryMapper.updateGalleryDownloader(3, user.getId()); - - userMapper.deleteUserByAuthCode(targetAuthCode); + coordinated(() -> { + User user = userMapper.selectUserByAuthCode(targetAuthCode); + ArrayList galleries = collectMapper.selectGidByCollector(user.getId()); + for(Integer gallery: galleries) + collectMapper.disCollect(gallery, user.getId()); + galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, user.getId()); + if (subMapper != null) { + subMapper.deleteSubUpdateRecord(user.getUsername()); + subMapper.deleteSubBind(user.getUsername()); + } + userMapper.deleteUserByAuthCode(targetAuthCode); + return "deleted"; + }); response.success("删除成功"); taskHandlerInterceptor.updateAuthCodes(); + if (remoteService != null) remoteService.requestSubscriptionSync(); }catch (Exception e){ log.error("删除授权码失败", e); response.failure("删除失败"); @@ -108,27 +134,29 @@ public class UserService{ } public String alterStatus(String AuthCode, boolean isEnable){ - Response response = Response.generateResponse(); - - if(userMapper.isExist(AuthCode) == 1){ - int id = userMapper.selectUserByAuthCode(AuthCode).getId(); - userMapper.updateIsEnableById(id, isEnable); - galleryMapper.updateGalleryDownloader(3, id); - response.success("修改用户状态成功"); - taskHandlerInterceptor.updateAuthCodes(); + try { + String result = coordinated(() -> { + if (userMapper.isExist(AuthCode) != 1) + return Response._failure("该用户不存在"); + int id = userMapper.selectUserByAuthCode(AuthCode).getId(); + userMapper.updateIsEnableById(id, isEnable); + if (!isEnable) galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, id); + return Response._success("修改用户状态成功"); + }); + // Refresh the in-memory identity view only after the database commits. + if (result.contains("\"result\":\"success\"")) taskHandlerInterceptor.updateAuthCodes(); + return result; + } catch (RuntimeException error) { + log.warn("修改用户状态失败: {}", error.getClass().getSimpleName()); + return Response._failure("修改用户状态失败"); } - else{ - response.failure("该用户不存在"); - } - - return response.toJSONString(); } public String getAllUser() { Response response = Response.generateResponse(); User[] users = userMapper.selectAllUser(); - response.success(objectMapper.valueToTree(users).toString()); + response.success(objectMapper.valueToTree(users)); return response.toJSONString(); } @@ -150,4 +178,14 @@ public class UserService{ User user = userMapper.selectUserByAuthCode(AuthCode); return user != null && user.getId() == ADMIN_USER_ID; } + + private String coordinated(Supplier action) { + Lock lock = subscriptionState == null ? null : subscriptionState.writeLock(); + if (lock != null) lock.lock(); + try { + return transactions == null ? action.get() : transactions.execute(ignored -> action.get()); + } finally { + if (lock != null) lock.unlock(); + } + } } diff --git a/src/main/java/com/lion/lionwebsite/Util/ClashSubscriptionParser.java b/src/main/java/com/lion/lionwebsite/Util/ClashSubscriptionParser.java new file mode 100644 index 0000000..9bd6521 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/ClashSubscriptionParser.java @@ -0,0 +1,45 @@ +package com.lion.lionwebsite.Util; + +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.DumperOptions; +import java.util.*; +import java.util.regex.Pattern; + +/** YAML semantics rather than indentation determine nodes and references. */ +public final class ClashSubscriptionParser { + private static final Pattern MULTIPLIER = Pattern.compile("(\\d+(?:\\.\\d+)?)x\\s*$", Pattern.CASE_INSENSITIVE); + private ClashSubscriptionParser() { } + + public static String transform(List lines, boolean filter, double threshold) { + Object loaded = new Yaml().load(String.join("\n", lines)); + if (!(loaded instanceof Map source)) throw new IllegalArgumentException("Clash 上游必须为 YAML 对象"); + if (!(source.get("proxies") instanceof List proxies)) throw new IllegalArgumentException("Clash 上游缺少 proxies 列表"); + Map output = new LinkedHashMap<>(source); + Set removed = new HashSet<>(); + List kept = new ArrayList<>(); + for (Object node : proxies) { + if (!(node instanceof Map fields) || !(fields.get("name") instanceof String name)) + throw new IllegalArgumentException("Clash 节点缺少名称"); + var multiplier = MULTIPLIER.matcher(name); + if (filter && multiplier.find() && Double.parseDouble(multiplier.group(1)) > threshold) + removed.add(name); + else kept.add(node); + } + output.put("proxies", kept); + if (source.get("proxy-groups") instanceof List groups) { + List result = new ArrayList<>(); + for (Object group : groups) { + if (!(group instanceof Map fields)) throw new IllegalArgumentException("Clash 分组必须为对象"); + Map copy = new LinkedHashMap<>(fields); + if (fields.get("proxies") instanceof List references) + copy.put("proxies", references.stream().filter(ref -> !removed.contains(ref)).toList()); + result.add(copy); + } + output.put("proxy-groups", result); + } + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setPrettyFlow(true); + return new Yaml(options).dump(output); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Util/GalleryPageParser.java b/src/main/java/com/lion/lionwebsite/Util/GalleryPageParser.java new file mode 100644 index 0000000..e3ae48b --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/GalleryPageParser.java @@ -0,0 +1,72 @@ +package com.lion.lionwebsite.Util; + +import com.lion.lionwebsite.Domain.Gallery; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import java.io.IOException; +import java.util.*; +import java.util.regex.Pattern; + +/** Pure parsing: upstream requests and download submission live outside this class. */ +public final class GalleryPageParser { + private GalleryPageParser() { } + + public static Gallery metadata(Document document, String link, int gid) throws IOException { + String title = document.select("#gn").text(); + if (title.isBlank()) title = document.select("#gj").text(); + if (title.isBlank()) throw new IOException("上游页面缺少画廊名称，可能是错误页"); + Map fields = new HashMap<>(); + for (Element row : document.select("#gdd tr")) { + var cells = row.select("td"); + if (cells.size() < 2) continue; + String name = cells.get(0).text().replace(":", "").strip().toLowerCase(Locale.ROOT); + fields.put(name, cells.get(1).text().strip()); + } + var pages = Pattern.compile("\\d+").matcher(fields.getOrDefault("length", "")); + if (!pages.find()) throw new IOException("上游页面缺少画廊页数"); + Gallery result = new Gallery(); + result.setGid(gid); result.setLink(link); result.setName(title + " [" + gid + "]"); + result.setCreateTime(System.currentTimeMillis() / 1000); result.setProceeding(0); + result.setLanguage(fields.getOrDefault("language", "").replace("\u00a0", " ")); + try { + result.setPages(Integer.parseInt(pages.group())); + result.setFileSize(CustomUtil.stringToFileSize(fields.getOrDefault("file size", "").replace(" ", ""))); + } catch (RuntimeException failure) { throw new IOException("上游画廊页数或文件大小格式不支持", failure); } + if (result.getPages() <= 0) throw new IOException("上游画廊页数必须为正数"); + result.setDisplayFileSize(CustomUtil.fileSizeToString(result.getFileSize())); + Element thumbnail = document.selectFirst("#gd1 [style*=url]"); + var image = Pattern.compile("url\\(\\s*['\"]?(.*?)['\"]?\\s*\\)").matcher(thumbnail == null ? "" : thumbnail.attr("style")); + String imageUrl = image.find() ? image.group(1) : document.select("#gd1 img").attr("src"); + result.setThumb_link(imageUrl.replace("https://s.exhentai.org", "")); + return result; + } + + public static String archiveLink(Document document) throws IOException { + for (Element anchor : document.select("#gd5 a")) { + var matcher = Pattern.compile("https?://[^'\"\\s]+").matcher(anchor.attr("onclick")); + if (matcher.find()) return matcher.group(); + String href = anchor.absUrl("href"); + if (!href.isBlank() && (anchor.text().toLowerCase(Locale.ROOT).contains("download") || href.contains("archive"))) + return href; + } + throw new IOException("上游页面缺少归档入口"); + } + + public static Map resolutions(Document document) throws IOException { + var expression = Pattern.compile("^(\\d+x|Original)\\s+([\\d.]+)\\s*([A-Za-z]+)$", Pattern.CASE_INSENSITIVE); + TreeMap sizes = new TreeMap<>(); + String original = null; + for (Element paragraph : document.select("#db td p")) { + var matcher = expression.matcher(paragraph.text().strip()); + if (!matcher.matches()) continue; + String size = matcher.group(2) + matcher.group(3); + if (matcher.group(1).equalsIgnoreCase("Original")) original = size; + else sizes.put(Integer.parseInt(matcher.group(1).replace("x", "")), size); + } + Map result = new LinkedHashMap<>(); + sizes.forEach((width, size) -> result.put(width + "x", size)); + if (original != null) result.put("Original", original); + if (result.isEmpty()) throw new IOException("上游页面没有可用分辨率"); + return result; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Util/GallerySearchParser.java b/src/main/java/com/lion/lionwebsite/Util/GallerySearchParser.java new file mode 100644 index 0000000..656cb25 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/GallerySearchParser.java @@ -0,0 +1,45 @@ +package com.lion.lionwebsite.Util; + +import com.lion.lionwebsite.Domain.GalleryForQuery; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** Search rows are located by gallery links, independent of table nesting and headings. */ +public final class GallerySearchParser { + private GallerySearchParser() { } + public static List parse(Document document) throws IOException { + List result = new ArrayList<>(); + for (Element row : document.select("tr")) { + Element link = row.selectFirst("a[href*=/g/]"); + if (link == null) continue; + String href = link.attr("href"); + Integer gid = GalleryUtil.parseGid(href); + if (gid == null) continue; + GalleryForQuery gallery = new GalleryForQuery(); + gallery.setGid(gid.toString()); gallery.setLink(href); + gallery.setName(link.select(".glink").isEmpty() ? link.text() : link.select(".glink").text()); + Element type = row.selectFirst(".cn, .cs"); + gallery.setType(type == null ? row.select("td").first().text() : type.text()); + Element image = row.selectFirst("img"); + String thumbnail = image == null ? "" : image.attr("src"); + if (thumbnail.isBlank() || thumbnail.startsWith("data:")) thumbnail = image == null ? "" : image.attr("data-src"); + gallery.setThumbnailUrl(thumbnail.replace("https://s.exhentai.org", "")); + var pages = Pattern.compile("(\\d+)\\s+pages?", Pattern.CASE_INSENSITIVE).matcher(row.text()); + if (!pages.find()) throw new IOException("上游搜索结果缺少页数 gid=" + gid); + gallery.setPage(Integer.parseInt(pages.group(1))); + Element date = row.selectFirst("[id^=posted_]"); + if (date != null) gallery.setUploadTime(date.text()); + else { + var dates = Pattern.compile("\\d{4}-\\d{2}-\\d{2}(?:\\s+\\d{2}:\\d{2})?").matcher(row.text()); + gallery.setUploadTime(dates.find() ? dates.group() : ""); + } + if (gallery.getName().isBlank()) throw new IOException("上游搜索结果缺少名称 gid=" + gid); + result.add(gallery); + } + return result; + } +} diff --git a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java index 34564e2..2ddd740 100644 --- a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java +++ b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java @@ -82,6 +82,12 @@ public class GalleryUtil { * @throws ResolutionNotMatchException 没有目标分辨率 */ public static Gallery parse(String url, boolean isDownload, String targetResolution) throws IOException, ResolutionNotMatchException { + PreparedGallery prepared = prepare(url); + if (prepared == null) return null; + return isDownload ? submit(prepared, targetResolution) : prepared.gallery(); + } + + public static PreparedGallery prepare(String url) throws IOException { String origin = verifyLink(url); if(origin == null){ @@ -94,66 +100,26 @@ public class GalleryUtil { log.warn("链接无法解析出 gid，按无效链接处理"); return null; } - Gallery gallery = new Gallery(); - gallery.setLink(url); - gallery.setCreateTime(System.currentTimeMillis()/1000); - gallery.setGid(gid); - gallery.setProceeding(0); - - //访问图片页面 - String galleryPage = requests(url, GET, null, null); - Document galleryDoc = Jsoup.parse(galleryPage); - - //收集图片基本信息 - gallery.setName(galleryDoc.select("#gn").text() + " [" + gallery.getGid() + ']'); - gallery.setLanguage(galleryDoc.select("#gdd > table > tbody > tr:nth-child(4) > td.gdt2").text().replace("%nbps", "")); - gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0])); - gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", ""))); - gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize())); - String thumb_link = galleryDoc.select("#gd1 > div").getFirst().attr("style"); - gallery.setThumb_link(thumb_link.substring(thumb_link.indexOf("url(") + 4, thumb_link.lastIndexOf(")")).replace("https://s.exhentai.org", "")); - - //查找下载页面链接并初始化参数 - String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1]; + Document galleryDoc = Jsoup.parse(requests(url, GET, null, null), url); + Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid); + String archiveLink = GalleryPageParser.archiveLink(galleryDoc); HashMap headers = new HashMap<>(); headers.put("origin", origin); + headers.put("referer", archiveLink); + Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink); + String download_link = downloadDoc.select("#hathdl_form").attr("abs:action"); + if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单"); + gallery.setAvailableResolution(GalleryPageParser.resolutions(downloadDoc)); + gallery.setStatus("等待确认下载"); + return new PreparedGallery(gallery, download_link, origin); + } + + public static Gallery submit(PreparedGallery prepared, String targetResolution) throws IOException, ResolutionNotMatchException { + Gallery gallery = prepared.gallery(); + String download_link = prepared.formUrl(); + HashMap headers = new HashMap<>(); + headers.put("origin", prepared.origin()); headers.put("referer", download_link); - - //访问下载页面，获取分辨率 - String downloadPage = requests(download_link, GET, headers, null); - Document downloadDoc = Jsoup.parse(downloadPage); - Elements resolutions = downloadDoc.select("#db > div > table > tbody > tr > td"); - download_link = downloadDoc.select("#hathdl_form").attr("action"); - - Map availableResolution = new LinkedHashMap<>(); - Map tempResolutions = new LinkedHashMap<>(); - ArrayList tempResolutionKeys = new ArrayList<>(); - - for(Element element: resolutions){ - String[] parameter = element.select("td > p").text().split(" "); - if(parameter[1].contains("N/A")) - continue; - tempResolutions.put(parameter[0], parameter[1] + parameter[2]); - if(parameter[0].contains("x")){ - tempResolutionKeys.add(Integer.valueOf(parameter[0].replace("x", ""))); - } - } - - tempResolutionKeys.sort(Comparator.comparingInt(o -> o)); - - for(Integer resolution: tempResolutionKeys){ - availableResolution.put(resolution.toString()+"x", tempResolutions.get(resolution+"x")); - } - - availableResolution.put("Original", tempResolutions.get("Original")); - gallery.setAvailableResolution(availableResolution); - - //如果不是下载，则直接返回 - if(!isDownload){ - gallery.setStatus("等待确认下载"); - return gallery; - } - //如果目标分辨率不存在，抛出错误 if(!gallery.getAvailableResolution().containsKey(targetResolution)){ log.warn("目标分辨率不存在，可用分辨率: {}", gallery.getAvailableResolution()); @@ -161,15 +127,15 @@ public class GalleryUtil { } gallery.setResolution(targetResolution); - gallery.setFileSize(CustomUtil.stringToFileSize(availableResolution.get(targetResolution).replace(" ", ""))); + gallery.setFileSize(CustomUtil.stringToFileSize(gallery.getAvailableResolution().get(targetResolution).replace(" ", ""))); gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize())); //提交下载请求 HashMap body = new HashMap<>(); body.put("payload", FORM_DATA); body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org")); - downloadPage = requests(download_link, POST, headers, body); - downloadDoc = Jsoup.parse(downloadPage); + String downloadPage = requests(download_link, POST, headers, body); + Document downloadDoc = Jsoup.parse(downloadPage); //判断下载请求是否提交成功 if(downloadDoc.select("#db > p:nth-child(2) > strong").text().startsWith("#")) @@ -192,14 +158,12 @@ public class GalleryUtil { if(content.trim().isEmpty()) return null; Document document = Jsoup.parse(content); - if(document.select("body > script").size() < 2) - return null; - Element script = document.select("body > script").get(1); - String[] scripts = script.html().split("\n"); + String script = document.select("script").stream().map(Element::data).collect(java.util.stream.Collectors.joining("\n")); + if (!script.contains("imagelist") || !script.contains("mpvkey")) return null; ArrayList imageKeyCaches = new ArrayList<>(); AtomicInteger page = new AtomicInteger(1); - gid2MpvKey.put(gid, scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", "")); - JsonNode nodes = parseImagelist(scripts[2]); + gid2MpvKey.put(gid, JavaScriptAssignments.string(script, "mpvkey")); + JsonNode nodes = objectMapper.readTree(JavaScriptAssignments.array(script, "imagelist")); nodes.forEach((n) -> { ImageKeyCache imageKeyCache = new ImageKeyCache(); imageKeyCache.setGid(gid); @@ -247,12 +211,14 @@ public class GalleryUtil { gid2MpvKey.remove(parseGid(url) + ""); return; } - Document document = Jsoup.parse(content); - Element script = document.select("body > script").get(1); - String[] scripts = script.html().split("\n"); - String mpvKey = scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", ""); - gid2MpvKey.put(parseGid(url) + "", mpvKey); - log.info("刷新key:{}", mpvKey); + try { + String scripts = Jsoup.parse(content).select("script").stream().map(Element::data) + .collect(java.util.stream.Collectors.joining("\n")); + gid2MpvKey.put(parseGid(url) + "", JavaScriptAssignments.string(scripts, "mpvkey")); + } catch (IOException e) { + gid2MpvKey.remove(parseGid(url) + ""); + log.warn("上游mpv脚本无法解析，缓存已失效"); + } } public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) { diff --git a/src/main/java/com/lion/lionwebsite/Util/JavaScriptAssignments.java b/src/main/java/com/lion/lionwebsite/Util/JavaScriptAssignments.java new file mode 100644 index 0000000..f9514d9 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/JavaScriptAssignments.java @@ -0,0 +1,34 @@ +package com.lion.lionwebsite.Util; + +import java.io.IOException; +import java.util.regex.Pattern; + +/** Extracts data assignments without depending on script position or line layout. */ +public final class JavaScriptAssignments { + private JavaScriptAssignments() { } + + public static String string(String source, String name) throws IOException { + var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*(['\"])(.*?)\\1", Pattern.DOTALL).matcher(source); + if (!matcher.find()) throw new IOException("上游脚本缺少 " + name); + return matcher.group(2); + } + + public static String array(String source, String name) throws IOException { + var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*\\[").matcher(source); + if (!matcher.find()) throw new IOException("上游脚本缺少 " + name); + int start = matcher.end() - 1, depth = 0; char quote = 0; boolean escaped = false; + for (int i = start; i < source.length(); i++) { + char c = source.charAt(i); + if (quote != 0) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == quote) quote = 0; + continue; + } + if (c == '"' || c == '\'') quote = c; + else if (c == '[') depth++; + else if (c == ']' && --depth == 0) return source.substring(start, i + 1); + } + throw new IOException("上游脚本的 " + name + " 不完整"); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Util/PreparedGallery.java b/src/main/java/com/lion/lionwebsite/Util/PreparedGallery.java new file mode 100644 index 0000000..22a2095 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/PreparedGallery.java @@ -0,0 +1,6 @@ +package com.lion.lionwebsite.Util; + +import com.lion.lionwebsite.Domain.Gallery; + +/** A read-only upstream preparation; creating this value never submits a download. */ +public record PreparedGallery(Gallery gallery, String formUrl, String origin) { } diff --git a/src/main/java/com/lion/lionwebsite/Util/SubscriptionCacheStore.java b/src/main/java/com/lion/lionwebsite/Util/SubscriptionCacheStore.java new file mode 100644 index 0000000..a74fa95 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/SubscriptionCacheStore.java @@ -0,0 +1,328 @@ +package com.lion.lionwebsite.Util; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +/** + * 订阅缓存的分代存储：v2ray 与 clash 两份内容必须同时生效。 + * + *

旧实现把两份文件各自原子替换，第二份失败时用户会拿到「新 v2ray + 旧 clash」的 + * 混合版本。这里把两份文件写进同一个不可变分代目录，校验通过后用一次原子替换发布 + * 指针（manifest），因此抓取、解析、写入或发布任何一步失败，都只会保留上一个完整分代。 + * + *

目录结构： + * 

+ * <cacheRoot>/<accountId>/current                      指针（原子替换，最后发布）
+ * <cacheRoot>/<accountId>/generations/<generation>/v2ray.txt
+ * <cacheRoot>/<accountId>/generations/<generation>/clash.yaml
+ * <cacheRoot>/<accountId>/v2ray.txt + clash.yaml          旧结构，两份都在才按 legacy 读
+ *
+ * + *

读取始终从指针指向的同一个分代里取两份文件，所以拿到的一定是同版本内容。 + */ +@Slf4j +public class SubscriptionCacheStore { + private static final String MANIFEST_FILE = "current"; + private static final String GENERATION_DIR = "generations"; + private static final String V2_FILE = "v2ray.txt"; + private static final String CLASH_FILE = "clash.yaml"; + /** 保留目录名，结构上永远不存在，用于表达「当前没有可分发版本」。 */ + private static final String UNAVAILABLE_DIR = ".unavailable"; + private static final String MANIFEST_GENERATION = "generation"; + private static final String MANIFEST_V2_SHA = "v2-sha256"; + private static final String MANIFEST_CLASH_SHA = "clash-sha256"; + /** 旧结构读出来的版本的 generation 标记；真实分代名是定宽时间戳，不会与之碰撞。 */ + private static final String LEGACY_GENERATION = "legacy"; + /** 保留分代数：当前与上一个，足够回退且不会无限增长。 */ + private static final int RETAINED_GENERATIONS = 2; + private static final AtomicLong SEQUENCE = new AtomicLong(); + + private final Path root; + + public SubscriptionCacheStore(String cacheRoot) { + this(Paths.get(cacheRoot)); + } + + public SubscriptionCacheStore(Path root) { + this.root = root; + } + + /** 同一个分代目录里读出的一对内容；两份文件必然同版本。 */ + public record Pair(String generation, byte[] v2, byte[] clash) { } + + public Path accountDir(int accountId) { + return root.resolve(String.valueOf(accountId)); + } + + /** 旧结构的账号根文件路径。仅供兼容测试定位 legacy 文件，不作为回退目标。 */ + public Path legacyPath(int accountId, String client) { + return accountDir(accountId).resolve(fileName(client)); + } + + /** + * 没有完整可分发版本时返回的哨兵路径。 + * + *

它保证不存在，因此调用方用 {@code Files.isRegularFile} 判定就会走「订阅暂不可用」 + * 分支，而不会被半份 legacy 文件骗过。 + */ + public Path unavailablePath(int accountId, String client) { + return accountDir(accountId).resolve(UNAVAILABLE_DIR).resolve(fileName(client)); + } + + private static String fileName(String client) { + return "v2".equals(client) ? V2_FILE : CLASH_FILE; + } + + /** + * 写入一份新分代并在最后发布指针。 + * + *

失败时抛出异常，且不改变指针与上一个分代，调用方据此保留上次的完整版本。 + */ + public Pair publish(int accountId, byte[] v2, byte[] clash) throws IOException { + Path account = accountDir(accountId); + Files.createDirectories(account); + String generation = nextGeneration(); + Path dir = account.resolve(GENERATION_DIR).resolve(generation); + boolean published = false; + try { + Files.createDirectories(dir); + writeFile(dir.resolve(V2_FILE), v2); + writeFile(dir.resolve(CLASH_FILE), clash); + // 发布前校验：两份文件都必须能按写入内容读回，避免把半截内容挂上指针。 + verify(dir.resolve(V2_FILE), v2); + verify(dir.resolve(CLASH_FILE), clash); + publishManifest(account, generation, sha256(v2), sha256(clash)); + published = true; + } finally { + if (!published) + deleteRecursively(dir); + } + prune(account, generation); + removeLegacy(account); + return new Pair(generation, v2, clash); + } + + /** 读取当前发布内容；只有构成一整个有效版本时才返回。 */ + public Optional read(int accountId) { + return Optional.ofNullable(load(accountDir(accountId))); + } + + /** + * 解析当前应分发的单个文件。 + * + *

与 {@link #read}、{@link #hasComplete} 使用同一份配对契约：只有两份文件 + * 齐全且哈希校验通过时才返回路径，因此分发路径不会发出半份或已损坏的缓存。 + */ + public Optional resolve(int accountId, String client) { + Path account = accountDir(accountId); + Pair pair = load(account); + if (pair == null) + return Optional.empty(); + Path path = LEGACY_GENERATION.equals(pair.generation()) + ? account.resolve(fileName(client)) + : account.resolve(GENERATION_DIR).resolve(pair.generation()).resolve(fileName(client)); + return Files.isRegularFile(path) ? Optional.of(path) : Optional.empty(); + } + + /** 是否已有可分发的一对内容（分代两份都在且哈希匹配，或旧结构两份都在）。 */ + public boolean hasComplete(int accountId) { + return load(accountDir(accountId)) != null; + } + + /** 删除该账号的全部缓存（指针、分代、旧结构文件），使后续请求判定为无缓存。 */ + public void invalidate(int accountId) { + Path account = accountDir(accountId); + deleteIfExists(account.resolve(MANIFEST_FILE)); + deleteRecursively(account.resolve(GENERATION_DIR)); + deleteIfExists(account.resolve(V2_FILE)); + deleteIfExists(account.resolve(CLASH_FILE)); + } + + /** 写入接缝：测试覆写以模拟第二份文件写入失败。 */ + protected void writeFile(Path target, byte[] data) throws IOException { + atomicWrite(target, data); + } + + /** 指针发布接缝：测试覆写以模拟 manifest 发布失败。 */ + protected void publishManifest(Path accountDir, String generation, String v2Sha256, String clashSha256) + throws IOException { + atomicWrite(accountDir.resolve(MANIFEST_FILE), + manifestText(generation, v2Sha256, clashSha256).getBytes(StandardCharsets.UTF_8)); + } + + /** + * 读取「一整个版本」：一次读入指针快照，再按该快照定位分代并校验双文件哈希。 + * + *

指针只读一次，因此并发 {@link #publish} 切换指针时不会出现「用新分代名配旧哈希」 + * 或哈希字段缺失导致的空指针；要么拿到完整的新版本，要么拿到完整的旧版本。 + */ + private Pair load(Path account) { + Properties manifest = readManifest(account); + if (manifest == null) { + // 指针文件不存在才是「旧结构」。指针存在但读不出来说明它已损坏，同样按无缓存处理。 + return Files.isRegularFile(account.resolve(MANIFEST_FILE)) ? null : loadLegacy(account); + } + Pair generation = loadGeneration(account, manifest); + if (generation != null) + return generation; + // 指针存在却读不出完整版本（半写、被篡改、内容缺失）时不得回退到 legacy： + // 那会发出一个比指针更旧的版本，掩盖损坏。此时按无可用缓存处理。 + return null; + } + + private Pair loadGeneration(Path account, Properties manifest) { + String generation = manifest.getProperty(MANIFEST_GENERATION); + if (generation == null || generation.isBlank() || generation.contains("/") || generation.contains("..")) { + log.warn("订阅缓存指针内容非法"); + return null; + } + String v2Sha256 = manifest.getProperty(MANIFEST_V2_SHA); + String clashSha256 = manifest.getProperty(MANIFEST_CLASH_SHA); + if (v2Sha256 == null || clashSha256 == null) { + log.warn("订阅缓存指针缺少哈希字段 generation={}", generation); + return null; + } + Path dir = account.resolve(GENERATION_DIR).resolve(generation); + byte[] v2 = readFile(dir.resolve(V2_FILE)); + byte[] clash = readFile(dir.resolve(CLASH_FILE)); + if (v2 == null || clash == null) { + log.warn("订阅缓存分代内容缺失 generation={}", generation); + return null; + } + if (!matches(v2, v2Sha256) || !matches(clash, clashSha256)) { + log.warn("订阅缓存分代内容校验失败 generation={}", generation); + return null; + } + return new Pair(generation, v2, clash); + } + + /** 旧结构：只有账号根下两份文件都在才构成一个有效版本。 */ + private Pair loadLegacy(Path account) { + byte[] v2 = readFile(account.resolve(V2_FILE)); + byte[] clash = readFile(account.resolve(CLASH_FILE)); + if (v2 == null || clash == null) + return null; + return new Pair(LEGACY_GENERATION, v2, clash); + } + + /** 指针读取接缝：包内可见，便于测试统计「每次查询只读一次指针」。 */ + Properties readManifest(Path account) { + Path manifest = account.resolve(MANIFEST_FILE); + if (!Files.isRegularFile(manifest)) + return null; + try { + Properties properties = new Properties(); + properties.load(new StringReader(Files.readString(manifest, StandardCharsets.UTF_8))); + return properties; + } catch (IOException e) { + log.warn("读取订阅缓存指针失败: {}", e.getMessage()); + return null; + } + } + + private static byte[] readFile(Path path) { + try { + return Files.isRegularFile(path) ? Files.readAllBytes(path) : null; + } catch (IOException e) { + log.warn("读取订阅缓存文件失败 path={} error={}", path, e.getMessage()); + return null; + } + } + + private static void verify(Path path, byte[] expected) throws IOException { + byte[] written = Files.readAllBytes(path); + if (written.length != expected.length || !MessageDigest.isEqual(written, expected)) + throw new IOException("订阅缓存写入校验失败 " + path.getFileName()); + } + + private static boolean matches(byte[] content, String expectedSha256) { + return expectedSha256 != null && expectedSha256.equals(sha256(content)); + } + + private static String manifestText(String generation, String v2Sha256, String clashSha256) { + return MANIFEST_GENERATION + "=" + generation + "\n" + + MANIFEST_V2_SHA + "=" + v2Sha256 + "\n" + + MANIFEST_CLASH_SHA + "=" + clashSha256 + "\n"; + } + + /** 分代名定宽可排序：时间戳在前保证单调，序号消除同毫秒碰撞。 */ + private static String nextGeneration() { + return String.format("%013d-%06d", System.currentTimeMillis(), SEQUENCE.incrementAndGet() % 1_000_000); + } + + private void prune(Path account, String current) { + Path dir = account.resolve(GENERATION_DIR); + List generations; + try (Stream entries = Files.list(dir)) { + generations = entries.filter(Files::isDirectory) + .sorted(Comparator.comparing(path -> path.getFileName().toString(), Comparator.reverseOrder())) + .toList(); + } catch (IOException e) { + log.warn("清理订阅缓存分代失败 path={} error={}", dir, e.getMessage()); + return; + } + for (int index = RETAINED_GENERATIONS; index < generations.size(); index++) { + Path stale = generations.get(index); + if (!stale.getFileName().toString().equals(current)) + deleteRecursively(stale); + } + } + + /** 分代发布成功后旧结构文件被取代，避免两份来源同时存在。 */ + private void removeLegacy(Path account) { + deleteIfExists(account.resolve(V2_FILE)); + deleteIfExists(account.resolve(CLASH_FILE)); + } + + private static void atomicWrite(Path target, byte[] data) throws IOException { + Path temp = target.resolveSibling(target.getFileName() + ".tmp"); + Files.write(temp, data); + try { + Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static void deleteIfExists(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + log.warn("删除订阅缓存文件失败 path={} error={}", path, e.getMessage()); + } + } + + private static void deleteRecursively(Path path) { + if (!Files.exists(path)) + return; + try (Stream entries = Files.walk(path)) { + entries.sorted(Comparator.reverseOrder()).forEach(SubscriptionCacheStore::deleteIfExists); + } catch (IOException e) { + log.warn("删除订阅缓存目录失败 path={} error={}", path, e.getMessage()); + } + } + + private static String sha256(byte[] input) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(input)); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/test/java/com/lion/lionwebsite/Controller/PublicControllerTest.java b/src/test/java/com/lion/lionwebsite/Controller/PublicControllerTest.java index 3464ebe..d980839 100644 --- a/src/test/java/com/lion/lionwebsite/Controller/PublicControllerTest.java +++ b/src/test/java/com/lion/lionwebsite/Controller/PublicControllerTest.java @@ -82,14 +82,7 @@ class PublicControllerTest { verify(publicService, times(1)).logIpAddress(anyString()); } - /** - * 返回体形如 {"result":"success","data":"{\"userId\": 7, ...}"}—— - * data 是「JSON 文本的字符串」（历史契约，前端按字符串解析后再反序列化）。 - * - *

内层 JSON 改由 ObjectMapper 生成（原先手工 String.format 拼接，用户名带引号 - * 会产出非法 JSON），因此不再有冒号后的空格；这里断言的是字段与取值， - * 断言的字符串形式随之调整。 - */ + /** 身份直接内嵌为 JSON 对象，保留 result/data 字段。 */ @Test void validateReturnsIdentityAndNodeAvailability() throws Exception { when(publicService.getUserId("code")).thenReturn(new User(7, "code", "alice", null, true)); @@ -97,11 +90,11 @@ class PublicControllerTest { mockMvc.perform(post("/validate").param("AuthCode", "code")) .andExpect(status().isOk()) - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"userId\\\":7"))) - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"username\\\":\\\"alice\\\""))) - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"isAvailable\\\":true"))) + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"userId\":7"))) + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"username\":\"alice\""))) + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"isAvailable\":true"))) // 普通用户 isAdmin=false，前端据此隐藏下载人信息。 - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"isAdmin\\\":false"))); + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"isAdmin\":false"))); } /** 管理员（id=3）必须带上 isAdmin=true，前端据此显示下载人信息与筛选。 */ @@ -112,7 +105,7 @@ class PublicControllerTest { mockMvc.perform(post("/validate").param("AuthCode", "admin")) .andExpect(status().isOk()) - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"isAdmin\\\":true"))); + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"isAdmin\":true"))); } /** @@ -129,11 +122,10 @@ class PublicControllerTest { .andExpect(status().isOk()) .andReturn(); - // 取出外层 data（是内层 JSON 的字符串形式），必须能被 JSON 解析回原名。 + // 直接读取结构化 data，特殊字符保持原样。 var outer = com.lion.lionwebsite.Util.CustomUtil.objectMapper .readTree(result.getResponse().getContentAsString()); - var inner = com.lion.lionwebsite.Util.CustomUtil.objectMapper - .readTree(outer.get("data").asText()); + var inner = outer.get("data"); assertEquals("a\"b\\c", inner.get("username").asText(), "特殊字符必须原样保留且 JSON 合法"); } @@ -145,7 +137,7 @@ class PublicControllerTest { mockMvc.perform(post("/validate").param("AuthCode", "code")) .andExpect(status().isOk()) - .andExpect(content().string(org.hamcrest.Matchers.containsString("\\\"isAvailable\\\":false"))); + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"isAvailable\":false"))); } @Test diff --git a/src/test/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapperSqlTest.java b/src/test/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapperSqlTest.java new file mode 100644 index 0000000..a7367c4 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapperSqlTest.java @@ -0,0 +1,219 @@ +package com.lion.lionwebsite.Dao.cache; + +import com.lion.lionwebsite.Domain.GidToKey; +import com.lion.lionwebsite.Domain.ImageKeyCache; +import org.apache.ibatis.datasource.unpooled.UnpooledDataSource; +import org.apache.ibatis.mapping.Environment; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; +import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * ImageCacheMapper 的实际 SQL 验证。 + * + *

服务层测试用的是内存替身，无法覆盖注解 SQL 本身（尤其是批量 INSERT 的 + * {@code "; - - try (var parser = mockStatic(com.lion.lionwebsite.Util.GalleryUtil.class)) { - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil.parseImageKeys(anyString())) - .thenCallRealMethod(); - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil - .requests(anyString(), anyString(), any(), any())) - .thenReturn(realMpvPage); - - String json = svc.cacheImagesKey("https://exhentai.org/g/1596929/f08534d87d/"); - - assertTrue(json.contains("\"result\":\"success\""), "应缓存成功，实际: " + json); - } - - verify(imageCacheMapper).insertGidToKey(any()); - verify(imageCacheMapper).insertImageKeyCache(any()); + void cacheApiReturnsStructuredIndexWithoutDoubleEncoding() throws Exception { + service.galleryIndexService = mock(GalleryIndexService.class); + var index = new com.lion.lionwebsite.Domain.GidToKey(); + index.setGid("123"); index.setKey("key"); index.setPages(20); + when(service.galleryIndexService.ensureIndex("link")).thenReturn(index); + var response = com.lion.lionwebsite.Util.CustomUtil.objectMapper.readTree(service.cacheImagesKey("link")); + assertEquals("success", response.get("result").asText()); + assertTrue(response.get("data").isObject()); + assertEquals(20, response.get("data").get("pages").asInt()); } - /** - * 第三方页面格式异常时必须回业务失败，不能再穿透成 500。 - * 这里让 requests 返回畸形 JSON，验证 catch 兜住解析异常。 - */ @Test - void cacheImagesKeyConvertsParseFailureToBusinessFailure() throws Exception { - ImageCacheMapper imageCacheMapper = mock(ImageCacheMapper.class); - when(imageCacheMapper.selectKeyByGid(anyString())).thenReturn(null); - GalleryManageService svc = new GalleryManageService(galleries, collectMapper, - configurationMapper, users, - imageCacheMapper, remote, push); - - String brokenPage = ""; - - try (var parser = mockStatic(com.lion.lionwebsite.Util.GalleryUtil.class)) { - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil.parseImageKeys(anyString())) - .thenCallRealMethod(); - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil - .requests(anyString(), anyString(), any(), any())) - .thenReturn(brokenPage); - - String json = assertDoesNotThrow(() -> - svc.cacheImagesKey("https://exhentai.org/g/1596929/f08534d87d/")); - - assertFalse(json.contains("\"result\":\"success\""), "不应报成功: " + json); - assertTrue(json.contains("网络波动或其他异常"), "实际: " + json); - } - - verify(imageCacheMapper, never()).insertGidToKey(any()); - verify(imageCacheMapper, never()).insertImageKeyCache(any()); - } - - /** 畸形链接（段数不足）必须回业务失败，不得抛 ArrayIndexOutOfBoundsException。 */ - @Test - void cacheImagesKeyRejectsMalformedLink() { - ImageCacheMapper imageCacheMapper = mock(ImageCacheMapper.class); - GalleryManageService svc = new GalleryManageService(galleries, collectMapper, - configurationMapper, users, - imageCacheMapper, remote, push); - - for (String bad : new String[]{"abc", "https://exhentai.org/g/1/", null}) { - String json = assertDoesNotThrow(() -> svc.cacheImagesKey(bad), - "畸形链接不应抛异常，实际输入: " + bad); - assertTrue(json.contains("链接错误"), "实际输出: " + json); - } - verify(imageCacheMapper, never()).insertGidToKey(any()); - } - - /** - * 回归：索引写入不是原子的。若 gidToKey 写入成功、逐页写 ImageKeyCache 时失败， - * 会留下「gidToKey 命中但页 key 缺失」的半截缓存，后续请求直接返回已缓存而永远取不到图。 - * 现在任何异常都要把已写入的部分清掉再回业务失败。 - */ - @Test - void cacheImagesKeyRollsBackWhenPageInsertFails() throws Exception { - ImageCacheMapper imageCacheMapper = mock(ImageCacheMapper.class); - when(imageCacheMapper.selectKeyByGid(anyString())).thenReturn(null); - doThrow(new RuntimeException("db down")).when(imageCacheMapper).insertImageKeyCache(any()); - GalleryManageService svc = new GalleryManageService(galleries, collectMapper, - configurationMapper, users, - imageCacheMapper, remote, push); - - String realMpvPage = ""; - - try (var parser = mockStatic(com.lion.lionwebsite.Util.GalleryUtil.class)) { - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil.parseImageKeys(anyString())) - .thenCallRealMethod(); - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil - .requests(anyString(), anyString(), any(), any())) - .thenReturn(realMpvPage); - - String json = assertDoesNotThrow(() -> - svc.cacheImagesKey("https://exhentai.org/g/1596929/f08534d87d/")); - assertFalse(json.contains("\"result\":\"success\""), "写入失败不应报成功: " + json); - } - - verify(imageCacheMapper).insertGidToKey(any()); - verify(imageCacheMapper).deleteImageKeyCacheByGid(anyString()); - verify(imageCacheMapper).deleteGidToKey(anyString()); - } - - /** 已缓存的索引若页 key 数对不上（半截缓存），必须重建而不是直接返回。 */ - @Test - void cacheImagesKeyRebuildsIncompleteCache() throws Exception { - ImageCacheMapper imageCacheMapper = mock(ImageCacheMapper.class); - var stale = new com.lion.lionwebsite.Domain.GidToKey(); - stale.setGid("1596929"); - stale.setKey("f08534d87d"); - stale.setPages(3); - when(imageCacheMapper.selectKeyByGid("1596929")).thenReturn(stale); - when(imageCacheMapper.countImageKeyCacheByGid("1596929")).thenReturn(1); // 只剩 1 页 - GalleryManageService svc = new GalleryManageService(galleries, collectMapper, - configurationMapper, users, - imageCacheMapper, remote, push); - - String realMpvPage = ""; - - try (var parser = mockStatic(com.lion.lionwebsite.Util.GalleryUtil.class)) { - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil.parseImageKeys(anyString())) - .thenCallRealMethod(); - parser.when(() -> com.lion.lionwebsite.Util.GalleryUtil - .requests(anyString(), anyString(), any(), any())) - .thenReturn(realMpvPage); - - String json = svc.cacheImagesKey("https://exhentai.org/g/1596929/f08534d87d/"); - assertTrue(json.contains("\"result\":\"success\""), "应重建成功: " + json); - } - - verify(imageCacheMapper).deleteImageKeyCacheByGid("1596929"); - verify(imageCacheMapper).deleteGidToKey("1596929"); - verify(imageCacheMapper).insertGidToKey(any()); + void cacheApiMapsPermanentAndTransientFailures() throws Exception { + service.galleryIndexService = mock(GalleryIndexService.class); + var invalid = new com.lion.lionwebsite.Exception.GalleryIndexException( + com.lion.lionwebsite.Exception.GalleryIndexException.Reason.INVALID_LINK, "invalid"); + when(service.galleryIndexService.ensureIndex("bad")).thenThrow(invalid); + assertTrue(service.cacheImagesKey("bad").contains("链接错误")); + var failed = new com.lion.lionwebsite.Exception.GalleryIndexException( + com.lion.lionwebsite.Exception.GalleryIndexException.Reason.TRANSIENT, "retry"); + when(service.galleryIndexService.ensureIndex("retry")).thenThrow(failed); + assertTrue(service.cacheImagesKey("retry").contains("网络波动或其他异常")); } // ---------- createTask 输入校验 ---------- @@ -553,6 +412,72 @@ class GalleryManageServiceTest { assertFalse(service.retryGallery(2).contains("\"result\":\"success\"")); } + // ---------- 生命周期拒绝路径 ---------- + + /** beginDelete 被拒（任务已在删除中/提交中）时必须回业务失败，且不得退避到裸删除。 */ + @Test + void deleteRefusedByLifecycleDoesNotFallBackToRawDelete() { + Gallery gallery = new Gallery(); + gallery.setGid(897); + gallery.setName("mine [897]"); + gallery.setDownloader(7); + when(galleries.selectGalleryByGid(897)).thenReturn(gallery); + when(collectMapper.selectCollectorByGid(897)).thenReturn(new java.util.ArrayList<>()); + service.lifecycleService = mock(GalleryTaskLifecycleService.class); + when(service.lifecycleService.beginDelete(897)) + .thenReturn(GalleryTaskLifecycleService.DeleteIntent.refused("任务正在提交中，请等待提交结果明确后再删除")); + + String response = service.deleteGalleryByGid(897, "code"); + + assertFalse(response.contains("\"result\":\"success\""), "被拒时必须回失败: " + response); + assertTrue(response.contains("提交中"), response); + verify(remote, never()).deleteGallery(any()); + verify(service.lifecycleService, never()).completeDelete(anyInt(), anyString()); + } + + /** reset 只重投可安全下发的任务，跳过需要人工对账的 UNKNOWN 等。 */ + @Test + void resetSkipsTasksThatMustNotBeDispatched() { + when(remote.isDead()).thenReturn(false); + Gallery safe = new Gallery(); + safe.setGid(900); + safe.setName("safe [900]"); + Gallery unknown = new Gallery(); + unknown.setGid(901); + unknown.setName("unknown [901]"); + when(galleries.selectUnDoneGalleries()).thenReturn(new Gallery[]{safe, unknown}); + service.lifecycleService = mock(GalleryTaskLifecycleService.class); + when(service.lifecycleService.resolveDispatch(900)) + .thenReturn(GalleryTaskLifecycleService.DispatchDecision.ready("dl-900")); + when(service.lifecycleService.resolveDispatch(901)) + .thenReturn(GalleryTaskLifecycleService.DispatchDecision.refuse()); + + String response = service.resetUndone(); + + assertTrue(response.contains("\"result\":\"success\""), response); + verify(remote).addGalleryToQueue(safe); + verify(remote, never()).addGalleryToQueue(unknown); + } + + /** 提交未知的任务重试必须被拒，提示人工对账，且不触碰节点。 */ + @Test + void retryRefusesWhenLifecycleRejects() { + Gallery gallery = new Gallery(); + gallery.setGid(902); + gallery.setName("unknown [902]"); + gallery.setStatus("提交未知"); + when(galleries.selectGalleryByGid(902)).thenReturn(gallery); + service.lifecycleService = mock(GalleryTaskLifecycleService.class); + when(service.lifecycleService.resolveDispatch(902)) + .thenReturn(GalleryTaskLifecycleService.DispatchDecision.refuse()); + + String response = service.retryGallery(902); + + assertFalse(response.contains("\"result\":\"success\""), response); + assertTrue(response.contains("人工对账"), response); + verify(remote, never()).retryGallery(any()); + } + // ---------- 列表 ---------- /** 用量查询应返回格式化后的已用量与上次重置时间。 */ diff --git a/src/test/java/com/lion/lionwebsite/Service/QueryServiceTest.java b/src/test/java/com/lion/lionwebsite/Service/QueryServiceTest.java index c496024..eadbe14 100644 --- a/src/test/java/com/lion/lionwebsite/Service/QueryServiceTest.java +++ b/src/test/java/com/lion/lionwebsite/Service/QueryServiceTest.java @@ -65,9 +65,9 @@ class QueryServiceTest { assertTrue(json.contains("Title Here"), "应解析出画廊名"); assertTrue(json.contains("1234567"), "应解析出 gid"); assertTrue(json.contains("Manga"), "应解析出类型"); - // data 是「数组序列化后的字符串」，字段因此带转义，这里按转义后的形态断言 - assertTrue(json.contains("\\\"page\\\":42"), "应解析出页数: " + json); - assertTrue(json.contains("\\\"gid\\\":\\\"1234567\\\""), "应解析出 gid: " + json); + // data 直接为结构化数组。 + assertTrue(json.contains("\"page\":42"), "应解析出页数: " + json); + assertTrue(json.contains("\"gid\":\"1234567\""), "应解析出 gid: " + json); // 缩略图地址去掉了站点前缀，前端要拿相对路径 assertTrue(json.contains("/t/thumb.jpg")); assertFalse(json.contains("s.exhentai.org")); diff --git a/src/test/java/com/lion/lionwebsite/Service/RemoteServiceEligibilityTest.java b/src/test/java/com/lion/lionwebsite/Service/RemoteServiceEligibilityTest.java new file mode 100644 index 0000000..4d12419 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Service/RemoteServiceEligibilityTest.java @@ -0,0 +1,159 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.normal.GalleryMapper; +import com.lion.lionwebsite.Domain.Gallery; +import com.lion.lionwebsite.Message.ProtocolHelloMessage; +import com.lion.lionwebsite.Message.TaskCommandMessage; +import io.netty.channel.embedded.EmbeddedChannel; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * 节点下发的安全边界：不是「能解析出 operationId 就发」。 + * + *

UNKNOWN / 预占中 / 待删除的任务一旦被重投，未计量的未知任务会在节点侧变成 + * 「下载完成却 0 额度」；协商完成前与启动恢复不安全时也不允许批量重投。 + */ +class RemoteServiceEligibilityTest { + + private GalleryMapper galleryMapper; + private RemoteService service; + private EmbeddedChannel channel; + private final CopyOnWriteArrayList written = new CopyOnWriteArrayList<>(); + private final java.util.concurrent.CountDownLatch helloWritten = new java.util.concurrent.CountDownLatch(1); + + @BeforeEach + void setUp() { + galleryMapper = mock(GalleryMapper.class); + service = new RemoteService(galleryMapper, mock(PushService.class), + mock(WebSocketService.class), mock(SubscriptionStandbySnapshotService.class)); + channel = new EmbeddedChannel(new io.netty.channel.ChannelOutboundHandlerAdapter() { + @Override + public void write(io.netty.channel.ChannelHandlerContext ctx, Object msg, + io.netty.channel.ChannelPromise promise) { + io.netty.util.concurrent.Promise inner = + service.promiseHashMap.get(((com.lion.lionwebsite.Message.AbstractMessage) msg).messageId); + if (inner != null) + inner.trySuccess(new com.lion.lionwebsite.Message.ResponseMessage()); + written.add(msg); + if (msg instanceof ProtocolHelloMessage) + helloWritten.countDown(); + promise.setSuccess(); + } + }, service.new MyChannelInboundHandlerAdapter()); + service.channel = channel; + io.netty.channel.ChannelFuture channelFuture = mock(io.netty.channel.ChannelFuture.class); + when(channelFuture.channel()).thenReturn(channel); + service.channelFuture = channelFuture; + } + + @AfterEach + void tearDown() { + service.shutdownResources(); + channel.finishAndReleaseAll(); + } + + private static Gallery gallery(int gid) { + Gallery gallery = new Gallery(); + gallery.setGid(gid); + gallery.setName("任务 [" + gid + "]"); + gallery.setStatus("已提交"); + return gallery; + } + + /** 未协商 v2 / 装配未就绪时绝不批量重投未完成任务。 */ + @Test + void resetUndoneSkipsBeforeNegotiationAndWiring() { + service.v2Negotiated = false; + service.resetUndone(); + + service.v2Negotiated = true; + service.lifecycleReady = false; + service.resetUndone(); + + service.lifecycleReady = true; + service.recoverySafe = false; + service.resetUndone(); + + verify(galleryMapper, never()).selectUnDoneGalleries(); + + // 四项条件都满足时才真正重投。 + service.recoverySafe = true; + when(galleryMapper.selectUnDoneGalleries()).thenReturn(new Gallery[0]); + service.resetUndone(); + verify(galleryMapper).selectUnDoneGalleries(); + } + + /** 判定解析器明确拒绝时，绝不能退化为无 operationId 的旧命令。 */ + @Test + void refusedDecisionNeverFallsBackToLegacyCommand() { + service.setDispatchResolver(gid -> GalleryTaskLifecycleService.DispatchDecision.refuse()); + + byte result = service.addGalleryToQueue(gallery(77)); + + assertEquals(RemoteService.REFUSED, result); + assertTrue(written.isEmpty(), "被拒的任务不得写入任何命令"); + } + + /** 判定为可下发时必须使用解析到的稳定 operationId。 */ + @Test + void readyDecisionSendsStableOperationId() { + service.setDispatchResolver(gid -> GalleryTaskLifecycleService.DispatchDecision.ready("dl-stable")); + service.v2Negotiated = true; + + byte result = service.addGalleryToQueue(gallery(78)); + + assertEquals(0, result); + assertEquals(1, written.size(), "应写入一条任务命令"); + Object sent = written.get(0); + assertInstanceOf(TaskCommandMessage.class, sent); + TaskCommandMessage command = (TaskCommandMessage) sent; + assertEquals("dl-stable", command.getOperationId()); + assertEquals(TaskCommandMessage.ACTION_DOWNLOAD, command.getAction()); + } + + /** UNKNOWN 任务的重试必须被拒，并给出人工对账提示。 */ + @Test + void retryRefusesWhenDecisionRejects() { + service.setDispatchResolver(gid -> GalleryTaskLifecycleService.DispatchDecision.refuse()); + + RemoteService.RetryResult result = service.retryGallery(gallery(79)); + + assertFalse(result.success()); + assertTrue(result.message().contains("人工对账"), result.message()); + } + + /** 节点回 Hello（不是 ResponseMessage）即完成协商 future，不再白等 10 秒。 */ + @Test + void nodeHelloCompletesNegotiationFuture() throws Exception { + CompletableFuture negotiation = new CompletableFuture<>(); + service.helloNegotiation = negotiation; + + channel.writeInbound(new ProtocolHelloMessage()); + + assertTrue(negotiation.get(2, TimeUnit.SECONDS), "节点声明的 v2 能力应完成协商"); + assertTrue(service.isReliableProtocolReady()); + } + + @Test + @SuppressWarnings("unchecked") + void realHandshakeWaitsForHelloWithoutResponseMessagePromise() throws Exception { + var method = RemoteService.class.getDeclaredMethod("queueHelloHandshake"); + method.setAccessible(true); + var negotiation = (CompletableFuture) method.invoke(service); + assertTrue(helloWritten.await(2, TimeUnit.SECONDS), "应实际发出 Hello"); + assertTrue(service.promiseHashMap.isEmpty(), "Hello 不应等待 ResponseMessage"); + assertFalse(negotiation.isDone(), "尚未收到 Hello 时不能宣告协商结束"); + channel.writeInbound(new ProtocolHelloMessage()); + assertTrue(negotiation.get(2, TimeUnit.SECONDS)); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Service/SubServiceTest.java b/src/test/java/com/lion/lionwebsite/Service/SubServiceTest.java index 25664fa..ab19a76 100644 --- a/src/test/java/com/lion/lionwebsite/Service/SubServiceTest.java +++ b/src/test/java/com/lion/lionwebsite/Service/SubServiceTest.java @@ -337,6 +337,32 @@ class SubServiceTest { verify(response).sendError(eq(503), anyString()); } + /** + * 半份缓存（旧结构只写了 v2ray.txt）也必须回 503，而不是把残缺内容当订阅发出。 + * + *

这里用真实的 {@link SubscriptionRefreshService} 而不是 mock，才能真正覆盖 + * {@code cachedPath} 的整套配对契约：只有两份齐全才解析出真实文件。 + */ + @Test + void publicSubReturns503ForIncompleteCacheFiles(@org.junit.jupiter.api.io.TempDir java.nio.file.Path cacheRoot) + throws Exception { + var realResolver = new SubscriptionRefreshService(subMapper, new SubscriptionStateCoordinator()); + realResolver.cacheRoot = cacheRoot.toString(); + var boundService = new SubService(subMapper, userMapper, realResolver, remoteService, + new SubscriptionStateCoordinator()); + java.nio.file.Files.createDirectories(cacheRoot.resolve("1")); + java.nio.file.Files.writeString(cacheRoot.resolve("1/v2ray.txt"), "half"); + when(subMapper.selectSubBind("key1")).thenReturn(new SubBind("key1", "alice", 1, "name", true, false)); + var request = mock(jakarta.servlet.http.HttpServletRequest.class); + when(request.getHeader("User-Agent")).thenReturn("Mozilla/5.0"); + when(request.getRemoteAddr()).thenReturn("203.0.113.5"); + var response = mock(jakarta.servlet.http.HttpServletResponse.class); + + boundService.updateSub(response, request, "v2", "key1"); + + verify(response).sendError(eq(503), anyString()); + } + @Test void publicSubIgnoresNullKeyOrClient() throws Exception { var response = mock(jakarta.servlet.http.HttpServletResponse.class); @@ -366,6 +392,7 @@ class SubServiceTest { // 也避免单测真的去请求 ip138。 var recorded = new java.util.concurrent.atomic.AtomicReference(); service.locationExecutor = command -> recorded.set(command); + doAnswer(call -> { call.getArgument(0, com.lion.lionwebsite.Domain.SubUpdateRecord.class).setId(81); return null; }).when(subMapper).insertSubUpdateRecord(any()); long start = System.nanoTime(); service.updateSub(response, request, "v2", "key1"); @@ -389,4 +416,46 @@ class SubServiceTest { assertEquals("unknown", SubService.resolveLocation("203.0.113.5")); } } + + @Test + void outOfOrderLocationLookupsUpdateTheirOwnGeneratedRows() { + when(subMapper.selectSubBind("key1")).thenReturn(new SubBind("key1", "alice", 1, "name", true, false)); + when(refreshService.cachedPath(eq(1), anyString())).thenReturn(java.nio.file.Path.of("/nonexistent/cache/v2.txt")); + var request = mock(jakarta.servlet.http.HttpServletRequest.class); + when(request.getHeader("User-Agent")).thenReturn("test"); + when(request.getRemoteAddr()).thenReturn("203.0.113.1", "203.0.113.2"); + var nextId = new java.util.concurrent.atomic.AtomicInteger(100); + doAnswer(call -> { call.getArgument(0, com.lion.lionwebsite.Domain.SubUpdateRecord.class).setId(nextId.getAndIncrement()); return null; }) + .when(subMapper).insertSubUpdateRecord(any()); + var lookups = new java.util.ArrayList(); + service.locationExecutor = lookups::add; + var response = mock(jakarta.servlet.http.HttpServletResponse.class); + service.updateSub(response, request, "v2", "key1"); + service.updateSub(response, request, "v2", "key1"); + try (var location = mockStatic(SubService.class, CALLS_REAL_METHODS)) { + location.when(() -> SubService.resolveLocation("203.0.113.1")).thenReturn("first-location"); + location.when(() -> SubService.resolveLocation("203.0.113.2")).thenReturn("second-location"); + lookups.get(1).run(); + lookups.get(0).run(); + } + verify(subMapper).updateSubUpdateRecordLocation(100, "first-location"); + verify(subMapper).updateSubUpdateRecordLocation(101, "second-location"); + verify(subMapper, never()).updateLatestSubUpdateRecordLocation(anyString(), anyString()); + } + @Test + void busyLocationQueueDoesNotBreakSubscriptionDistribution() throws Exception { + when(subMapper.selectSubBind("key1")).thenReturn(new SubBind("key1", "alice", 1, "name", true, false)); + when(refreshService.cachedPath(eq(1), anyString())).thenReturn(java.nio.file.Path.of("/nonexistent/cache/v2.txt")); + doAnswer(call -> { call.getArgument(0, com.lion.lionwebsite.Domain.SubUpdateRecord.class).setId(91); return null; }) + .when(subMapper).insertSubUpdateRecord(any()); + var request = mock(jakarta.servlet.http.HttpServletRequest.class); + when(request.getHeader("User-Agent")).thenReturn("test"); + when(request.getRemoteAddr()).thenReturn("203.0.113.1"); + service.locationExecutor = command -> { throw new java.util.concurrent.RejectedExecutionException("busy"); }; + var response = mock(jakarta.servlet.http.HttpServletResponse.class); + assertDoesNotThrow(() -> service.updateSub(response, request, "v2", "key1")); + verify(subMapper).insertSubUpdateRecord(any()); + verify(response).sendError(jakarta.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE, "subscription unavailable"); + } + } diff --git a/src/test/java/com/lion/lionwebsite/Service/SubscriptionCacheGenerationTest.java b/src/test/java/com/lion/lionwebsite/Service/SubscriptionCacheGenerationTest.java new file mode 100644 index 0000000..84279c3 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Service/SubscriptionCacheGenerationTest.java @@ -0,0 +1,165 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.normal.SubMapper; +import com.lion.lionwebsite.Domain.SubscriptionAccount; +import com.lion.lionwebsite.Util.SubscriptionClientProfile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * 跨刷新的版本完整性：一次刷新产出的两份文件必须同版本，且过期的在途刷新不得覆盖更新的版本。 + */ +class SubscriptionCacheGenerationTest { + + private static SubscriptionAccount account(int id, String key) { + SubscriptionAccount a = new SubscriptionAccount(); + a.setId(id); + a.setName("acc-" + id); + a.setUpstreamKey(key); + a.setEnabled(true); + return a; + } + + private static List v2(String node) { + return List.of(Base64.getEncoder().encodeToString(node.getBytes(StandardCharsets.UTF_8))); + } + + private static String decodedV2(SubscriptionRefreshService service, int accountId) throws IOException { + Path path = service.cachedPath(accountId, "v2"); + return new String(Base64.getMimeDecoder().decode(Files.readString(path, StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } + + /** + * 在途的旧刷新在失效并发布新版本后才返回：必须被丢弃，不能把旧内容写回。 + */ + @Test + void staleInFlightRefreshCannotOverwriteNewerGeneration(@TempDir Path cacheRoot) throws Exception { + SubMapper mapper = mock(SubMapper.class); + SubscriptionAccount stale = account(1, "old-key"); + SubscriptionAccount fresh = account(1, "new-key"); + when(mapper.selectSubscriptionAccount(1)).thenReturn(stale); + SubscriptionStateCoordinator coordinator = new SubscriptionStateCoordinator(); + CountDownLatch blocked = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + + SubscriptionRefreshService service = new SubscriptionRefreshService(mapper, coordinator) { + @Override + List download(String url, SubscriptionClientProfile profile) throws IOException { + if (url.contains("old-key")) { + blocked.countDown(); + try { + if (!release.await(10, TimeUnit.SECONDS)) + throw new IOException("test timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + return v2("vmess://stale-node"); + } + return url.contains("client=v2") ? v2("vmess://fresh-node") : List.of("proxies: []"); + } + }; + service.cacheRoot = cacheRoot.toString(); + service.v2UrlTemplate = "https://upstream.invalid/{key}?client=v2"; + service.clashUrlTemplate = "https://upstream.invalid/{key}?client=clashmeta"; + + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future inFlight = worker.submit(() -> service.refresh(1)); + assertTrue(blocked.await(5, TimeUnit.SECONDS), "旧刷新应已进入下载阶段"); + + // 上游 key 被改掉并失效缓存，随后新刷新发布新版本。 + when(mapper.selectSubscriptionAccount(1)).thenReturn(fresh); + service.invalidateCache(1); + assertFalse(service.hasCompleteCache(1), "失效后不应还有可分发版本"); + + release.countDown(); + assertFalse(inFlight.get(10, TimeUnit.SECONDS), "过期的在途刷新必须被丢弃"); + + assertTrue(service.refresh(1)); + assertEquals("vmess://fresh-node\n", decodedV2(service, 1), "应只保留新版本内容"); + assertFalse(decodedV2(service, 1).contains("stale-node"), "过期刷新内容不得出现"); + // 只有新的那次刷新可以记成功；过期刷新必须完全没有留下成功记录。 + verify(mapper, times(1)).markSubscriptionRefreshSuccess(eq(1), anyLong()); + } finally { + release.countDown(); + worker.shutdownNow(); + } + } + + /** + * 过期刷新被丢弃后不得留下任何半成品：缓存要么是上一次的完整版本，要么为空。 + */ + @Test + void discardedRefreshLeavesNoPartialCache(@TempDir Path cacheRoot) throws Exception { + SubMapper mapper = mock(SubMapper.class); + SubscriptionAccount stale = account(2, "k-old"); + when(mapper.selectSubscriptionAccount(2)).thenReturn(stale); + SubscriptionStateCoordinator coordinator = new SubscriptionStateCoordinator(); + CountDownLatch blocked = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + + SubscriptionRefreshService service = new SubscriptionRefreshService(mapper, coordinator) { + @Override + List download(String url, SubscriptionClientProfile profile) throws IOException { + if (url.contains("k-old")) { + blocked.countDown(); + try { + if (!release.await(10, TimeUnit.SECONDS)) + throw new IOException("test timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + return v2("vmess://stale"); + } + return url.contains("client=v2") ? v2("vmess://x") : List.of("proxies: []"); + } + }; + service.cacheRoot = cacheRoot.toString(); + service.v2UrlTemplate = "https://upstream.invalid/{key}?client=v2"; + service.clashUrlTemplate = "https://upstream.invalid/{key}?client=clashmeta"; + + // 先成功发布一版，作为「上一次完整版本」。 + when(mapper.selectSubscriptionAccount(2)).thenReturn(account(2, "k-new")); + assertTrue(service.refresh(2)); + String before = decodedV2(service, 2); + + when(mapper.selectSubscriptionAccount(2)).thenReturn(stale); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future inFlight = worker.submit(() -> service.refresh(2)); + assertTrue(blocked.await(5, TimeUnit.SECONDS)); + when(mapper.selectSubscriptionAccount(2)).thenReturn(account(2, "k-newer")); + service.invalidateCache(2); + release.countDown(); + assertFalse(inFlight.get(10, TimeUnit.SECONDS)); + + assertEquals("vmess://x\n", before); + // 失效已清掉旧版本，而被丢弃的刷新又不得重新发布，因此当前应无可分发缓存。 + assertFalse(service.hasCompleteCache(2), "被丢弃的刷新不得留下缓存"); + // 只有首次成功刷新记过成功，被丢弃的那次不得再记。 + verify(mapper, times(1)).markSubscriptionRefreshSuccess(eq(2), anyLong()); + } finally { + release.countDown(); + worker.shutdownNow(); + } + } +} diff --git a/src/test/java/com/lion/lionwebsite/Service/SubscriptionFilteringTest.java b/src/test/java/com/lion/lionwebsite/Service/SubscriptionFilteringTest.java index f507532..99b74fe 100644 --- a/src/test/java/com/lion/lionwebsite/Service/SubscriptionFilteringTest.java +++ b/src/test/java/com/lion/lionwebsite/Service/SubscriptionFilteringTest.java @@ -67,6 +67,25 @@ class SubscriptionFilteringTest { doReturn(clash).when(service).download(contains("client=clashmeta"), any()); } + /** 读取该账号当前发布分代里的 v2 内容并解回明文。 */ + private String readV2(int accountId) throws Exception { + Path path = service.cachedPath(accountId, "v2"); + assertTrue(Files.isRegularFile(path), "应存在已发布的 v2 缓存: " + path); + return decodeV2(List.of(Files.readString(path, StandardCharsets.UTF_8))); + } + + /** 读取该账号当前发布分代里的 clash 内容。 */ + private String readClash(int accountId) throws Exception { + Path path = service.cachedPath(accountId, "cat"); + assertTrue(Files.isRegularFile(path), "应存在已发布的 clash 缓存: " + path); + return Files.readString(path, StandardCharsets.UTF_8); + } + + /** 空的合法 Clash 上游，用于只关心 v2 侧行为的用例。 */ + private static List emptyClash() { + return List.of("proxies: []"); + } + // ---------- URL 模板 ---------- /** {key} 必须被 URL 编码后替换，避免特殊字符破坏 URL。 */ @@ -100,10 +119,12 @@ class SubscriptionFilteringTest { assertTrue(service.refresh(1)); - Path v2 = cacheRoot.resolve("1/v2ray.txt"); - Path clash = cacheRoot.resolve("1/clash.yaml"); + Path v2 = service.cachedPath(1, "v2"); + Path clash = service.cachedPath(1, "cat"); assertTrue(Files.isRegularFile(v2), "应写入 v2 缓存"); assertTrue(Files.isRegularFile(clash), "应写入 clash 缓存"); + // 两份文件位于同一分代目录，才能保证读取到的内容同版本。 + assertEquals(v2.getParent(), clash.getParent(), "两份缓存必须来自同一分代"); // v2 缓存本身是 Base64 文本，需解码后再断言节点内容 String decodedV2 = new String(Base64.getMimeDecoder() .decode(Files.readString(v2)), StandardCharsets.UTF_8); @@ -111,7 +132,9 @@ class SubscriptionFilteringTest { verify(subMapper).markSubscriptionRefreshSuccess(eq(1), anyLong()); verify(subMapper, never()).markSubscriptionRefreshFailure(anyInt(), anyString()); // 临时文件不应残留 - assertFalse(Files.exists(cacheRoot.resolve("1/clash.yaml.tmp"))); + try (var files = Files.walk(cacheRoot)) { + assertTrue(files.noneMatch(path -> path.toString().endsWith(".tmp")), "临时文件不应残留"); + } } /** 停用账号或不存在账号直接返回 false，且不发任何网络请求。 */ @@ -136,12 +159,11 @@ class SubscriptionFilteringTest { stubUpstream(v2Upstream( "vmess://aaa#keep-1", "vmess://bbb#香港 5x", - "vmess://ccc#keep-2"), List.of("proxies:")); + "vmess://ccc#keep-2"), emptyClash()); assertTrue(service.refresh(5)); - String content = Files.readString(cacheRoot.resolve("5/v2ray.txt"), StandardCharsets.UTF_8); - String decoded = new String(Base64.getMimeDecoder().decode(content), StandardCharsets.UTF_8); + String decoded = readV2(5); assertTrue(decoded.contains("keep-1"), "实际内容: " + decoded); assertTrue(decoded.contains("keep-2"), "实际内容: " + decoded); assertFalse(decoded.contains("5x"), "高倍率节点应被剔除: " + decoded); @@ -152,12 +174,11 @@ class SubscriptionFilteringTest { void v2WithoutFilterKeepsHighMultiplierNodes() throws Exception { SubscriptionAccount acc = account(6, "key6", true, false); when(subMapper.selectSubscriptionAccount(6)).thenReturn(acc); - stubUpstream(v2Upstream("vmess://bbb#香港 5x"), List.of("proxies:")); + stubUpstream(v2Upstream("vmess://bbb#香港 5x"), emptyClash()); assertTrue(service.refresh(6)); - String content = Files.readString(cacheRoot.resolve("6/v2ray.txt"), StandardCharsets.UTF_8); - String decoded = new String(Base64.getMimeDecoder().decode(content), StandardCharsets.UTF_8); + String decoded = readV2(6); assertTrue(decoded.contains("5x"), "未开启过滤时应保留: " + decoded); } @@ -166,11 +187,11 @@ class SubscriptionFilteringTest { void v2FilterKeepsNodesWithoutMultiplierMarker() throws Exception { SubscriptionAccount acc = account(7, "key7", true, true); when(subMapper.selectSubscriptionAccount(7)).thenReturn(acc); - stubUpstream(v2Upstream("vmess://aaa#plain-node", "trojan://bbb#another"), List.of("proxies:")); + stubUpstream(v2Upstream("vmess://aaa#plain-node", "trojan://bbb#another"), emptyClash()); assertTrue(service.refresh(7)); - String decoded = decodeV2(List.of(Files.readString(cacheRoot.resolve("7/v2ray.txt")))); + String decoded = readV2(7); assertTrue(decoded.contains("plain-node")); assertTrue(decoded.contains("another")); } @@ -180,11 +201,11 @@ class SubscriptionFilteringTest { void v2FilterTreatsThresholdAsExclusive() throws Exception { SubscriptionAccount acc = account(8, "key8", true, true); when(subMapper.selectSubscriptionAccount(8)).thenReturn(acc); - stubUpstream(v2Upstream("vmess://aaa#节点 2x", "vmess://bbb#节点 2.1x"), List.of("proxies:")); + stubUpstream(v2Upstream("vmess://aaa#节点 2x", "vmess://bbb#节点 2.1x"), emptyClash()); assertTrue(service.refresh(8)); - String decoded = decodeV2(List.of(Files.readString(cacheRoot.resolve("8/v2ray.txt")))); + String decoded = readV2(8); assertTrue(decoded.contains("节点 2x"), "等于阈值应保留: " + decoded); assertFalse(decoded.contains("2.1x"), "超过阈值应剔除: " + decoded); } @@ -212,7 +233,7 @@ class SubscriptionFilteringTest { assertTrue(service.refresh(9)); - String yaml = Files.readString(cacheRoot.resolve("9/clash.yaml"), StandardCharsets.UTF_8); + String yaml = readClash(9); assertTrue(yaml.contains("keep"), "保留节点应在: " + yaml); assertFalse(yaml.contains("drop 5x"), "高倍率节点不应出现（含分组引用）: " + yaml); } @@ -234,7 +255,7 @@ class SubscriptionFilteringTest { assertTrue(service.refresh(10)); - String yaml = Files.readString(cacheRoot.resolve("10/clash.yaml"), StandardCharsets.UTF_8); + String yaml = readClash(10); assertTrue(yaml.contains("keep 5x"), "未过滤时应保留: " + yaml); } @@ -253,7 +274,7 @@ class SubscriptionFilteringTest { assertTrue(service.refresh(11)); - String yaml = Files.readString(cacheRoot.resolve("11/clash.yaml"), StandardCharsets.UTF_8); + String yaml = readClash(11); assertTrue(yaml.contains("group 9x"), "proxy-groups 段不应被过滤: " + yaml); } @@ -270,7 +291,7 @@ class SubscriptionFilteringTest { verify(subMapper).markSubscriptionRefreshFailure(eq(12), contains("为空")); verify(subMapper, never()).markSubscriptionRefreshSuccess(anyInt(), anyLong()); - assertFalse(Files.exists(cacheRoot.resolve("12/v2ray.txt")), "失败时不应产出缓存"); + assertFalse(service.hasCompleteCache(12), "失败时不应产出缓存"); } /** 上游不是合法 Base64 时必须失败并记录原因。 */ @@ -283,7 +304,7 @@ class SubscriptionFilteringTest { assertFalse(service.refresh(13)); verify(subMapper).markSubscriptionRefreshFailure(eq(13), contains("Base64")); - assertFalse(Files.exists(cacheRoot.resolve("13/v2ray.txt"))); + assertFalse(service.hasCompleteCache(13)); } /** 网络异常同样必须被兜住并记账，不能向上抛。 */ @@ -334,7 +355,7 @@ class SubscriptionFilteringTest { throw new java.io.IOException("account 22 upstream down"); if (url.contains("client=v2")) return List.of(Base64.getEncoder().encodeToString("vmess://a".getBytes(StandardCharsets.UTF_8))); - return List.of("proxies:"); + return emptyClash(); }).when(service).download(anyString(), any()); assertTrue(service.refresh(20)); @@ -342,8 +363,8 @@ class SubscriptionFilteringTest { verify(subMapper).markSubscriptionRefreshSuccess(eq(20), anyLong()); verify(subMapper).markSubscriptionRefreshFailure(eq(22), anyString()); - assertTrue(Files.exists(cacheRoot.resolve("20/v2ray.txt")), "成功账号应落盘"); - assertFalse(Files.exists(cacheRoot.resolve("22/v2ray.txt")), "失败账号不应落盘"); + assertTrue(service.hasCompleteCache(20), "成功账号应落盘"); + assertFalse(service.hasCompleteCache(22), "失败账号不应落盘"); } // ---------- 缓存状态 ---------- @@ -358,9 +379,12 @@ class SubscriptionFilteringTest { Files.writeString(cacheRoot.resolve("40/clash.yaml"), "y"); assertTrue(service.hasCompleteCache(40)); + // 旧结构两份齐全时也应能定位到可分发路径。 + assertEquals(cacheRoot.resolve("40/v2ray.txt"), service.cachedPath(40, "v2")); + assertEquals(cacheRoot.resolve("40/clash.yaml"), service.cachedPath(40, "cat")); } - /** 失效缓存应删除两份文件，使同一账号的被拒请求不再拿到旧内容。 */ + /** 失效缓存应清掉该账号的全部缓存，使同一账号的被拒请求不再拿到旧内容。 */ @Test void invalidateCacheRemovesBothFiles() throws Exception { Files.createDirectories(cacheRoot.resolve("41")); @@ -371,6 +395,7 @@ class SubscriptionFilteringTest { assertFalse(Files.exists(cacheRoot.resolve("41/v2ray.txt"))); assertFalse(Files.exists(cacheRoot.resolve("41/clash.yaml"))); + assertFalse(service.hasCompleteCache(41)); } /** 缓存文件本就不存在时不应抛异常。 */ @@ -378,4 +403,43 @@ class SubscriptionFilteringTest { void invalidateCacheIsIdempotent() { assertDoesNotThrow(() -> service.invalidateCache(42)); } + + /** + * 分发取用的路径必须来自完整版本：半缓存不能解析成真实文件， + * 否则 SubService 的 {@code Files.isRegularFile} 会放行残缺内容。 + */ + @Test + void cachedPathNeverPointsAtIncompleteLegacyCache() throws Exception { + Files.createDirectories(cacheRoot.resolve("43")); + Files.writeString(cacheRoot.resolve("43/v2ray.txt"), "half"); + + assertFalse(Files.isRegularFile(service.cachedPath(43, "v2")), + "只有一份旧文件时 cachedPath 不得指向真实文件"); + assertFalse(Files.isRegularFile(service.cachedPath(43, "cat"))); + assertFalse(service.hasCompleteCache(43)); + + // 补齐第二份后同一契约应立刻放行，且指向 legacy 文件本身。 + Files.writeString(cacheRoot.resolve("43/clash.yaml"), "half-yaml"); + assertTrue(service.hasCompleteCache(43)); + assertEquals(cacheRoot.resolve("43/v2ray.txt"), service.cachedPath(43, "v2")); + assertEquals(cacheRoot.resolve("43/clash.yaml"), service.cachedPath(43, "cat")); + } + + /** 抓取失败时必须保留上一份完整版本，而不是留下半截状态。 */ + @Test + void failedRefreshKeepsPreviousCompleteGeneration() throws Exception { + SubscriptionAccount acc = account(50, "key50", true, true); + when(subMapper.selectSubscriptionAccount(50)).thenReturn(acc); + stubUpstream(v2Upstream("vmess://good#node"), emptyClash()); + + assertTrue(service.refresh(50)); + String goodV2 = readV2(50); + assertTrue(goodV2.contains("good")); + + doThrow(new java.io.IOException("upstream 500")).when(service).download(anyString(), any()); + assertFalse(service.refresh(50)); + + assertTrue(service.hasCompleteCache(50), "失败后仍应保留上次完整版本"); + assertEquals(goodV2, readV2(50)); + } } diff --git a/src/test/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotServiceTest.java b/src/test/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotServiceTest.java index bed4489..cb6b5c8 100644 --- a/src/test/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotServiceTest.java +++ b/src/test/java/com/lion/lionwebsite/Service/SubscriptionStandbySnapshotServiceTest.java @@ -41,12 +41,11 @@ class SubscriptionStandbySnapshotServiceTest { default -> throw new UnsupportedOperationException(method.getName()); }); SubscriptionStateCoordinator coordinator = new SubscriptionStateCoordinator(); + // 旧结构（账号根下两份文件）仍应被读取为 legacy 版本。 Path accountDirectory = directory.resolve("1"); Files.createDirectories(accountDirectory); - Path v2 = accountDirectory.resolve("v2ray.txt"); - Path clash = accountDirectory.resolve("clash.yaml"); - Files.writeString(v2, "v2-content", StandardCharsets.UTF_8); - Files.writeString(clash, "clash-content", StandardCharsets.UTF_8); + Files.writeString(accountDirectory.resolve("v2ray.txt"), "v2-content", StandardCharsets.UTF_8); + Files.writeString(accountDirectory.resolve("clash.yaml"), "clash-content", StandardCharsets.UTF_8); SubscriptionStandbySnapshotService service = new SubscriptionStandbySnapshotService(mapper, coordinator); service.syncSecret = "test-sync-secret"; @@ -69,4 +68,79 @@ class SubscriptionStandbySnapshotServiceTest { assertEquals(2, payload.getBindings().size()); assertTrue(payload.getAccounts().getFirst().isFilterHighMultiplier()); } + + /** + * 快照必须从同一分代读出成对内容：即使分代目录里另有一份更新（未发布）的内容， + * 也只能取到指针指向的那一对，不能出现「新 v2 + 旧 clash」的组合。 + */ + @Test + void snapshotReadsPairedContentFromOneGeneration(@TempDir Path directory) throws Exception { + SubscriptionAccount account = new SubscriptionAccount(); + account.setId(5); + account.setName("paired"); + account.setUpstreamKey("upstream-secret"); + account.setEnabled(true); + ArrayList accounts = new ArrayList<>(java.util.List.of(account)); + SubMapper mapper = (SubMapper) Proxy.newProxyInstance(SubMapper.class.getClassLoader(), + new Class[]{SubMapper.class}, (proxy, method, args) -> switch (method.getName()) { + case "selectAllSubscriptionAccounts" -> accounts; + case "selectAllSubBind" -> new ArrayList(); + default -> throw new UnsupportedOperationException(method.getName()); + }); + var store = new com.lion.lionwebsite.Util.SubscriptionCacheStore(directory); + store.publish(5, "v2-published".getBytes(StandardCharsets.UTF_8), + "clash-published".getBytes(StandardCharsets.UTF_8)); + + SubscriptionStandbySnapshotService service = + new SubscriptionStandbySnapshotService(mapper, new SubscriptionStateCoordinator()); + service.syncSecret = "test-sync-secret"; + service.cacheRoot = directory.toString(); + service.cacheStore = store; + + SubscriptionSnapshotMessage message = service.build(); + byte[] compressed = Base64.getDecoder().decode(message.getPayloadBase64()); + byte[] json; + try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + json = gzip.readAllBytes(); + } + SubscriptionSnapshotPayload payload = CustomUtil.objectMapper.readValue(json, SubscriptionSnapshotPayload.class); + assertEquals(1, payload.getAccounts().size()); + var sent = payload.getAccounts().getFirst(); + assertEquals("v2-published", new String(Base64.getDecoder().decode(sent.getV2ContentBase64()), StandardCharsets.UTF_8)); + assertEquals("clash-published", new String(Base64.getDecoder().decode(sent.getClashContentBase64()), StandardCharsets.UTF_8)); + } + + /** 只有一份 legacy 文件（不完整）的账号不得进入快照。 */ + @Test + void snapshotSkipsIncompleteLegacyAccount(@TempDir Path directory) throws Exception { + SubscriptionAccount account = new SubscriptionAccount(); + account.setId(6); + account.setName("incomplete"); + account.setUpstreamKey("upstream-secret"); + account.setEnabled(true); + ArrayList accounts = new ArrayList<>(java.util.List.of(account)); + SubMapper mapper = (SubMapper) Proxy.newProxyInstance(SubMapper.class.getClassLoader(), + new Class[]{SubMapper.class}, (proxy, method, args) -> switch (method.getName()) { + case "selectAllSubscriptionAccounts" -> accounts; + case "selectAllSubBind" -> new ArrayList(); + default -> throw new UnsupportedOperationException(method.getName()); + }); + Path accountDirectory = directory.resolve("6"); + Files.createDirectories(accountDirectory); + Files.writeString(accountDirectory.resolve("v2ray.txt"), "only-v2", StandardCharsets.UTF_8); + + SubscriptionStandbySnapshotService service = + new SubscriptionStandbySnapshotService(mapper, new SubscriptionStateCoordinator()); + service.syncSecret = "test-sync-secret"; + service.cacheRoot = directory.toString(); + + SubscriptionSnapshotMessage message = service.build(); + byte[] compressed = Base64.getDecoder().decode(message.getPayloadBase64()); + byte[] json; + try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + json = gzip.readAllBytes(); + } + SubscriptionSnapshotPayload payload = CustomUtil.objectMapper.readValue(json, SubscriptionSnapshotPayload.class); + assertTrue(payload.getAccounts().isEmpty(), "不完整的旧缓存不应进入快照"); + } } diff --git a/src/test/java/com/lion/lionwebsite/Service/TaskLifecycleIntegrationTest.java b/src/test/java/com/lion/lionwebsite/Service/TaskLifecycleIntegrationTest.java new file mode 100644 index 0000000..241af00 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Service/TaskLifecycleIntegrationTest.java @@ -0,0 +1,818 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.normal.*; +import com.lion.lionwebsite.Domain.*; +import com.lion.lionwebsite.Exception.ResolutionNotMatchException; +import com.lion.lionwebsite.Message.TaskCommandMessage; +import com.lion.lionwebsite.Message.TaskReportMessage; +import com.lion.lionwebsite.Util.PreparedGallery; +import com.lion.lionwebsite.Util.Response; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mybatis.spring.SqlSessionFactoryBean; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.Connection; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * 任务生命周期在真实 SQLite 上的故障行为。 + * + *

这里刻意不用 mock 的 mapper：只有真实事务、真实唯一约束与真实回滚才能验证 + * 「先持久化意图再 POST」「提交未知不重发」「任务+额度同事务」「删除失败保留记录」。 + * 数据库是 /tmp 下的临时副本，不接触生产库。 + */ +class TaskLifecycleIntegrationTest { + + @TempDir + Path directory; + + private SqlSessionTemplate template; + private DriverManagerDataSource source; + private GalleryMapper galleries; + private CollectMapper collects; + private CustomConfigurationMapper configurations; + private TaskOperationRepository operations; + private RemoteService remote; + private GalleryTaskLifecycleService service; + private FakeGateway gateway; + + @BeforeEach + void setUp() throws Exception { + // 主数据源刻意带上 busy_timeout：生产库同样是多线程共用一个 SQLite 文件， + // 没有 busy 重试时并发事务会直接抛 SQLITE_BUSY。 + source = new DriverManagerDataSource("jdbc:sqlite:" + directory.resolve("lifecycle.db") + "?busy_timeout=10000"); + try (Connection connection = source.getConnection(); var statement = connection.createStatement()) { + statement.executeUpdate("create table gallery (gid integer not null primary key, name varchar(100)," + + " link varchar(100), language varchar(20), pages integer, status varchar(20), fileSize long," + + " createTime double, proceeding integer, resolution varchar(20), displayFileSize varchar(30)," + + " downloader integer, thumb_link varchar(255), is_download boolean)"); + statement.executeUpdate("create table collect (id integer primary key autoincrement, gid integer not null," + + " collector integer not null)"); + statement.executeUpdate("create table customConfiguration (parameter varchar(30), value varchar(100))"); + statement.executeUpdate("create table task_operation (gid integer not null primary key, operationId varchar(64)," + + " deleteOperationId varchar(64), state varchar(20) not null, lastRevision integer not null default 0," + + " lastReportId varchar(64), updateTime long not null default 0)"); + statement.executeUpdate("insert into customConfiguration values('weekUsedAmount','0')"); + statement.executeUpdate("insert into customConfiguration values('lastResetAmountTime','2026-10-01 00:00:00')"); + // 任务生命周期会并发开启事务写同一个 SQLite 文件。WAL 让读不阻塞写， + // 并让写-写冲突走 busy 重试而不是直接 SQLITE_BUSY（与生产库要求的配置一致）。 + statement.execute("PRAGMA journal_mode=WAL"); + } + var bean = new SqlSessionFactoryBean(); + bean.setDataSource(source); + var factory = bean.getObject(); + factory.getConfiguration().addMapper(GalleryMapper.class); + factory.getConfiguration().addMapper(CollectMapper.class); + factory.getConfiguration().addMapper(CustomConfigurationMapper.class); + factory.getConfiguration().addMapper(TaskOperationRepository.class); + template = new SqlSessionTemplate(factory); + galleries = template.getMapper(GalleryMapper.class); + collects = template.getMapper(CollectMapper.class); + configurations = template.getMapper(CustomConfigurationMapper.class); + operations = template.getMapper(TaskOperationRepository.class); + + remote = mock(RemoteService.class); + gateway = new FakeGateway(); + service = new GalleryTaskLifecycleService(galleries, collects, configurations, mock(UserMapper.class), + operations, gateway, remote, new TransactionTemplate(new DataSourceTransactionManager(source))); + } + + /** 可编程的假上游：记录调用顺序，按脚本决定 prepare/submit 的结果。 */ + private class FakeGateway implements UpstreamGateway { + final List calls = new CopyOnWriteArrayList<>(); + PreparedGallery prepared; + boolean prepareReturnsNull; + IOException prepareFailure; + IOException submitFailure; + String submitStatus = TaskStatus.SUBMITTED; + long fileSize = 4096L; + Runnable duringPrepare = () -> { }; + Runnable duringSubmit = () -> { }; + + @Override + public PreparedGallery prepare(String link) throws IOException { + calls.add("prepare"); + duringPrepare.run(); + if (prepareFailure != null) + throw prepareFailure; + if (prepareReturnsNull) + return null; + if (prepared != null) + return prepared; + Gallery gallery = new Gallery(); + gallery.setStatus("等待确认下载"); + gallery.setFileSize(fileSize); + gallery.setName("sample [" + 123 + "]"); + gallery.setPages(10); + return new PreparedGallery(gallery, "https://example.org/dl", "https://example.org"); + } + + @Override + public Gallery submit(PreparedGallery prepared, String targetResolution) throws IOException { + calls.add("submit"); + duringSubmit.run(); + if (submitFailure != null) + throw submitFailure; + Gallery gallery = prepared.gallery(); + gallery.setStatus(submitStatus); + gallery.setResolution(targetResolution); + return gallery; + } + } + + private static Gallery gallery(int gid, String name, String status) { + Gallery gallery = new Gallery(); + gallery.setGid(gid); + gallery.setName(name); + gallery.setStatus(status); + return gallery; + } + + private TaskOperation operation(int gid) { + return operations.selectByGid(gid); + } + + /** 与主服务相同的真实事务模板；用于构造额外的服务实例。 */ + private TransactionTemplate sasTransactions() { + return new TransactionTemplate(new DataSourceTransactionManager(source)); + } + + /** 读取任务行；对 SQLITE_BUSY 做有限重试，避免并发用例里的瞬时锁冲突影响断言。 */ + private Gallery storedGallery(int gid) { + for (int attempt = 0; ; attempt++) { + try { + return galleries.selectGalleryByGid(gid); + } catch (RuntimeException e) { + if (attempt >= 4 || !String.valueOf(e.getMessage()).contains("locked")) + throw e; + try { + Thread.sleep(200); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + } + + private long weekUsed() { + return Long.parseLong(configurations.selectConfiguration(CustomConfiguration.WEEK_USED_AMOUNT).getValue()); + } + + /** 直接用 JDBC 数行，避免依赖 MyBatis 的动态 SQL 语句 ID。 */ + private int rawInt(String sql) { + try (Connection connection = new DriverManagerDataSource(dbUrl()).getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery(sql)) { + return rows.next() ? rows.getInt(1) : 0; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private String dbUrl() { + return "jdbc:sqlite:" + directory.resolve("lifecycle.db") + "?busy_timeout=10000"; + } + + // ---------- 意图先于外部副作用 ---------- + + /** CREATE 意图（含可见占位行）必须在上游 POST 之前已经落库。 */ + @Test + void createIntentIsPersistedBeforeUpstreamSubmit() { + gateway.duringPrepare = () -> { + TaskOperation reserved = operation(123); + assertNotNull(reserved, "prepare 之前必须有唯一 CREATE 意图"); + assertEquals(TaskOperation.STATE_RESERVED, reserved.getState()); + Gallery placeholder = galleries.selectGalleryByGid(123); + assertNotNull(placeholder, "提交期间任务必须在列表里可见"); + assertEquals(TaskStatus.PENDING_SUBMIT, placeholder.getStatus()); + }; + gateway.duringSubmit = () -> { + assertEquals(TaskOperation.STATE_SUBMITTING, operation(123).getState(), + "真正 POST 之前必须已持久化 SUBMITTING"); + }; + + Response response = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + + assertTrue(response.isSuccess(), response.toJSONString()); + assertEquals(List.of("prepare", "submit"), gateway.calls); + assertEquals(TaskOperation.STATE_SUBMITTED, operation(123).getState()); + assertEquals(TaskStatus.SUBMITTED, galleries.selectGalleryByGid(123).getStatus()); + assertEquals(7, galleries.selectGalleryByGid(123).getDownloader()); + verify(remote).sendTaskCommand(eq(TaskCommandMessage.ACTION_DOWNLOAD), anyString(), eq(123), anyString()); + } + + /** + * 并发提交同一 gid（真实 SQLite + 真实唯一键）：恰好一个赢家、上游只 POST 一次， + * 其余全部返回业务失败，绝不抛异常穿透成 500。 + */ + @Test + void concurrentCreateForSameGidSubmitsUpstreamOnce() throws Exception { + int threads = 6; + var start = new CountDownLatch(1); + var pool = Executors.newFixedThreadPool(threads); + var accepted = new AtomicInteger(); + var failed = new AtomicInteger(); + List> futures = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + futures.add(pool.submit(() -> { + start.await(); + Response response = service.createTask(4242, "任务 [4242]", + "https://example.org/g/4242/key/", "original", 7); + if (response.isSuccess()) + accepted.incrementAndGet(); + else { + // 输家必须拿到结构化业务失败（锁竞争不得逃逸成异常）。 + assertNotNull(response.getData()); + failed.incrementAndGet(); + } + return null; + })); + } + start.countDown(); + for (Future future : futures) + future.get(30, TimeUnit.SECONDS); + pool.shutdownNow(); + + assertEquals(1, accepted.get(), "同一 gid 必须恰好有一个成功创建者"); + assertEquals(threads - 1, failed.get(), "其余请求必须全部返回业务失败"); + assertEquals(List.of("prepare", "submit"), gateway.calls, "上游只应被提交一次"); + assertEquals(1, (int) rawInt("select count(*) from gallery where gid=4242"), + "同一 gid 不允许出现两条任务行"); + assertEquals(1, (int) rawInt("select count(*) from task_operation where gid=4242")); + assertNotNull(operation(4242), "唯一意图必须存在"); + } + + /** 预占阶段的数据库失败必须转成业务失败且绝不 POST 上游，不能穿透成 500。 */ + @Test + void reservationDatabaseFailureBecomesBusinessFailureWithoutSubmit() { + GalleryMapper failing = mock(GalleryMapper.class); + when(failing.selectGalleryByGid(anyInt())).thenReturn(null); + doThrow(new org.springframework.dao.DataIntegrityViolationException("db down")) + .when(failing).insertGallery(any()); + GalleryTaskLifecycleService failingService = new GalleryTaskLifecycleService(failing, collects, + configurations, mock(UserMapper.class), operations, gateway, remote, sasTransactions()); + + Response response = assertDoesNotThrow( + () -> failingService.createTask(555, "任务 [555]", "https://example.org/g/555/key/", "original", 7)); + + assertFalse(response.isSuccess()); + assertTrue(gateway.calls.isEmpty(), "预占失败绝不能产生上游副作用"); + } + + /** 解析阶段失败（读取异常，未产生上游副作用）：预占被释放，可重新提交。 */ + @Test + void prepareFailureReleasesIntentSoResubmissionIsPossible() { + gateway.prepareFailure = new IOException("network"); + + Response first = service.createTask(55, "任务 [55]", "https://example.org/g/55/key/", "original", 7); + assertFalse(first.isSuccess()); + assertNull(operation(55), "没有副作用时预占应被释放"); + assertNull(galleries.selectGalleryByGid(55), "占位行也应一并释放"); + + gateway.prepareFailure = null; + Response second = service.createTask(55, "任务 [55]", "https://example.org/g/55/key/", "original", 7); + assertTrue(second.isSuccess(), "释放后应能重新提交"); + } + + /** + * 释放预占必须严格匹配世代：若解析期间该 gid 已换代（出现新的 RESERVED 记录）， + * 旧请求的清理绝不能删掉新世代的意图与占位行。 + */ + @Test + void releaseReservationDoesNotTouchNewerGeneration() { + gateway.prepareFailure = new IOException("network"); + service.beforePrepare = () -> { + String staleGeneration = operation(123).getOperationId(); + assertNotNull(staleGeneration); + // 模拟该 gid 已被新的创建请求接管：换成新 operationId，占位行仍在。 + operations.updateOperationId(123, "dl-newer-generation", TaskOperation.STATE_RESERVED, 1); + }; + + Response response = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + + assertFalse(response.isSuccess()); + assertNotNull(operation(123), "新世代的意图不得被旧请求释放"); + assertEquals("dl-newer-generation", operation(123).getOperationId()); + assertNotNull(galleries.selectGalleryByGid(123), "新世代的占位行不得被旧请求删除"); + } + + // ---------- 提交未知绝不自动重发 ---------- + + /** 上游提交超时/断连：落 UNKNOWN，任务行保留，后续同 gid 创建不得再次 POST 上游。 */ + @Test + void unknownSubmitNeverResendsUpstream() { + gateway.submitFailure = new IOException("timeout after request was sent"); + + Response first = service.createTask(77, "任务 [77]", "https://example.org/g/77/key/", "original", 7); + + assertFalse(first.isSuccess(), "结果未知不能报成功"); + assertEquals(TaskOperation.STATE_UNKNOWN, operation(77).getState()); + assertNotNull(galleries.selectGalleryByGid(77), "未知任务必须留在列表里可对账"); + // Gallery 与 TaskOperation 必须一致表达「提交未知」：否则运营侧会以为还能自动重试。 + assertEquals(TaskStatus.UNKNOWN, galleries.selectGalleryByGid(77).getStatus()); + assertEquals(List.of("prepare", "submit"), gateway.calls); + + // 用户再点一次：只能拿到「已有此任务」，绝不能第二次 POST 上游。 + gateway.submitFailure = null; + Response second = service.createTask(77, "任务 [77]", "https://example.org/g/77/key/", "original", 7); + assertFalse(second.isSuccess()); + assertEquals(List.of("prepare", "submit"), gateway.calls, "未知结果不得自动重发上游"); + assertEquals(TaskOperation.STATE_UNKNOWN, operation(77).getState()); + } + + /** 重启恢复：中断在 SUBMITTING 的意图被判为 UNKNOWN，而不是重新提交。 */ + @Test + void restartRecoversInterruptedSubmittingToUnknown() { + TaskOperation interrupted = new TaskOperation(); + interrupted.setGid(88); + interrupted.setOperationId("dl-interrupted"); + interrupted.setState(TaskOperation.STATE_SUBMITTING); + interrupted.setUpdateTime(1); + operations.insert(interrupted); + galleries.insertGallery(gallery(88, "任务 [88]", TaskStatus.PENDING_SUBMIT)); + + assertEquals(1, service.recoverInterruptedSubmissions()); + + assertEquals(TaskOperation.STATE_UNKNOWN, operation(88).getState()); + assertEquals("dl-interrupted", operation(88).getOperationId(), "恢复不得改写 operationId"); + assertEquals(TaskStatus.UNKNOWN, galleries.selectGalleryByGid(88).getStatus(), + "启动恢复后任务行也必须显示提交未知，供人工对账"); + assertTrue(gateway.calls.isEmpty(), "恢复过程不得再 POST 上游"); + } + + /** 重启恢复：还没产生上游副作用的 RESERVED 预占应被安全释放，让用户能重新提交。 */ + @Test + void restartReleasesReservedPlaceholders() { + TaskOperation reserved = new TaskOperation(); + reserved.setGid(89); + reserved.setOperationId("dl-reserved"); + reserved.setState(TaskOperation.STATE_RESERVED); + reserved.setUpdateTime(1); + operations.insert(reserved); + galleries.insertGallery(gallery(89, "任务 [89]", TaskStatus.PENDING_SUBMIT)); + + assertEquals(1, service.recoverInterruptedSubmissions()); + + assertNull(operation(89), "RESERVED 尚未 POST，重启应释放占位意图"); + assertNull(galleries.selectGalleryByGid(89), "占位行也应一并释放"); + assertTrue(gateway.calls.isEmpty(), "释放预占不得 POST 上游"); + } + + /** 释放预占只有确实删掉匹配记录时才删占位行：不匹配时任务行必须保留。 */ + @Test + void restartKeepsPlaceholderWhenReservationRowDoesNotMatch() { + TaskOperation mismatched = new TaskOperation(); + mismatched.setGid(90); + mismatched.setOperationId(null); + mismatched.setState(TaskOperation.STATE_RESERVED); + mismatched.setUpdateTime(1); + operations.insert(mismatched); + galleries.insertGallery(gallery(90, "任务 [90]", TaskStatus.PENDING_SUBMIT)); + + service.recoverInterruptedSubmissions(); + + assertNotNull(galleries.selectGalleryByGid(90), "没有删掉匹配意图时不得删占位行"); + } + + // ---------- 任务行与额度同一短事务 ---------- + + /** 提交成功：任务行与周用量一起写入；重复落库不重复计量。 */ + @Test + void submittedTaskAndQuotaCommitTogetherAndQuotaIsIdempotent() { + Response response = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + assertTrue(response.isSuccess(), response.toJSONString()); + assertEquals(4096L, weekUsed(), "用量应按文件大小累计一次"); + + // 模拟上游已提交后重跑同一意图：不得再计一次用量。 + service.persistSubmitted(galleries.selectGalleryByGid(123), 7, operation(123).getOperationId()); + assertEquals(4096L, weekUsed(), "重复落库不得重复计用量"); + } + + /** 落库必须严格匹配本次创建世代：被删除/重建后不得复活已删 gid 或污染新世代。 */ + @Test + void persistSubmittedRejectsStaleGeneration() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String liveGeneration = operation(123).getOperationId(); + + assertThrows(IllegalStateException.class, + () -> service.persistSubmitted(galleries.selectGalleryByGid(123), 7, "dl-stale-generation"), + "operationId 不匹配时禁止落库"); + assertEquals(4096L, weekUsed(), "被拒的落库不得重复计用量"); + assertNotNull(operation(123), "被拒后原世代记录保持不动"); + assertEquals(liveGeneration, operation(123).getOperationId()); + + service.beginDelete(123); + assertThrows(IllegalStateException.class, + () -> service.persistSubmitted(galleries.selectGalleryByGid(123), 7, liveGeneration), + "待删除/已删除的任务禁止落库复活"); + } + + /** 用量写入失败时任务行必须回滚，不能留下「有任务没计量」的半截状态。 */ + @Test + void quotaFailureRollsBackTaskRow() throws Exception { + // 用真实触发器让 customConfiguration 更新失败，验证事务真的回滚任务行。 + try (Connection connection = new DriverManagerDataSource("jdbc:sqlite:" + directory.resolve("lifecycle.db") + "?busy_timeout=10000") + .getConnection(); var statement = connection.createStatement()) { + statement.executeUpdate("create trigger reject_quota before update on customConfiguration" + + " when new.parameter='weekUsedAmount' begin select raise(ABORT,'simulated quota failure'); end"); + } + + Response response = service.createTask(124, "任务 [124]", "https://example.org/g/124/key/", "original", 7); + + assertFalse(response.isSuccess(), "额度写失败必须回业务失败"); + // 提交后的落库事务失败时要回滚本次写入，但「上游可能已受理」这件事不能丢： + // 占位行仍在列表里可见，操作记录标为提交未知，二者一起支持后续人工对账， + // 且绝不允许自动重发上游。 + assertNotNull(storedGallery(124), "上游已受理的任务必须保持可见，供对账"); + assertEquals(TaskStatus.UNKNOWN, storedGallery(124).getStatus(), "落库失败也必须明确标为提交未知"); + assertNull(storedGallery(124).getResolution(), "失败的落库不得留下分辨率等半截字段"); + assertEquals(TaskOperation.STATE_UNKNOWN, operation(124).getState(), + "上游已受理但本地未落库：只能记 UNKNOWN 等对账"); + } + + // ---------- 稳定 operationId ---------- + + /** 重投/重试同一任务必须复用同一个下载 operationId。 */ + @Test + void redispatchReusesStableOperationId() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String first = operation(123).getOperationId(); + + service.dispatchDownload(123, "任务 [123]"); + service.dispatchDownload(123, "任务 [123]"); + + assertEquals(first, operation(123).getOperationId(), "重投不得更换 operationId"); + verify(remote, times(3)).sendTaskCommand(eq(TaskCommandMessage.ACTION_DOWNLOAD), eq(first), eq(123), anyString()); + } + + // ---------- 删除：失败保留、成功留墓碑 ---------- + + /** 删除失败：任务行与待删除意图都保留，可沿原记录重试并复用同一个删除 ID。 */ + @Test + void deleteFailureRetainsTaskAndReusesDeleteOperationId() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + collects.collect(123, 7); + + GalleryTaskLifecycleService.DeleteIntent intent = service.beginDelete(123); + assertTrue(intent.accepted(), intent.message()); + String deleteId = intent.deleteOperationId(); + assertNotNull(deleteId); + assertEquals(TaskOperation.STATE_DELETE_PENDING, operation(123).getState()); + assertEquals(TaskStatus.DELETE_PENDING, galleries.selectGalleryByGid(123).getStatus()); + + service.failDelete(123); + assertNotNull(galleries.selectGalleryByGid(123), "删除失败必须保留任务记录"); + assertEquals(TaskStatus.DELETE_FAILED, galleries.selectGalleryByGid(123).getStatus()); + + assertTrue(service.beginDelete(123).accepted()); + assertEquals(deleteId, service.beginDelete(123).deleteOperationId(), + "重试必须复用同一个删除 operationId"); + assertEquals(1, (int) rawInt("select count(*) from collect where gid=123"), + "删除未完成前收藏记录保持不动"); + } + + /** 删除完成：任务行、收藏与状态在一个事务里落定并留墓碑。 */ + @Test + void completedDeleteRemovesRowAndCollectAndLeavesTombstone() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + collects.collect(123, 7); + GalleryTaskLifecycleService.DeleteIntent intent = service.beginDelete(123); + + assertTrue(service.completeDelete(123, intent.deleteOperationId())); + + assertNull(galleries.selectGalleryByGid(123)); + assertEquals(0, (int) rawInt("select count(*) from collect where gid=123")); + assertEquals(TaskOperation.STATE_DELETED, operation(123).getState()); + assertNull(operation(123).getDeleteOperationId(), "墓碑不再持有删除 ID"); + } + + /** 删除意图 ID 不匹配时不得完成删除：防止并发换代把别的一代抹掉。 */ + @Test + void completeDeleteRequiresMatchingDeleteIntent() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + service.beginDelete(123); + + assertFalse(service.completeDelete(123, "del-not-mine"), "不匹配的删除意图必须被拒"); + assertNotNull(galleries.selectGalleryByGid(123), "被拒时任务行必须保留"); + assertEquals(TaskOperation.STATE_DELETE_PENDING, operation(123).getState()); + } + + /** 正在提交（RESERVED/SUBMITTING）的任务不能开始删除：否则 DELETE 会与创建 POST 同时发生。 */ + @Test + void beginDeleteRefusesWhileCreateIsInFlight() { + service.beforePrepare = () -> { + GalleryTaskLifecycleService.DeleteIntent duringPrepare = service.beginDelete(123); + assertFalse(duringPrepare.accepted(), "提交中的任务必须拒绝删除"); + assertEquals("任务正在提交中，请等待提交结果明确后再删除", duringPrepare.message()); + }; + service.afterMarkSubmitting = () -> { + GalleryTaskLifecycleService.DeleteIntent duringSubmitting = service.beginDelete(123); + assertFalse(duringSubmitting.accepted(), "SUBMITTING 阶段仍必须拒绝删除"); + }; + + Response response = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + + assertTrue(response.isSuccess(), response.toJSONString()); + assertEquals(TaskOperation.STATE_SUBMITTED, operation(123).getState()); + assertEquals(TaskStatus.SUBMITTED, galleries.selectGalleryByGid(123).getStatus()); + } + + /** + * latch 控制的跨线程并发删除：删除线程在创建线程处于 RESERVED（prepare 期间）时进入， + * 必须被拒；创建线程随后正常完成，两者都不产生「删了又回写」。 + */ + @Test + void concurrentDeleteDuringPrepareIsRejectedAndCreateCompletes() throws Exception { + var prepared = new CountDownLatch(1); + var deleteAttempted = new CountDownLatch(1); + var deleteAccepted = new java.util.concurrent.atomic.AtomicBoolean(); + service.beforePrepare = () -> { + prepared.countDown(); + Thread deleter = new Thread(() -> { + try { + prepared.await(); + deleteAccepted.set(service.beginDelete(123).accepted()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + deleteAttempted.countDown(); + } + }); + deleter.start(); + }; + + Response response = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + assertTrue(deleteAttempted.await(10, TimeUnit.SECONDS), "删除线程应已给出结论"); + + assertTrue(response.isSuccess(), response.toJSONString()); + assertFalse(deleteAccepted.get(), "创建意图未决时并发删除必须被拒"); + assertEquals(TaskOperation.STATE_SUBMITTED, operation(123).getState()); + assertEquals(TaskStatus.SUBMITTED, galleries.selectGalleryByGid(123).getStatus()); + } + + /** 墓碑允许用全新 operationId 重新创建；旧世代的迟到上报仍被丢弃。 */ + @Test + void tombstoneAllowsFreshCreateWithNewGeneration() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String oldOperationId = operation(123).getOperationId(); + String deleteId = service.beginDelete(123).deleteOperationId(); + service.completeDelete(123, deleteId); + + Response recreated = service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + + assertTrue(recreated.isSuccess(), "墓碑之后应能用新 operationId 重新创建"); + assertNotEquals(oldOperationId, operation(123).getOperationId()); + assertEquals(0L, operation(123).getLastRevision(), "新世代 revision 必须清零"); + + // 旧世代的迟到上报：operationId 不匹配 → 丢弃，且不得把新任务改回完成。 + TaskReportMessage stale = report("stale-report", 5, Map.of(123, oldOperationId), + task(123, TaskStatus.COMPLETE, 10)); + TaskReportHandler.CommittedReport committed = service.apply(stale); + assertNotNull(committed, "应当回执以停止重放"); + assertTrue(committed.acceptedTasks().isEmpty(), "旧世代上报不得被接受"); + assertNotEquals(TaskStatus.COMPLETE, galleries.selectGalleryByGid(123).getStatus()); + assertEquals(0L, operation(123).getLastRevision()); + } + + /** 墓碑必须永久忽略旧上报：不管上报里的 operationId 是否等于已删除的那一代。 */ + @Test + void tombstoneIgnoresMatchingGenerationReport() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String deletedOperationId = operation(123).getOperationId(); + String deleteId = service.beginDelete(123).deleteOperationId(); + service.completeDelete(123, deleteId); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-tomb", 9, Map.of(123, deletedOperationId), task(123, TaskStatus.COMPLETE, 10))); + + assertNotNull(committed, "墓碑上报仍要回执以停止重放"); + assertTrue(committed.acceptedTasks().isEmpty(), "已删除任务不得被上报复活"); + assertNull(galleries.selectGalleryByGid(123), "墓碑不得重新插入任务行"); + assertEquals(TaskOperation.STATE_DELETED, operation(123).getState(), "墓碑状态必须保持"); + } + + // ---------- v2 状态上报 ---------- + + private static GalleryTask task(int gid, String status, int proceeding) { + GalleryTask task = new GalleryTask(); + task.setGid(gid); + task.setName("任务 [" + gid + "]"); + task.setStatus(switch (status) { + case TaskStatus.DOWNLOADING -> GalleryTask.DOWNLOADING; + case TaskStatus.WAITING_COMPRESS -> GalleryTask.DOWNLOAD_COMPLETE; + case TaskStatus.COMPRESSING -> GalleryTask.COMPRESSING; + default -> GalleryTask.COMPRESS_COMPLETE; + }); + task.setProceeding(proceeding); + return task; + } + + private static TaskReportMessage report(String reportId, long revision, + Map operationIds, GalleryTask... tasks) { + TaskReportMessage report = new TaskReportMessage(); + report.setReportId(reportId); + report.setRevision(revision); + report.setGalleryTasks(tasks); + report.setOperationIds(new LinkedHashMap<>(operationIds)); + return report; + } + + /** 正常上报：落库、记录回执、并把下载中这类中间状态回给前端。 */ + @Test + void reportAppliesProgressAndRecordsReceipt() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String operationId = operation(123).getOperationId(); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-1", 10, Map.of(123, operationId), task(123, TaskStatus.DOWNLOADING, 4))); + + assertNotNull(committed); + assertEquals("r-1", committed.reportId()); + assertEquals(1, committed.acceptedTasks().size(), "中间状态也要回给前端"); + assertEquals(TaskStatus.DOWNLOADING, galleries.selectGalleryByGid(123).getStatus()); + assertEquals(4, galleries.selectGalleryByGid(123).getProceeding()); + assertEquals(10L, operation(123).getLastRevision()); + assertEquals("r-1", operation(123).getLastReportId()); + } + + /** 乱序/过期 revision：丢弃但不影响已提交状态，且仍需回执停止重放。 */ + @Test + void staleRevisionIsIgnoredButStillAcked() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String operationId = operation(123).getOperationId(); + service.apply(report("r-5", 50, Map.of(123, operationId), task(123, TaskStatus.DOWNLOADING, 7))); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-old", 20, Map.of(123, operationId), task(123, TaskStatus.DOWNLOADING, 1))); + + assertNotNull(committed); + assertTrue(committed.acceptedTasks().isEmpty()); + assertEquals(7, galleries.selectGalleryByGid(123).getProceeding(), "过期上报不得回退进度"); + assertEquals(50L, operation(123).getLastRevision()); + } + + /** 重复上报同一份快照：仍需回执（让节点停止重放），但不得重复通知。 */ + @Test + void duplicateReportAcksWithoutDuplicateCompletion() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String operationId = operation(123).getOperationId(); + TaskReportMessage completion = report("r-done", 30, Map.of(123, operationId), + task(123, TaskStatus.COMPLETE, 10)); + + TaskReportHandler.CommittedReport first = service.apply(completion); + TaskReportHandler.CommittedReport repeated = service.apply(completion); + + assertNotNull(first); + assertEquals(1, first.newlyCompleted().size(), "首次完成应触发一次通知"); + assertNotNull(repeated, "重复上报仍要回执"); + assertTrue(repeated.newlyCompleted().isEmpty(), "重复上报不得重复通知"); + assertEquals(TaskStatus.COMPLETE, galleries.selectGalleryByGid(123).getStatus()); + } + + /** 待删除任务的迟到完成上报：忽略，不得把它改回完成。 */ + @Test + void reportForPendingDeleteTaskIsIgnored() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String operationId = operation(123).getOperationId(); + service.beginDelete(123); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-late", 40, Map.of(123, operationId), task(123, TaskStatus.COMPLETE, 10))); + + assertNotNull(committed); + assertTrue(committed.acceptedTasks().isEmpty()); + assertEquals(TaskStatus.DELETE_PENDING, galleries.selectGalleryByGid(123).getStatus()); + } + + /** 未知 gid（任务已删除）：忽略但回执，避免节点无限重放。 */ + @Test + void unknownGidIsIgnoredButAcked() { + TaskReportHandler.CommittedReport committed = service.apply( + report("r-ghost", 1, Map.of(999, "dl-ghost"), task(999, TaskStatus.DOWNLOADING, 1))); + + assertNotNull(committed, "未知 gid 也要回执"); + assertTrue(committed.acceptedTasks().isEmpty()); + assertNull(galleries.selectGalleryByGid(999)); + } + + /** v2 任务缺 operationId 的上报：无法归属，必须丢弃而不是改当前任务。 */ + @Test + void reportWithoutOperationIdForV2TaskIsRejected() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-noop", 60, Map.of(), task(123, TaskStatus.COMPLETE, 10))); + + assertNotNull(committed); + assertTrue(committed.acceptedTasks().isEmpty(), "缺 operationId 不得改当前任务"); + assertNotEquals(TaskStatus.COMPLETE, galleries.selectGalleryByGid(123).getStatus()); + assertEquals(0L, operation(123).getLastRevision()); + } + + /** 迁移前的历史任务行：首次上报按显式策略建立世代后正常接受。 */ + @Test + void migratedLegacyTaskAdoptsGenerationOnFirstReport() { + galleries.insertGallery(gallery(321, "legacy [321]", TaskStatus.DOWNLOADING)); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-legacy", 3, Map.of(321, "dl-legacy"), task(321, TaskStatus.COMPRESSING, 9))); + + assertNotNull(committed); + assertEquals(1, committed.acceptedTasks().size()); + assertEquals("dl-legacy", operation(321).getOperationId(), "应采纳上报里的世代 ID"); + assertEquals(TaskStatus.COMPRESSING, galleries.selectGalleryByGid(321).getStatus()); + } + + /** + * 旧节点不带 operationId 的两份连续上报都必须被接受： + * 第一份补建世代后，后续上报不能因为「缺 operationId」而被全部丢弃。 + */ + @Test + void legacyReportWithoutOperationIdKeepsAcceptingLaterReports() { + galleries.insertGallery(gallery(322, "legacy [322]", TaskStatus.DOWNLOADING)); + + TaskReportHandler.CommittedReport first = service.apply( + report("r-legacy-1", 1, Map.of(), task(322, TaskStatus.DOWNLOADING, 3))); + assertNotNull(first); + assertEquals(1, first.acceptedTasks().size(), "旧节点首份无 operationId 上报必须接受"); + assertNull(operation(322).getOperationId(), + "旧节点不带 operationId 时先保留 null，稳定 ID 只在首次下发时落库"); + assertEquals(TaskStatus.DOWNLOADING, galleries.selectGalleryByGid(322).getStatus()); + + TaskReportHandler.CommittedReport second = service.apply( + report("r-legacy-2", 2, Map.of(), task(322, TaskStatus.COMPRESSING, 8))); + assertNotNull(second); + assertEquals(1, second.acceptedTasks().size(), "同一旧世代的后续上报不得被丢弃"); + assertEquals(TaskStatus.COMPRESSING, galleries.selectGalleryByGid(322).getStatus()); + assertEquals(2L, operation(322).getLastRevision()); + } + + /** 新 v2 任务（已知 operationId）缺归属的上报仍必须被拒，不能污染当前任务。 */ + @Test + void knownOperationStillRejectsReportWithoutOperationId() { + service.createTask(323, "任务 [323]", "https://example.org/g/323/key/", "original", 7); + String operationId = operation(323).getOperationId(); + service.apply(report("r-known-1", 1, Map.of(323, operationId), task(323, TaskStatus.DOWNLOADING, 4))); + + TaskReportHandler.CommittedReport committed = service.apply( + report("r-known-2", 2, Map.of(), task(323, TaskStatus.COMPLETE, 10))); + + assertNotNull(committed); + assertTrue(committed.acceptedTasks().isEmpty(), "缺归属不得改已知世代任务"); + assertNotEquals(TaskStatus.COMPLETE, galleries.selectGalleryByGid(323).getStatus()); + assertEquals(1L, operation(323).getLastRevision(), "被拒上报不得推进 revision"); + } + + /** + * 迁移前历史任务的首份上报已采纳世代 ID 时，后续同 ID 上报继续被接受（不走 null 兼容分支）。 + */ + @Test + void legacyReportWithAdoptedIdStaysAccepted() { + galleries.insertGallery(gallery(324, "legacy [324]", TaskStatus.DOWNLOADING)); + service.apply(report("r324-1", 1, Map.of(324, "dl-legacy-324"), task(324, TaskStatus.DOWNLOADING, 2))); + assertEquals("dl-legacy-324", operation(324).getOperationId()); + + TaskReportHandler.CommittedReport second = service.apply( + report("r324-2", 2, Map.of(324, "dl-legacy-324"), task(324, TaskStatus.COMPRESSING, 7))); + + assertNotNull(second); + assertEquals(1, second.acceptedTasks().size(), "同 ID 的后续旧上报必须继续被接受"); + assertEquals(TaskStatus.COMPRESSING, galleries.selectGalleryByGid(324).getStatus()); + } + + /** 数据库失败：apply 返回 null（不可回 ACK），节点会保留快照重投。 */ + @Test + void databaseFailureOnReportReturnsNullSoNodeRetainsSnapshot() { + service.createTask(123, "任务 [123]", "https://example.org/g/123/key/", "original", 7); + String operationId = operation(123).getOperationId(); + TaskReportMessage invalid = report("r-bad", 70, Map.of(123, operationId), task(123, TaskStatus.DOWNLOADING, 2)); + + // 用真实触发器让回执写入失败，模拟提交阶段的数据库故障。 + try (Connection connection = new DriverManagerDataSource(dbUrl()).getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate("create trigger reject_receipt before update on task_operation" + + " when new.lastReportId='r-bad' begin select raise(ABORT,'simulated receipt failure'); end"); + } catch (Exception e) { + fail("准备失败条件时出错: " + e.getMessage()); + } + + assertNull(service.apply(invalid), "数据库失败不得返回可回执结果"); + assertEquals(0L, operation(123).getLastRevision(), "失败的回执不得留下半截 revision"); + assertNotEquals(TaskStatus.DOWNLOADING, galleries.selectGalleryByGid(123).getStatus(), + "整个上报事务必须回滚，任务状态不得被改成下载中"); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Service/UserIdentityIntegrationTest.java b/src/test/java/com/lion/lionwebsite/Service/UserIdentityIntegrationTest.java new file mode 100644 index 0000000..40e7773 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Service/UserIdentityIntegrationTest.java @@ -0,0 +1,52 @@ +package com.lion.lionwebsite.Service; + +import com.lion.lionwebsite.Dao.normal.*; +import com.lion.lionwebsite.Interceptor.TaskHandlerInterceptor; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mybatis.spring.SqlSessionFactoryBean; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.transaction.support.TransactionTemplate; + +import java.nio.file.Path; +import java.sql.Connection; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class UserIdentityIntegrationTest { + @TempDir Path directory; + + @Test + void renameAndBindingAreAtomicWhenBindingUpdateFails() throws Exception { + var source = new DriverManagerDataSource("jdbc:sqlite:" + directory.resolve("identity.db")); + try (Connection connection = source.getConnection(); var statement = connection.createStatement()) { + statement.executeUpdate("create table User(id integer primary key, AuthCode text, username text, lastAccessTime text, isEnable integer)"); + statement.executeUpdate("create table sub_bind(key text primary key, user text unique, subscription_account_id integer, user_id integer)"); + statement.executeUpdate("insert into User values(5,'example-code','alice',null,1)"); + statement.executeUpdate("insert into sub_bind values('example-key','alice',1,5)"); + statement.executeUpdate("create trigger reject_binding_rename before update on sub_bind when new.user='bob' begin select raise(ABORT,'simulated failure'); end"); + } + var bean = new SqlSessionFactoryBean(); + bean.setDataSource(source); + var factory = bean.getObject(); + factory.getConfiguration().addMapper(UserMapper.class); + factory.getConfiguration().addMapper(SubMapper.class); + var template = new SqlSessionTemplate(factory); + var service = new UserService(template.getMapper(UserMapper.class), mock(GalleryMapper.class), + mock(CollectMapper.class), mock(TaskHandlerInterceptor.class)); + service.subMapper = template.getMapper(SubMapper.class); + service.subscriptionState = new SubscriptionStateCoordinator(); + service.transactions = new TransactionTemplate(new DataSourceTransactionManager(source)); + String result = service.alterUsername("example-code", "bob"); + assertTrue(result.contains("failure")); + assertEquals("alice", template.getMapper(UserMapper.class).selectUserById(5).getUsername()); + try (Connection connection = source.getConnection(); var statement = connection.createStatement(); + var row = statement.executeQuery("select user,user_id from sub_bind")) { + assertTrue(row.next()); + assertEquals("alice", row.getString(1)); + assertEquals(5, row.getInt(2)); + } + } +} diff --git a/src/test/java/com/lion/lionwebsite/Service/UserServiceTest.java b/src/test/java/com/lion/lionwebsite/Service/UserServiceTest.java index b55bf79..302c2be 100644 --- a/src/test/java/com/lion/lionwebsite/Service/UserServiceTest.java +++ b/src/test/java/com/lion/lionwebsite/Service/UserServiceTest.java @@ -214,14 +214,14 @@ class UserServiceTest { assertTrue(json.contains("bob")); } - /** 空表也要返回成功，且 data 是「数组序列化后的字符串」（历史契约，前端按字符串解析）。 */ + /** 空表返回结构化空数组。 */ @Test void getAllUserHandlesEmptyTable() { when(userMapper.selectAllUser()).thenReturn(new User[0]); String json = service.getAllUser(); assertTrue(ok(json)); - assertTrue(json.contains("\"data\":\"[]\""), "实际输出: " + json); + assertTrue(json.contains("\"data\":[]"), "实际输出: " + json); } @Test @@ -239,4 +239,22 @@ class UserServiceTest { when(userMapper.selectUserByAuthCode("ghost")).thenReturn(null); assertThrows(NullPointerException.class, () -> service.getUserId("ghost")); } + + @Test + void enablingUserDoesNotTransferExistingDownloads() { + when(userMapper.isExist("code")).thenReturn(1); + when(userMapper.selectUserByAuthCode("code")).thenReturn(user(5, "code", "alice")); + assertTrue(ok(service.alterStatus("code", true))); + verify(userMapper).updateIsEnableById(5, true); + verify(galleryMapper, never()).updateGalleryDownloader(anyInt(), anyInt()); + } + + @Test + void renameKeepsSubscriptionBindingLinkedToUserId() { + service.subMapper = mock(com.lion.lionwebsite.Dao.normal.SubMapper.class); + when(userMapper.isExist("code")).thenReturn(1); + when(userMapper.selectUserByAuthCode("code")).thenReturn(user(5, "code", "alice")); + assertTrue(ok(service.alterUsername("code", "bob"))); + verify(service.subMapper).renameBoundUser(5, "alice", "bob"); + } } diff --git a/src/test/java/com/lion/lionwebsite/Util/ClashSubscriptionParserTest.java b/src/test/java/com/lion/lionwebsite/Util/ClashSubscriptionParserTest.java new file mode 100644 index 0000000..c52c2c4 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Util/ClashSubscriptionParserTest.java @@ -0,0 +1,25 @@ +package com.lion.lionwebsite.Util; + +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; +import java.util.*; +import static org.junit.jupiter.api.Assertions.*; + +class ClashSubscriptionParserTest { + @Test void inlineNodesAndReferencesAreFilteredRegardlessOfIndentation() { + String yaml = "proxies: [{name: 'Slow 1x', type: ss, server: a}, {name: 'Fast 3x', type: ss, server: b}]\n" + + "proxy-groups: [{name: Choice, type: select, proxies: ['Fast 3x', 'Slow 1x', DIRECT]}]\n" + + "rules: ['MATCH,Choice']\n"; + Map result = new Yaml().load(ClashSubscriptionParser.transform(yaml.lines().toList(), true, 2)); + var proxies = (List) result.get("proxies"); + assertEquals(1, proxies.size()); + assertEquals("Slow 1x", ((Map) proxies.getFirst()).get("name")); + var group = (Map) ((List)result.get("proxy-groups")).getFirst(); + assertEquals(List.of("Slow 1x", "DIRECT"), group.get("proxies")); + assertEquals(List.of("MATCH,Choice"), result.get("rules")); + } + @Test void malformedBodiesCannotReplaceLastGoodSubscription() { + assertThrows(IllegalArgumentException.class, () -> ClashSubscriptionParser.transform(List.of("error page"), false, 2)); + assertThrows(IllegalArgumentException.class, () -> ClashSubscriptionParser.transform(List.of("proxies: [{type: ss}]"), false, 2)); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Util/GalleryParsingTest.java b/src/test/java/com/lion/lionwebsite/Util/GalleryParsingTest.java index 1ca4700..563e644 100644 --- a/src/test/java/com/lion/lionwebsite/Util/GalleryParsingTest.java +++ b/src/test/java/com/lion/lionwebsite/Util/GalleryParsingTest.java @@ -75,7 +75,7 @@ class GalleryParsingTest { /** 仅解析（不下载）：应填好基本信息与可选分辨率，状态停在等待确认。 */ @Test void parseCollectsMetadataWithoutSubmitting() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { stubPages(ms); Gallery g = GalleryUtil.parse(URL, false, ""); @@ -98,7 +98,7 @@ class GalleryParsingTest { /** 下载且分辨率可用：应设定分辨率与体积，并把提交结果记为「已提交」。 */ @Test void parseSubmitsDownloadForAvailableResolution() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { stubPages(ms); Gallery g = GalleryUtil.parse(URL, true, "1280x"); @@ -112,7 +112,7 @@ class GalleryParsingTest { /** 请求的分辨率不在可选列表里必须抛业务异常，而不是提交一个无效任务。 */ @Test void parseThrowsWhenResolutionUnavailable() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { stubPages(ms); var thrown = assertThrows(ResolutionNotMatchException.class, @@ -128,7 +128,7 @@ class GalleryParsingTest { @Test void parseMarksFailureWhenSubmissionNotConfirmed() throws Exception { String failurePage = "

a

Error occurred

"; - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenAnswer(inv -> { String u = inv.getArgument(0); if (u.contains("/g/")) return galleryPage(); @@ -147,7 +147,7 @@ class GalleryParsingTest { /** 非法链接应直接返回 null，不产生任何网络请求。 */ @Test void parseReturnsNullForInvalidLink() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.verifyLink(anyString())).thenCallRealMethod(); ms.when(() -> GalleryUtil.parse(anyString(), anyBoolean(), any())).thenCallRealMethod(); @@ -164,7 +164,7 @@ class GalleryParsingTest { + "

Original 30.2 MiB

" + "" + "
"; - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenAnswer(inv -> { String u = inv.getArgument(0); if (u.contains("/g/")) return galleryPage(); @@ -206,7 +206,7 @@ class GalleryParsingTest { */ @Test void parseImageKeysAcceptsRealPageFormatWithTrailingSemicolon() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenReturn(mpvPage()); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); @@ -225,7 +225,7 @@ class GalleryParsingTest { @Test void parseImageKeysAcceptsVariantWithoutTrailingSemicolon() throws Exception { String noSemicolon = mpvPage().replace("\"k\":\"key3\"}];", "\"k\":\"key3\"}]"); - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenReturn(noSemicolon); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); @@ -238,7 +238,7 @@ class GalleryParsingTest { /** 页码必须从 1 开始逐页递增（前端按页码取图）。 */ @Test void parseImageKeysNumbersPagesFromOne() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenReturn(mpvPage()); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); @@ -252,7 +252,7 @@ class GalleryParsingTest { /** 空响应表示画廊已下架/被删，应返回 null 交由上层提示。 */ @Test void parseImageKeysReturnsNullForEmptyResponse() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenReturn(" "); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); @@ -263,7 +263,7 @@ class GalleryParsingTest { /** 页面缺少预期 script 标签时返回 null，而不是抛索引越界。 */ @Test void parseImageKeysReturnsNullWhenScriptsMissing() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())) .thenReturn(""); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); @@ -275,7 +275,7 @@ class GalleryParsingTest { /** 解析成功后 mpvKey 应进入缓存，后续 getMpvKey 可直接命中。 */ @Test void parseImageKeysCachesMpvKey() throws Exception { - try (var ms = mockStatic(GalleryUtil.class)) { + try (var ms = mockStatic(GalleryUtil.class, CALLS_REAL_METHODS)) { ms.when(() -> GalleryUtil.requests(anyString(), anyString(), any(), any())).thenReturn(mpvPage()); ms.when(() -> GalleryUtil.parseImageKeys(anyString())).thenCallRealMethod(); ms.when(() -> GalleryUtil.getMpvKey(anyString())).thenCallRealMethod(); diff --git a/src/test/java/com/lion/lionwebsite/Util/SubscriptionCacheStoreTest.java b/src/test/java/com/lion/lionwebsite/Util/SubscriptionCacheStoreTest.java new file mode 100644 index 0000000..a216210 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Util/SubscriptionCacheStoreTest.java @@ -0,0 +1,296 @@ +package com.lion.lionwebsite.Util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * 订阅缓存分代存储的核心契约：两份文件作为一个版本发布，失败时保留上次完整版本。 + */ +class SubscriptionCacheStoreTest { + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + /** 发布成功后，读取到的两份内容应来自同一分代。 */ + @Test + void publishExposesBothFilesFromOneGeneration(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + + store.publish(1, bytes("v2-new"), bytes("clash-new")); + + SubscriptionCacheStore.Pair pair = store.read(1).orElseThrow(); + assertEquals("v2-new", new String(pair.v2(), StandardCharsets.UTF_8)); + assertEquals("clash-new", new String(pair.clash(), StandardCharsets.UTF_8)); + assertEquals(pair.generation(), store.read(1).orElseThrow().generation()); + assertTrue(store.hasComplete(1)); + assertEquals(store.resolve(1, "v2").orElseThrow().getParent(), + store.resolve(1, "cat").orElseThrow().getParent(), "两份文件必须同目录"); + } + + /** 第二份文件写入失败时，指针不得前进，上一次的完整版本必须原样可读。 */ + @Test + void secondFileFailureKeepsPreviousPairIntact(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root) { + boolean firstPublished; + + @Override + protected void writeFile(Path target, byte[] data) throws java.io.IOException { + if (firstPublished && target.getFileName().toString().equals("clash.yaml")) + throw new java.io.IOException("disk full"); + super.writeFile(target, data); + } + + @Override + protected void publishManifest(Path accountDir, String generation, String v2Sha256, String clashSha256) + throws java.io.IOException { + super.publishManifest(accountDir, generation, v2Sha256, clashSha256); + firstPublished = true; + } + }; + store.publish(7, bytes("v2-old"), bytes("clash-old")); + + assertThrows(java.io.IOException.class, () -> store.publish(7, bytes("v2-new"), bytes("clash-new"))); + + SubscriptionCacheStore.Pair pair = store.read(7).orElseThrow(); + assertEquals("v2-old", new String(pair.v2(), StandardCharsets.UTF_8), + "第二份写入失败时不应出现新旧混用"); + assertEquals("clash-old", new String(pair.clash(), StandardCharsets.UTF_8)); + // 未发布的分代目录不应残留。 + try (var generations = Files.list(root.resolve("7/generations"))) { + assertEquals(1, generations.count(), "失败的分代应被清理"); + } + } + + /** 指针发布失败时同样不得改变当前版本。 */ + @Test + void manifestPublishFailureKeepsPreviousPairIntact(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(8, bytes("v2-old"), bytes("clash-old")); + // 用同一目录上的第二个实例覆写发布接缝，模拟指针写入失败。 + SubscriptionCacheStore failing = new SubscriptionCacheStore(root) { + @Override + protected void publishManifest(Path accountDir, String generation, String v2Sha256, String clashSha256) + throws java.io.IOException { + throw new java.io.IOException("manifest write failed"); + } + }; + + assertThrows(java.io.IOException.class, () -> failing.publish(8, bytes("v2-new"), bytes("clash-new"))); + + SubscriptionCacheStore.Pair pair = store.read(8).orElseThrow(); + assertEquals("v2-old", new String(pair.v2(), StandardCharsets.UTF_8), + "指针发布失败时不应切换版本"); + assertEquals("clash-old", new String(pair.clash(), StandardCharsets.UTF_8)); + } + + /** 旧结构：账号根目录下的两份文件都要在，才算可读的 legacy 版本。 */ + @Test + void legacyRootIsReadOnlyWhenBothFilesExist(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("9"); + Files.createDirectories(account); + Files.writeString(account.resolve("v2ray.txt"), "legacy-v2"); + assertTrue(store.read(9).isEmpty(), "只有一份旧文件时不应视为可用"); + assertFalse(store.hasComplete(9)); + + Files.writeString(account.resolve("clash.yaml"), "legacy-clash"); + SubscriptionCacheStore.Pair pair = store.read(9).orElseThrow(); + assertEquals("legacy-v2", new String(pair.v2(), StandardCharsets.UTF_8)); + assertEquals("legacy-clash", new String(pair.clash(), StandardCharsets.UTF_8)); + assertEquals("legacy", pair.generation()); + assertEquals(account.resolve("v2ray.txt"), store.resolve(9, "v2").orElseThrow()); + } + + /** 半份旧缓存不得被解析为可分发路径，否则分发端会把残缺内容当订阅发出。 */ + @Test + void incompleteLegacyIsNotResolvable(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("20"); + Files.createDirectories(account); + Files.writeString(account.resolve("v2ray.txt"), "only-v2"); + + assertTrue(store.resolve(20, "v2").isEmpty(), "只有一份旧文件时不得解析出 v2 路径"); + assertTrue(store.resolve(20, "cat").isEmpty(), "缺 clash 时不得解析出 cat 路径"); + assertFalse(store.hasComplete(20)); + } + + /** 分代里缺一份文件时，两份都必须判定为不可用。 */ + @Test + void generationMissingOneFileIsNotResolvable(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(21, bytes("v2"), bytes("clash")); + Files.delete(store.resolve(21, "cat").orElseThrow()); + + assertTrue(store.resolve(21, "v2").isEmpty()); + assertTrue(store.resolve(21, "cat").isEmpty()); + assertFalse(store.hasComplete(21)); + assertTrue(store.read(21).isEmpty()); + } + + /** 内容与指针哈希不符时，resolve 与 hasComplete 必须一起拒绝。 */ + @Test + void tamperedGenerationIsNotResolvable(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(22, bytes("v2"), bytes("clash")); + Files.writeString(store.resolve(22, "v2").orElseThrow(), "tampered"); + + assertTrue(store.resolve(22, "v2").isEmpty()); + assertTrue(store.resolve(22, "cat").isEmpty(), "同一版本里另一份也应一并拒绝"); + assertFalse(store.hasComplete(22)); + } + + /** 指针被写坏（缺哈希字段）时不得抛异常，也不得回退到半份旧缓存。 */ + @Test + void corruptedManifestNeverFallsBackToLegacy(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("23"); + Files.createDirectories(account); + // 故意只放一份旧文件，模拟「指针损坏 + 半份 legacy」的最坏组合。 + Files.writeString(account.resolve("v2ray.txt"), "legacy-only-v2"); + Files.writeString(account.resolve("current"), "generation=g1\n"); + + assertDoesNotThrow(() -> store.read(23)); + assertTrue(store.read(23).isEmpty(), "坏指针不应回退半份 legacy"); + assertTrue(store.resolve(23, "v2").isEmpty()); + assertFalse(store.hasComplete(23)); + } + + /** 指针指向的分代不存在时同样不得回退，且不应抛异常。 */ + @Test + void manifestPointingAtMissingGenerationIsNotResolvable(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("24"); + Files.createDirectories(account); + Files.writeString(account.resolve("v2ray.txt"), "legacy-v2"); + Files.writeString(account.resolve("clash.yaml"), "legacy-clash"); + Files.writeString(account.resolve("current"), + "generation=0000000000000-000001\nv2-sha256=deadbeef\nclash-sha256=deadbeef\n"); + + assertTrue(store.read(24).isEmpty(), "指针存在时不得回退 legacy"); + assertTrue(store.resolve(24, "v2").isEmpty()); + assertFalse(store.hasComplete(24)); + } + + /** + * 每个 API 调用只读一次指针：整份版本由同一次快照校验，不会出现 + * 「用新分代名配旧哈希」的撕裂窗口。 + */ + @Test + void everyLookupReadsTheManifestExactlyOnce(@TempDir Path root) throws Exception { + var reads = new int[1]; + SubscriptionCacheStore counted = new SubscriptionCacheStore(root) { + @Override + Properties readManifest(Path account) { + reads[0]++; + return super.readManifest(account); + } + }; + counted.publish(25, bytes("v2"), bytes("clash")); + + reads[0] = 0; + counted.read(25); + assertEquals(1, reads[0], "read 应只读一次指针"); + reads[0] = 0; + counted.resolve(25, "v2"); + assertEquals(1, reads[0], "resolve 应只读一次指针"); + reads[0] = 0; + counted.hasComplete(25); + assertEquals(1, reads[0], "hasComplete 应只读一次指针"); + } + + /** 哨兵路径必须保证不存在，且与真实的 legacy 路径不同。 */ + @Test + void unavailablePathIsNeverAnExistingFile(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("26"); + Files.createDirectories(account); + Files.writeString(account.resolve("v2ray.txt"), "legacy-v2"); + + Path sentinel = store.unavailablePath(26, "v2"); + assertFalse(Files.exists(sentinel)); + assertNotEquals(store.legacyPath(26, "v2"), sentinel); + assertTrue(store.resolve(26, "v2").isEmpty(), "半缓存不得解析成真实路径"); + } + + /** 新分代发布后取代旧结构，读取应切到分代内容。 */ + @Test + void publishedGenerationSupersedesLegacyFiles(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + Path account = root.resolve("10"); + Files.createDirectories(account); + Files.writeString(account.resolve("v2ray.txt"), "legacy-v2"); + Files.writeString(account.resolve("clash.yaml"), "legacy-clash"); + + store.publish(10, bytes("new-v2"), bytes("new-clash")); + + SubscriptionCacheStore.Pair pair = store.read(10).orElseThrow(); + assertEquals("new-v2", new String(pair.v2(), StandardCharsets.UTF_8)); + assertNotEquals("legacy", pair.generation()); + assertFalse(Files.exists(account.resolve("v2ray.txt")), "发布新分代后旧结构文件应被移除"); + } + + /** 失效后不应再读到任何版本，并允许后续重新发布。 */ + @Test + void invalidateRemovesAllVersionsAndAllowsRepublish(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(11, bytes("v2"), bytes("clash")); + + store.invalidate(11); + + assertTrue(store.read(11).isEmpty()); + assertFalse(store.hasComplete(11)); + assertTrue(store.resolve(11, "v2").isEmpty()); + + store.publish(11, bytes("v2-again"), bytes("clash-again")); + assertEquals("v2-again", new String(store.read(11).orElseThrow().v2(), StandardCharsets.UTF_8)); + } + + /** 分代目录只保留有限数量，避免长期运行撑爆磁盘。 */ + @Test + void oldGenerationsArePruned(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + for (int i = 0; i < 6; i++) + store.publish(12, bytes("v2-" + i), bytes("clash-" + i)); + + try (var generations = Files.list(root.resolve("12/generations"))) { + List retained = generations.toList(); + assertTrue(retained.size() <= 2, "分代数应被裁剪，实际 " + retained.size()); + assertTrue(retained.contains(store.resolve(12, "v2").orElseThrow().getParent()), + "当前分代必须保留"); + } + assertEquals("v2-5", new String(store.read(12).orElseThrow().v2(), StandardCharsets.UTF_8)); + } + + /** 指针缺失但分代目录存在时不应误读为当前版本。 */ + @Test + void missingManifestIsNotTreatedAsPublished(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(13, bytes("v2"), bytes("clash")); + Files.delete(root.resolve("13/current")); + + assertTrue(store.read(13).isEmpty(), "没有指针就没有已发布版本"); + assertFalse(store.hasComplete(13)); + } + + /** 内容被篡改（与指针记录的哈希不符）时应拒绝读取，避免把坏内容推给备机。 */ + @Test + void tamperedGenerationContentIsRejected(@TempDir Path root) throws Exception { + SubscriptionCacheStore store = new SubscriptionCacheStore(root); + store.publish(14, bytes("v2"), bytes("clash")); + Path published = store.resolve(14, "v2").orElseThrow(); + Files.writeString(published, "tampered"); + + Optional pair = store.read(14); + assertTrue(pair.isEmpty(), "校验失败的分代不应被读取"); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Util/UpstreamParserVariantsTest.java b/src/test/java/com/lion/lionwebsite/Util/UpstreamParserVariantsTest.java new file mode 100644 index 0000000..9f9ebe9 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Util/UpstreamParserVariantsTest.java @@ -0,0 +1,36 @@ +package com.lion.lionwebsite.Util; + +import org.jsoup.Jsoup; +import org.junit.jupiter.api.Test; +import java.io.IOException; +import static org.junit.jupiter.api.Assertions.*; + +class UpstreamParserVariantsTest { + @Test void metadataUsesLabelsInsteadOfRowPositions() throws Exception { + var document = Jsoup.parse("

Alternate title

" + + "" + + "" + + "
Length:12 pages
Extra:ignored
File Size:2 MiB
Language:Chinese
"); + var gallery = GalleryPageParser.metadata(document, "https://exhentai.org/g/12/key/", 12); + assertEquals(12, gallery.getPages()); + assertEquals("Chinese", gallery.getLanguage()); + assertEquals("Alternate title [12]", gallery.getName()); + assertEquals("/cover.jpg", gallery.getThumb_link()); + assertThrows(IOException.class, () -> GalleryPageParser.metadata(Jsoup.parse("

Error

"), "x", 1)); + } + + @Test void archiveEntrySupportsRelativeHrefAndMovedAnchor() throws Exception { + var document = Jsoup.parse("", "https://exhentai.org/g/12/key/"); + assertEquals("https://exhentai.org/archive/12", GalleryPageParser.archiveLink(document)); + assertThrows(IOException.class, () -> GalleryPageParser.archiveLink(Jsoup.parse(""))); + } + + @Test void arrayAssignmentHandlesMultilineAndClosingBracketsInsideStrings() throws Exception { + String script = "let unrelated=[];\nconst imagelist = [\n{\"k\":\"one]two\"},\n{\"k\":\"three\"}\n]; var after=1;"; + var array = CustomUtil.objectMapper.readTree(JavaScriptAssignments.array(script, "imagelist")); + assertEquals(2, array.size()); + assertEquals("one]two", array.get(0).get("k").asText()); + assertEquals("value", JavaScriptAssignments.string("let mpvkey = 'value';", "mpvkey")); + assertThrows(IOException.class, () -> JavaScriptAssignments.array("imagelist = [{\"k\":\"x\"}", "imagelist")); + } +}