Rewrite durable task lifecycle, gallery indexing and subscription publication

This commit is contained in:
root
2026-10-07 17:46:32 +08:00
parent 776eb8cf53
commit e6c7dab12e
72 changed files with 7291 additions and 551 deletions
@@ -27,6 +27,9 @@ import java.util.Random;
Chat.class, MessageEntity.class,
AbstractMethodError.class, DeleteGalleryMessage.class, DownloadPostMessage.class, DownloadStatusMessage.class,
IdentityMessage.class, MaintainMessage.class, ResponseMessage.class, AvailableCheckMessage.class,
// 协议 v2 的消息与任务操作实体:原生镜像下 Jackson/MyBatis 都要反射构造它们。
ProtocolHelloMessage.class, TaskCommandMessage.class, TaskReportMessage.class, TaskReportAckMessage.class,
TaskOperation.class,
SubscriptionSnapshotMessage.class, SubscriptionSnapshotPayload.class, SubscriptionAccountSnapshot.class,
SubscriptionBindingSnapshot.class, LinkPreviewOptions.class})
public class CustomBean {
@@ -0,0 +1,83 @@
package com.lion.lionwebsite.Configuration;
import com.lion.lionwebsite.Dao.cache.ImageCacheMapper;
import com.lion.lionwebsite.Domain.ImageKeyCache;
import com.lion.lionwebsite.Service.GalleryIndexService;
import com.lion.lionwebsite.Util.GalleryUtil;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import javax.sql.DataSource;
import java.util.List;
/**
* 图片索引服务的事务装配。
*
* <p>单独成文件是为了不触碰既有的 SqlConfiguration:那里的两个 SqlSessionFactory
* 保持原样,这里只额外声明缓存库的事务管理器与模板。
*
* <p>事务管理器必须绑定 datasource-cache —— 索引只写缓存库,
* 若误用主库事务管理器,整组替换就失去原子性(删除与写入落在不同连接上)。
*/
@Configuration
public class GalleryIndexConfiguration {
@Bean
public PlatformTransactionManager cacheTransactionManager(
@Qualifier("datasource-cache") DataSource cacheDataSource) {
return new DataSourceTransactionManager(cacheDataSource);
}
/** 只用于图片索引的短事务发布,超时留给上层请求超时控制,这里显式设为 30 秒。 */
@Bean
public TransactionTemplate cacheTransactionTemplate(
@Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) {
TransactionTemplate template = new TransactionTemplate(cacheTransactionManager);
template.setTimeout(30);
return template;
}
/**
* 只读快照事务,用于一次性读取头部与页码。
*
* <p>读事务不写库,因此只读连接可以与写入并发;它的作用是让「读头部 + 读页码」
* 落在同一个快照上,避免恰好读到整组替换的中间状态。
*/
@Bean
public TransactionTemplate cacheReadTransactionTemplate(
@Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) {
TransactionTemplate template = new TransactionTemplate(cacheTransactionManager);
template.setTimeout(10);
return template;
}
/**
* 这里显式不设置 {@code readOnly}:sqlite-jdbc 不允许在已建立的连接上改只读标志
* (实测抛 SQLException: Cannot change read-only flag after establishing a connection),
* 而事务模板拿到的是连接池里的既有连接,设置只读会让每次读取都直接失败。
* 快照一致性由「同一个事务」保证,不需要额外的只读标志。
*/
@Bean
public GalleryIndexService galleryIndexService(
ImageCacheMapper imageCacheMapper,
@Qualifier("cacheTransactionTemplate") TransactionTemplate cacheTransactionTemplate,
@Qualifier("cacheReadTransactionTemplate") TransactionTemplate cacheReadTransactionTemplate) {
return new GalleryIndexService(imageCacheMapper, cacheTransactionTemplate,
cacheReadTransactionTemplate,
GalleryIndexConfiguration::fetchPageKeys);
}
/**
* 生产环境的上游实现:沿用 GalleryUtil.parseImageKeys(HTTP + JS 解析)。
*
* <p>放在这里而不是服务内部,是为了让服务在测试中可以脱离静态工具类。
*/
static List<ImageKeyCache> fetchPageKeys(String url) throws Exception {
return GalleryUtil.parseImageKeys(url);
}
}
@@ -8,6 +8,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import com.zaxxer.hikari.HikariDataSource;
import javax.sql.DataSource;
@@ -16,7 +21,9 @@ public class SqlConfiguration {
@Bean(name = "datasource-main")
@ConfigurationProperties(prefix = "spring.datasource-main")
public DataSource dataSourceMain() {
return DataSourceBuilder.create().build();
DataSource dataSource = DataSourceBuilder.create().build();
applyBusyTimeout(dataSource);
return dataSource;
}
@Bean(name = "datasource-cache")
@@ -25,6 +32,18 @@ public class SqlConfiguration {
return DataSourceBuilder.create().build();
}
/**
* 给主库连接打开 busy 等待。
*
* <p>主库是单文件 SQLite,任务生命周期会并发开启短事务(创建、状态回执、删除)。
* 默认没有 busy 超时,第二个写事务会立刻拿到 SQLITE_BUSY 并被当作 500;
* 打开等待后,写冲突会按序排队重试而不是失败。
*/
private void applyBusyTimeout(DataSource dataSource) {
if (dataSource instanceof HikariDataSource hikari)
hikari.setConnectionInitSql("PRAGMA busy_timeout=5000");
}
@Bean(name = "sqlSessionFactory-main")
public SqlSessionFactory sqlSessionFactoryMain(@Qualifier("datasource-main") DataSource dataSource) throws Exception {
SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
@@ -48,4 +67,30 @@ public class SqlConfiguration {
public SqlSessionTemplate sqlSessionTemplateCache(@Qualifier("sqlSessionFactory-cache") SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory);
}
/**
* 主库事务管理器。
*
* <p>必须显式绑定到 datasource-main:应用里有两个 DataSource,Spring Boot 的自动配置
* 会退避,若没有这个 bean 就没有任何事务管理器,@Transactional 会静默失效。
* @Primary 同时让「不指定名字就注入 TransactionManager」的旧代码继续可用。
*/
@Bean(name = "mainTransactionManager")
@Primary
public DataSourceTransactionManager mainTransactionManager(@Qualifier("datasource-main") DataSource dataSource) {
return new DataSourceTransactionManager(dataSource);
}
/**
* 主库编程式事务模板。
*
* <p>任务生命周期需要把「短事务」放在网络调用之外,并且要求「回调里捕获异常后不再抛出」
* 不能错误提交,因此调用方必须用 {@code execute} 的返回状态或再次抛出决定回滚,
* 而不是在回调里吞掉异常(见 GalleryTaskLifecycleService)。
*/
@Bean(name = "mainTransactionTemplate")
@Primary
public TransactionTemplate mainTransactionTemplate(@Qualifier("mainTransactionManager") DataSourceTransactionManager transactionManager) {
return new TransactionTemplate(transactionManager);
}
}
@@ -0,0 +1,57 @@
package com.lion.lionwebsite.Configuration;
import com.lion.lionwebsite.Domain.TaskOperation;
import com.lion.lionwebsite.Service.GalleryTaskLifecycleService;
import com.lion.lionwebsite.Service.RemoteService;
import com.lion.lionwebsite.Service.TaskReportHandler;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
/**
* 装配协议 v2 的双向依赖,并做启动恢复。
*
* <p>RemoteService 消费上报时需要生命周期服务,而生命周期服务下发命令时需要 RemoteService。
* 直接互相注入构造参数会形成环,所以这里在两端都构造完成之后用 setter 接上;
* 同时也把「启动时把中断的 SUBMITTING 判为 UNKNOWN、释放 RESERVED 占位」这一步接到应用就绪事件上。
*
* <p>顺序是不可交换的:
* <ol>
* <li>先绑定上报处理器、下发判定解析器与 operationId 解析器;</li>
* <li>再执行启动恢复(SUBMITTING → UNKNOWN、释放 RESERVED);</li>
* <li>最后才允许建立连接与自动重投。连接由本类显式触发,不再由 RemoteService 的
* PostConstruct 在装配前发起,否则会把状态不明的任务盲目重投给节点。</li>
* </ol>
* 恢复失败时仍然允许连接(保持节点在线/心跳),但明确禁止自动派发。
*/
@Component
@Slf4j
@RequiredArgsConstructor
public class TaskLifecycleWiring {
final RemoteService remoteService;
final GalleryTaskLifecycleService lifecycleService;
@EventListener(ApplicationReadyEvent.class)
public void wire() {
TaskReportHandler handler = lifecycleService;
remoteService.setTaskReportHandler(handler);
remoteService.setDownloadOperationIdResolver(lifecycleService::currentOperationId);
remoteService.setDispatchResolver(lifecycleService::resolveDispatch);
boolean recoverySafe = false;
try {
int recovered = lifecycleService.recoverInterruptedSubmissions();
recoverySafe = true;
if (recovered > 0)
log.warn("启动恢复:{} 个中断的创建/预占任务已处理(提交中的标为提交未知)", recovered);
} catch (RuntimeException e) {
// 恢复失败说明库中任务真实状态未知:可以保持连接,但绝不能自动重投/派发。
log.error("启动恢复失败,本次启动将拒绝自动重投未完成任务", e);
}
// 绑定与恢复完成之后才连接:连接成功后的自动重投受 lifecycleReady/recoverySafe 双重约束。
remoteService.startAfterWiring(recoverySafe);
}
}
@@ -62,16 +62,14 @@ public class PublicController {
public String validate(String AuthCode){
Response response = Response.generateResponse();
User user = publicService.getUserId(AuthCode);
// 用 ObjectMapper 组装内层 JSON:手工 String.format 拼用户名时,
// 名字里带引号或反斜杠会直接产出非法 JSON,前端 JSON.parse 随即失败。
// 这里仍以「JSON 文本」形式放进 data(历史契约,前端按字符串再解析一次)。
// 身份字段直接作为结构化 data 返回;共享前端解码器兼容旧版本字符串。
ObjectNode identity = CustomUtil.objectMapper.createObjectNode();
identity.put("userId", user.getId());
identity.put("username", user.getUsername());
identity.put("isAvailable", !remoteService.isDead());
// 管理员标记随登录一起下发,前端据此决定是否显示下载人信息与筛选。
identity.put("isAdmin", user.getId() == UserService.ADMIN_USER_ID);
response.success(identity.toString());
response.success(identity);
return response.toJSONString();
}
@@ -8,6 +8,8 @@ import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper
public interface ImageCacheMapper {
@Insert("insert into ImageKeyCache values (#{gid}, #{imgkey}, #{page})")
@@ -31,4 +33,32 @@ public interface ImageCacheMapper {
@Delete("delete from gidToKey where gid=#{gid}")
void deleteGidToKey(String gid);
/**
* 某个 gid 已落库的页码列表(升序)。
*
* <p>只按行数判断完整性会把「页码 1 和 3、共 2 行」当成完整索引,
* 因此完整性判定改为按页码集合比对——先取回页码,再校验它是否恰好是 1..pages。
*/
@Select("select page from ImageKeyCache where gid=#{gid} order by page")
List<Integer> selectPageNumbersByGid(String gid);
/** 该 gid 下 imgkey 为空的脏行数;任何一行脏行都说明索引不可用。 */
@Select("select count(*) from ImageKeyCache where gid=#{gid} and (imgkey is null or trim(imgkey)='')")
int countInvalidImageKeysByGid(String gid);
/**
* 批量写入页 key(单条多值 INSERT)。
*
* <p>整本画廊的索引可能上千页,逐行 INSERT 会拉长写事务并放大锁竞争;
* 调用方按固定大小分片后,一个写事务里只需少量语句即可写完。
*/
// MyBatis 只在注解值「以 <script> 开头」时才按 XML 动态 SQL 解析,因此这里不能有前导空白。
@Insert("<script>"
+ "insert into ImageKeyCache (gid, imgkey, page) values "
+ "<foreach collection='pages' item='p' separator=','>"
+ "(#{p.gid}, #{p.imgkey}, #{p.page})"
+ "</foreach>"
+ "</script>")
int insertImageKeyCacheBatch(@Param("pages") List<ImageKeyCache> pages);
}
@@ -20,4 +20,8 @@ public interface CollectMapper {
@Select("select collector from collect where gid=#{gid}")
ArrayList<Integer> selectCollectorByGid(int gid);
/** 任务删除完成后清理该 gid 的全部收藏记录(与删任务行同一事务)。 */
@Delete("delete from collect where gid=#{gid}")
void deleteByGid(@Param("gid") int gid);
}
@@ -36,9 +36,10 @@ public interface GalleryMapper {
@Update("""
update gallery set name=#{name}, link=#{link}, language=#{language},
update gallery set name=#{name}, link=#{link}, language=#{language}, downloader=#{downloader},
pages=#{pages}, status=#{status}, fileSize=#{fileSize}, createTime=#{createTime},
proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize}, is_download=#{is_download}
proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize},
thumb_link=#{thumb_link}, is_download=#{is_download}
where gid=#{gid}""")
void updateGallery(Gallery gallery);
@@ -47,4 +48,8 @@ public interface GalleryMapper {
@Delete("delete from gallery where gid=#{gid}")
void deleteGalleryByGid(Integer gid);
/** 只改状态列,用于「待删除」这类不牵动其他字段的意图标记。 */
@Update("update gallery set status=#{status} where gid=#{gid}")
int updateGalleryStatus(@Param("gid") int gid, @Param("status") String status);
}
@@ -43,49 +43,57 @@ public interface SubMapper {
@Delete("delete from subscription_account where id=#{id}")
void deleteSubscriptionAccount(Integer id);
@Insert("insert into sub_bind (key, user, subscription_account_id) values (#{key}, #{user}, #{subscriptionAccountId})")
@Insert("insert into sub_bind (key, user, subscription_account_id, user_id) values (#{key}, #{user}, #{subscriptionAccountId}, (select id from User where username=#{user}))")
void insertSubBind(SubBind subBind);
@Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user")
@Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user")
ArrayList<SubBind> selectAllSubBind();
@Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}")
@Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}")
SubBind selectSubBind(String key);
@Select("select count(key) from sub_bind where key=#{key}")
boolean selectSubBindExist(String key);
@Select("select count(*) from sub_bind where user=#{user}")
@Select("select count(*) from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
int countSubBindByUser(String user);
@Update("update sub_bind set subscription_account_id=#{accountId} where user=#{user}")
@Update("update sub_bind set subscription_account_id=#{accountId} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
int updateSubBindAccount(@Param("user") String user, @Param("accountId") Integer accountId);
@Update("update sub_bind set key=#{key} where user=#{user}")
@Update("update sub_bind set key=#{key} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
int updateSubBindKey(@Param("user") String user, @Param("key") String key);
@Delete("delete from sub_bind where user=#{user}")
@Delete("delete from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
void deleteSubBind(String user);
@Insert("insert into sub_update_record (user, ip, UA, time, location) values (#{user}, #{ip}, #{UA}, #{time}, #{location})")
@Insert("insert into sub_update_record (user, ip, UA, time, location, user_id) values (#{user}, #{ip}, #{UA}, #{time}, #{location}, coalesce(#{userId}, (select id from User where username=#{user})))")
@Options(useGeneratedKeys = true, keyProperty = "id")
void insertSubUpdateRecord(SubUpdateRecord subUpdateRecord);
@Select("select * from sub_update_record order by time desc")
@Select("select id,user,ip,UA,time,location,user_id as userId from sub_update_record order by time desc")
ArrayList<SubUpdateRecord> selectAllSubUpdateRecord();
@Delete("delete from sub_update_record where user=#{user}")
@Delete("delete from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
void deleteSubUpdateRecord(String user);
@Select("select count(user) from sub_update_record where user=#{user}")
@Select("select count(user) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
Integer selectUpdateRecordCount(String user);
@Select("select min(id) from sub_update_record where user=#{user}")
@Select("select min(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
Integer selectMinUpdateRecordId(String user);
/** 后台补齐归属地:只更新该用户最近一条记录,避免覆盖更早的历史记录。 */
@Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user=#{user})")
@Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user}))")
void updateLatestSubUpdateRecordLocation(@Param("user") String user, @Param("location") String location);
@Update("update sub_update_record set location=#{location} where id=#{id}")
void updateSubUpdateRecordLocation(@Param("id") int id, @Param("location") String location);
@Update("update sub_bind set user=#{newName} where user_id=#{userId} or (user_id is null and user=#{oldName})")
void renameBoundUser(@Param("userId") int userId, @Param("oldName") String oldName,
@Param("newName") String newName);
@Delete("delete from sub_update_record where id=#{id}")
void deleteSubUpdateRecordById(int id);
}
@@ -0,0 +1,107 @@
package com.lion.lionwebsite.Dao.normal;
import com.lion.lionwebsite.Domain.TaskOperation;
import org.apache.ibatis.annotations.*;
/**
* 任务操作记录的持久化入口(主库)。
*
* <p>所有写操作都在调用方的事务里执行;这里的注解只描述 SQL,不包含业务判断。
* 主键为 gid,保证「同一 gid 只能有一条有效操作意图」。
*/
@Mapper
public interface TaskOperationRepository {
@Select("select * from task_operation where gid=#{gid}")
TaskOperation selectByGid(int gid);
/** 生成该 gid 的稳定下载 operationId:只有还没有记录时才写入。 */
@Insert("insert or ignore into task_operation (gid, operationId, deleteOperationId, state, lastRevision," +
" lastReportId, updateTime) values (#{gid}, #{operationId}, null, #{state}, 0, null, #{updateTime})")
int insertIfAbsent(@Param("gid") int gid, @Param("operationId") String operationId,
@Param("state") String state, @Param("updateTime") long updateTime);
@Insert("insert into task_operation (gid, operationId, deleteOperationId, state, lastRevision, lastReportId, updateTime)" +
" values (#{gid}, #{operationId}, #{deleteOperationId}, #{state}, #{lastRevision}, #{lastReportId}, #{updateTime})")
void insert(TaskOperation operation);
/** 推进创建意图的状态(RESERVED → SUBMITTING → SUBMITTED/UNKNOWN)。 */
@Update("update task_operation set state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
int updateState(@Param("gid") int gid, @Param("state") String state, @Param("updateTime") long updateTime);
/**
* 带前置条件的创建状态推进:只有当前状态与 operationId 都匹配时才写入。
*
* <p>这是「提交前先确认这一代意图仍然有效」的 CAS:若期间被删除/重建,
* 返回值 0,调用方必须停下,绝不能再产生上游副作用。
*/
@Update("update task_operation set state=#{state}, updateTime=#{updateTime}" +
" where gid=#{gid} and operationId=#{operationId} and state=#{expectedState}")
int compareAndSetCreateState(@Param("gid") int gid, @Param("operationId") String operationId,
@Param("expectedState") String expectedState, @Param("state") String state,
@Param("updateTime") long updateTime);
/**
* 只按当前删除意图 ID 匹配的删除完成:期间若已被删除/重建,则不改动新世代记录。
*/
@Update("update task_operation set deleteOperationId=null, state=#{state}, updateTime=#{updateTime}" +
" where gid=#{gid} and deleteOperationId=#{deleteOperationId}")
int completeDeleteIfSameGeneration(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId,
@Param("state") String state, @Param("updateTime") long updateTime);
/**
* 删除「本次创建意图」的预占记录:必须同时匹配 gid 与 operationId,且状态仍是
* RESERVED/SUBMITTING。
*
* <p>三重匹配保证释放流程只能清掉自己那一代:期间被删除/重建(新 operationId 或墓碑)
* 的记录一律不动,避免把新世代抹掉。
*/
@Delete("delete from task_operation where gid=#{gid} and operationId=#{operationId}" +
" and state in ('RESERVED','SUBMITTING')")
int deleteReservationIfSameGeneration(@Param("gid") int gid, @Param("operationId") String operationId);
/** 记录/替换当前下载 operationId 并推进状态。 */
@Update("update task_operation set operationId=#{operationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
int updateOperationId(@Param("gid") int gid, @Param("operationId") String operationId,
@Param("state") String state, @Param("updateTime") long updateTime);
/**
* 给「有记录但缺 operationId」的历史任务补一个稳定 operationId。
* 只在确实缺失时写入,避免覆盖并发创建出来的新世代。
*/
@Update("update task_operation set operationId=#{operationId}, updateTime=#{updateTime}" +
" where gid=#{gid} and operationId is null")
int assignOperationIdIfMissing(@Param("gid") int gid, @Param("operationId") String operationId,
@Param("updateTime") long updateTime);
/**
* 墓碑复活:删除完成后的 gid 允许用全新 operationId 重新创建。
* revision 与回执一并清零,旧操作的迟到上报会因 operationId 不匹配被丢弃。
*/
@Update("update task_operation set operationId=#{operationId}, deleteOperationId=null, state=#{state}," +
" lastRevision=0, lastReportId=null, updateTime=#{updateTime} where gid=#{gid}")
int reactivate(@Param("gid") int gid, @Param("operationId") String operationId,
@Param("state") String state, @Param("updateTime") long updateTime);
/** 记录删除意图与其稳定的删除操作 ID(重试必须复用同一个 ID)。 */
@Update("update task_operation set deleteOperationId=#{deleteOperationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
int updateDeleteIntent(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId,
@Param("state") String state, @Param("updateTime") long updateTime);
/**
* 写入状态上报回执(revision + reportId)。
*
* <p>回执与任务行更新在同一个事务里提交,提交成功之后才回 ACK;
* 因此节点重复投递同一份上报时,只能看到已经提交过的回执,不会重复落库或重复通知。
*/
@Update("update task_operation set lastRevision=#{revision}, lastReportId=#{reportId}, updateTime=#{updateTime}" +
" where gid=#{gid}")
int updateReportReceipt(@Param("gid") int gid, @Param("revision") long revision,
@Param("reportId") String reportId, @Param("updateTime") long updateTime);
@Select("select * from task_operation where state=#{state}")
java.util.List<TaskOperation> selectByState(String state);
@Delete("delete from task_operation where gid=#{gid}")
int deleteByGid(int gid);
}
@@ -14,4 +14,10 @@ public class SubBind {
String subscriptionAccountName;
boolean subscriptionAccountEnabled;
boolean filterHighMultiplier;
Integer userId;
public SubBind(String key, String user, Integer accountId, String accountName,
boolean accountEnabled, boolean filterHighMultiplier) {
this(key, user, accountId, accountName, accountEnabled, filterHighMultiplier, null);
}
}
@@ -16,4 +16,9 @@ public class SubUpdateRecord {
String UA;
Date time;
String location;
Integer userId;
public SubUpdateRecord(int id, String user, String ip, String UA, Date time, String location) {
this(id, user, ip, UA, time, location, null);
}
}
@@ -0,0 +1,53 @@
package com.lion.lionwebsite.Domain;
import lombok.Data;
/**
* 单个 gid 的下载生命周期操作记录(协议 v2 的持久化意图)。
*
* <p>这张表是「先持久化意图、再产生外部副作用」的落点:
* <ul>
* <li>CREATE 意图先于上游 POST 落库,用来保证同一 gid 只有一个未完成意图;</li>
* <li>提交结果不确定时只能停在 {@link #STATE_UNKNOWN},绝不自动重复上游 POST;</li>
* <li>DELETE 意图先于节点删除落库,节点确认后(或文件已不存在)才在同一事务里删任务行并留墓碑。</li>
* </ul>
*
* <p>旧任务行没有对应操作记录,查询时按 gid 惰性补建(见 GalleryTaskLifecycleService)。
*/
@Data
public class TaskOperation {
/** 已登记创建意图,尚未向上游提交。 */
public static final String STATE_RESERVED = "RESERVED";
/** 正在向上游提交;此时不可再发第二次 POST。 */
public static final String STATE_SUBMITTING = "SUBMITTING";
/** 上游确认已接受下载请求。 */
public static final String STATE_SUBMITTED = "SUBMITTED";
/**
* 提交结果未知:超时、断连或在 SUBMITTING 期间重启。
* 上游不支持幂等,因此这里只能等对账(reconcile),不能自动重发。
*/
public static final String STATE_UNKNOWN = "UNKNOWN";
/** 已登记删除意图,等待节点确认。 */
public static final String STATE_DELETE_PENDING = "DELETE_PENDING";
/** 节点已确认删除,任务行也随之删除;本行作为墓碑保留,用于停止迟到状态的重放。 */
public static final String STATE_DELETED = "DELETED";
private int gid;
/** 当前有效的下载操作 ID;同一 gid 的 DOWNLOAD 重试必须复用它。 */
private String operationId;
/** 独立于下载操作 ID 的删除操作 ID,删除重试复用它。 */
private String deleteOperationId;
private String state;
/** 最近一次应用的状态上报 revision;用于丢弃乱序/重放的旧上报。 */
private long lastRevision;
/** 节点已确认收到的最新上报 ID;用于重复 ACK 的幂等判断。 */
private String lastReportId;
private long updateTime;
}
@@ -0,0 +1,44 @@
package com.lion.lionwebsite.Exception;
/**
* 图片索引构建失败。
*
* <p>调用方需要区分三类结果,才能给出正确的用户可见响应:
* 链接本身有问题({@link Reason#INVALID_LINK},属于请求错误)、
* 画廊已不存在({@link Reason#UNAVAILABLE},重试无意义)、
* 上游或数据库暂时不可用({@link Reason#TRANSIENT},可以重试)。
*/
public class GalleryIndexException extends Exception {
public enum Reason {
/** 链接无法解析出 gid 或 key。 */
INVALID_LINK,
/** 上游明确表示画廊不存在(下架/删除/解析不到页 key)。 */
UNAVAILABLE,
/** 上游返回的页 key 无法自洽(gid 不匹配、页码重复/跳号、imgkey 为空)。 */
MALFORMED,
/** 上游波动或索引写入失败,重试可能成功。 */
TRANSIENT
}
private final Reason reason;
public GalleryIndexException(Reason reason, String message) {
super(message);
this.reason = reason;
}
public GalleryIndexException(Reason reason, String message, Throwable cause) {
super(message, cause);
this.reason = reason;
}
public Reason getReason() {
return reason;
}
/** 重试没有意义(链接错误或画廊不可用);上游数据畸形与会话级故障仍可重试。 */
public boolean isPermanent() {
return reason == Reason.INVALID_LINK || reason == Reason.UNAVAILABLE;
}
}
@@ -0,0 +1,319 @@
package com.lion.lionwebsite.Maintenance;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 缓存库图片索引的离线体检与修复工具。
*
* <p>它是独立可执行类(只用 JDBC,不依赖 Spring / MyBatis),因此可以在服务停止后
* 直接对数据库文件运行,也可以先对生产库的副本演练。</p>
*
* <pre>
* java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db
* java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db --apply
* </pre>
*
* <p>它只做两类写入:删除不完整或有冲突的 gid 整组索引(gidToKey 头部加该 gid 的全部
* 页 key),以及删除没有头部记录的孤儿页 key。被删掉的 gid 由下一次在线阅读重新抓取,
* 因此不需要猜测保留哪一组 key。工具不建表、不改列、不碰其它业务表。</p>
*/
public final class GalleryIndexMaintenance {
/** 生产运行时数据库所在目录;默认拒绝直接写入,必须显式加 --allow-production。 */
private static final String PRODUCTION_DIR = "/home/lionwebsite";
private static final String PLAIN_REPAIR_NOTE =
"这些画廊将在下次被访问时重新抓取。";
public static void main(String[] args) {
Map<String, String> options = parseOptions(args);
if (options.containsKey("help") || !options.containsKey("db")) {
printUsage();
System.exit(options.containsKey("help") ? 0 : 2);
return;
}
Path database = Path.of(options.get("db")).toAbsolutePath().normalize();
boolean apply = options.containsKey("apply");
if (!Files.isRegularFile(database)) {
System.err.println("数据库不存在: " + database);
System.exit(2);
return;
}
if (apply && isProductionPath(database) && !options.containsKey("allow-production")) {
System.err.println("拒绝写入生产库路径 " + database
+ "。请先在副本上演练;确需在本机生产库执行时显式加 --allow-production,"
+ "并确保服务已停止。");
System.exit(3);
return;
}
try (Connection connection = open(database)) {
Report before = inspect(connection);
System.out.println(before.render(apply ? "修复前" : "体检结果"));
if (!apply) {
System.out.println();
System.out.println("dry-run:未做任何写入。确认无误后加 --apply 执行修复。");
System.exit(before.isHealthy() ? 0 : 1);
return;
}
if (before.isHealthy()) {
System.out.println();
System.out.println("索引已健康,无需修复。");
return;
}
connection.setAutoCommit(false);
int removed = repair(connection, before.badGids, before.orphanGids);
connection.commit();
System.out.println();
System.out.println("已删除 " + removed + " 个 gid 的索引整组," + PLAIN_REPAIR_NOTE);
Report after = inspect(connection);
System.out.println();
System.out.println(after.render("修复后"));
if (!after.isHealthy()) {
System.err.println("修复后仍未达到健康状态,请人工检查。");
System.exit(4);
}
} catch (SQLException e) {
System.err.println("执行失败: " + e.getMessage());
e.printStackTrace(System.err);
System.exit(5);
}
}
private static Connection open(Path database) throws SQLException {
Connection connection = DriverManager.getConnection("jdbc:sqlite:" + database);
try (Statement statement = connection.createStatement()) {
statement.execute("pragma busy_timeout=5000");
statement.execute("pragma foreign_keys=off");
}
return connection;
}
/** 是否位于生产运行时目录;工具据此拒绝在未显式确认时写入生产库。 */
static boolean isProductionPath(Path database) {
return database.startsWith(Path.of(PRODUCTION_DIR).toAbsolutePath().normalize());
}
// ---------- 体检 ----------
static Report inspect(Connection connection) throws SQLException {
Report report = new Report();
report.hasHeaderTable = tableExists(connection, "gidToKey");
report.hasPageTable = tableExists(connection, "ImageKeyCache");
if (!report.hasHeaderTable || !report.hasPageTable)
return report;
report.headerRows = count(connection, "select count(*) from gidToKey");
report.pageRows = count(connection, "select count(*) from ImageKeyCache");
report.duplicateHeaderGids = count(connection,
"select count(*) from (select gid from gidToKey group by gid having count(*) > 1)");
report.duplicatePageGroups = count(connection,
"select count(*) from (select gid, page from ImageKeyCache group by gid, page having count(*) > 1)");
try (PreparedStatement statement = connection.prepareStatement(BAD_GID_QUERY);
ResultSet rows = statement.executeQuery()) {
while (rows.next()) {
String gid = rows.getString(1);
report.badGids.add(gid);
report.badReasons.put(gid, describe(rows));
}
}
try (PreparedStatement statement = connection.prepareStatement(
"select distinct gid from ImageKeyCache where gid not in (select gid from gidToKey)");
ResultSet rows = statement.executeQuery()) {
while (rows.next())
report.orphanGids.add(rows.getString(1));
}
return report;
}
/**
* 一个 gid 触发的全部不健康条件集中在一句里检查,避免多次全表扫描。
*
* <p>必须按 gid 分组后再判断:重复 gid(同一 gid 有多条头部记录)本身就是要清除的对象,
* 而重复行里的某一条可能恰好与页 key 数一致,逐行判断会把它当成健康记录漏掉。
*/
private static final String BAD_GID_QUERY = """
select g.gid as gid,
count(*) as header_rows,
max(g.pages) as pages,
min(g.key) as key,
(select count(*) from ImageKeyCache i where i.gid = g.gid) as page_rows,
(select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) as distinct_pages,
(select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as min_page,
(select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as max_page,
(select count(*) from ImageKeyCache i where i.gid = g.gid
and (i.imgkey is null or trim(i.imgkey) = '')) as dirty_keys
from gidToKey g
group by g.gid
having count(*) > 1
or max(g.pages) is null or max(g.pages) <= 0
or min(g.key) is null or trim(min(g.key)) = ''
or (select count(*) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
or (select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
or (select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> 1
or (select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
or (select count(*) from ImageKeyCache i where i.gid = g.gid
and (i.imgkey is null or trim(i.imgkey) = '')) > 0
order by g.gid
""";
private static String describe(ResultSet row) throws SQLException {
List<String> reasons = new ArrayList<>();
int headerRows = row.getInt("header_rows");
int pages = row.getInt("pages");
String key = row.getString("key");
int pageRows = row.getInt("page_rows");
int distinctPages = row.getInt("distinct_pages");
int minPage = row.getInt("min_page");
int maxPage = row.getInt("max_page");
int dirtyKeys = row.getInt("dirty_keys");
if (headerRows > 1) reasons.add("重复头部记录 " + headerRows + " 条");
if (key == null || key.isBlank()) reasons.add("key 为空");
if (pages <= 0) reasons.add("pages 非法(" + pages + ")");
if (pageRows != pages) reasons.add("页 key 行数 " + pageRows + " 与 pages " + pages + " 不符");
if (distinctPages != pages) reasons.add("页码去重后 " + distinctPages + " 个");
if (minPage != 1 || maxPage != pages) reasons.add("页码区间 " + minPage + "-" + maxPage);
if (dirtyKeys > 0) reasons.add("imgkey 为空 " + dirtyKeys + " 行");
return String.join("、", reasons);
}
// ---------- 修复 ----------
/** 删除不健康 gid 的整组索引与孤儿页 key,返回被移除索引的 gid 个数。 */
static int repair(Connection connection, List<String> badGids, List<String> orphanGids)
throws SQLException {
int touched = 0;
try (PreparedStatement pages = connection.prepareStatement("delete from ImageKeyCache where gid = ?");
PreparedStatement header = connection.prepareStatement("delete from gidToKey where gid = ?")) {
for (String gid : badGids) {
pages.setString(1, gid);
pages.executeUpdate();
header.setString(1, gid);
header.executeUpdate();
touched++;
}
for (String gid : orphanGids) {
pages.setString(1, gid);
if (pages.executeUpdate() > 0)
touched++;
}
}
return touched;
}
// ---------- 基础设施 ----------
private static boolean tableExists(Connection connection, String table) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"select count(*) from sqlite_master where type='table' and name=?")) {
statement.setString(1, table);
try (ResultSet rows = statement.executeQuery()) {
return rows.next() && rows.getInt(1) > 0;
}
}
}
private static int count(Connection connection, String sql) throws SQLException {
try (Statement statement = connection.createStatement();
ResultSet rows = statement.executeQuery(sql)) {
return rows.next() ? rows.getInt(1) : 0;
}
}
private static Map<String, String> parseOptions(String[] args) {
Map<String, String> options = new LinkedHashMap<>();
for (int i = 0; i < args.length; i++) {
String arg = args[i];
if (!arg.startsWith("--"))
continue;
String name = arg.substring(2);
if (name.equals("apply") || name.equals("help") || name.equals("allow-production")) {
options.put(name, "true");
continue;
}
if (i + 1 >= args.length) {
System.err.println("缺少参数值: " + arg);
System.exit(2);
}
options.put(name, args[++i]);
}
return options;
}
private static void printUsage() {
System.out.println("""
用法: java -cp <sqlite-jdbc.jar> GalleryIndexMaintenance.java --db <cache.db> [--apply] [--allow-production]
--db <path> 必填,缓存库文件
--apply 执行修复;不加则只做 dry-run 体检
--allow-production 允许写入 /home/lionwebsite 下的数据库(默认拒绝)
退出码: 0 健康或修复成功 / 1 存在待修复项(dry-run) / 2 参数错误 /
3 拒绝写生产路径 / 4 修复后仍不健康 / 5 执行异常
""");
}
private GalleryIndexMaintenance() { }
/** 一次体检结果。 */
static final class Report {
boolean hasHeaderTable;
boolean hasPageTable;
int headerRows;
int pageRows;
int duplicateHeaderGids;
int duplicatePageGroups;
final List<String> badGids = new ArrayList<>();
final List<String> orphanGids = new ArrayList<>();
final Map<String, String> badReasons = new LinkedHashMap<>();
boolean isHealthy() {
return hasHeaderTable && hasPageTable && badGids.isEmpty() && orphanGids.isEmpty();
}
String render(String title) {
StringBuilder out = new StringBuilder();
out.append("=== 图片索引").append(title).append("(").append(Instant.now()).append(")===\n");
if (!hasHeaderTable || !hasPageTable) {
out.append("缺少缓存表: gidToKey=").append(hasHeaderTable)
.append(" ImageKeyCache=").append(hasPageTable).append('\n');
return out.toString();
}
out.append("gidToKey 行数: ").append(headerRows).append('\n');
out.append("ImageKeyCache 行数: ").append(pageRows).append('\n');
out.append("重复 gid 组: ").append(duplicateHeaderGids).append('\n');
out.append("重复 (gid,page) 组: ").append(duplicatePageGroups).append('\n');
out.append("不完整或冲突 gid: ").append(badGids.size()).append('\n');
int limit = 20;
for (int i = 0; i < badGids.size() && i < limit; i++) {
String gid = badGids.get(i);
out.append(" - gid=").append(gid).append(' ')
.append(badReasons.getOrDefault(gid, "")).append('\n');
}
if (badGids.size() > limit)
out.append(" ... 其余 ").append(badGids.size() - limit).append(" 个略\n");
out.append("孤儿页 key 的 gid: ").append(orphanGids.size()).append('\n');
out.append(isHealthy() ? "结论: 健康\n" : "结论: 需要修复\n");
return out.toString();
}
}
}
@@ -21,6 +21,20 @@ public class AbstractMessage {
public static final byte SUBSCRIPTION_SNAPSHOT_MESSAGE = 9;
// ---- 协议 v2(可靠任务)。旧类型 0..9 的字段保持不变,不能新增字段。 ----
/** 能力协商:{@code {protocolVersion: 2, features: ["reliable-task-v2"]}}。 */
public static final byte PROTOCOL_HELLO_MESSAGE = 10;
/** 带稳定 operationId 的任务命令(DOWNLOAD/DELETE)。 */
public static final byte TASK_COMMAND_MESSAGE = 11;
/** 带 reportId + revision 的状态上报,替代旧的无确认上报。 */
public static final byte TASK_REPORT_MESSAGE = 12;
/** 主站提交事务成功后的上报回执,节点据此清理快照。 */
public static final byte TASK_REPORT_ACK_MESSAGE = 13;
public byte messageType;
public int messageId;
@@ -45,6 +45,10 @@ public class MessageCodec extends ByteToMessageCodec<AbstractMessage> {
case AbstractMessage.MAINTAIN_MESSAGE -> objectMapper.readValue(metadata, MaintainMessage.class);
case AbstractMessage.AVAILABLE_CHECK_MESSAGE -> objectMapper.readValue(metadata, AvailableCheckMessage.class);
case AbstractMessage.SUBSCRIPTION_SNAPSHOT_MESSAGE -> objectMapper.readValue(metadata, SubscriptionSnapshotMessage.class);
case AbstractMessage.PROTOCOL_HELLO_MESSAGE -> objectMapper.readValue(metadata, ProtocolHelloMessage.class);
case AbstractMessage.TASK_COMMAND_MESSAGE -> objectMapper.readValue(metadata, TaskCommandMessage.class);
case AbstractMessage.TASK_REPORT_MESSAGE -> objectMapper.readValue(metadata, TaskReportMessage.class);
case AbstractMessage.TASK_REPORT_ACK_MESSAGE -> objectMapper.readValue(metadata, TaskReportAckMessage.class);
default -> null;
};
@@ -0,0 +1,30 @@
package com.lion.lionwebsite.Message;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
/** 协议 v2 能力协商(类型 10)。双方都以对方声明了 version>=2 为前提才启用新消息。 */
@Data
@NoArgsConstructor
public class ProtocolHelloMessage extends AbstractMessage {
{
messageType = PROTOCOL_HELLO_MESSAGE;
}
public static final int PROTOCOL_V2 = 2;
public static final String FEATURE_RELIABLE_TASK_V2 = "reliable-task-v2";
private int protocolVersion = PROTOCOL_V2;
private List<String> features = new ArrayList<>(List.of(FEATURE_RELIABLE_TASK_V2));
/** 对端是否声明了 v2 可靠任务能力。 */
public boolean supportsReliableTasks() {
return protocolVersion >= PROTOCOL_V2
&& features != null && features.contains(FEATURE_RELIABLE_TASK_V2);
}
}
@@ -0,0 +1,29 @@
package com.lion.lionwebsite.Message;
import lombok.Data;
/**
* 任务命令(类型 11)。
*
* <p>只有确认对端支持 v2 之后才发送。同一逻辑操作在重试/重连后必须复用同一个
* {@link #operationId},节点据此返回原结果而不重复执行副作用。
*/
@Data
public class TaskCommandMessage extends AbstractMessage {
{
messageType = TASK_COMMAND_MESSAGE;
}
public static final String ACTION_DOWNLOAD = "DOWNLOAD";
public static final String ACTION_DELETE = "DELETE";
/** 稳定操作 ID:DOWNLOAD 与 DELETE 各自独立。 */
private String operationId;
private String action;
private int gid;
private String name;
}
@@ -0,0 +1,29 @@
package com.lion.lionwebsite.Message;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 上报回执(类型 13)。
*
* <p>只在任务行与回执已经提交之后才发送;重复上报(同一 reportId)也会得到回执,
* 让节点停止重放。{@link #result} 0 表示已落库。
*/
@Data
@NoArgsConstructor
public class TaskReportAckMessage extends AbstractMessage {
{
messageType = TASK_REPORT_ACK_MESSAGE;
}
public static final byte RESULT_SUCCESS = 0;
private String reportId;
private byte result;
public TaskReportAckMessage(String reportId, byte result) {
this.reportId = reportId;
this.result = result;
}
}
@@ -0,0 +1,32 @@
package com.lion.lionwebsite.Message;
import com.lion.lionwebsite.Domain.GalleryTask;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* 节点状态上报(类型 12)。
*
* <p>相比旧的无确认上报,这里带上单调递增的 {@link #revision} 与唯一 {@link #reportId}:
* 主站据此丢弃乱序/重放的旧快照,并在事务提交后回 {@link TaskReportAckMessage},
* 节点只有在收到匹配回执后才清理自己的快照。
*/
@Data
public class TaskReportMessage extends AbstractMessage {
{
messageType = TASK_REPORT_MESSAGE;
}
/** 本次上报的唯一 ID,重复投递同一份快照时保持不变。 */
private String reportId;
/** 节点侧单调递增版本号;主站只接受比已记录更大的 revision。 */
private long revision;
private GalleryTask[] galleryTasks;
/** gid → 产生该状态的下载 operationId,用于丢弃属于旧操作的迟到上报。 */
private Map<Integer, String> operationIds = new LinkedHashMap<>();
}
@@ -0,0 +1,262 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Dao.cache.ImageCacheMapper;
import com.lion.lionwebsite.Domain.GidToKey;
import com.lion.lionwebsite.Domain.ImageKeyCache;
import com.lion.lionwebsite.Exception.GalleryIndexException;
import com.lion.lionwebsite.Util.GalleryUtil;
import com.lion.lionwebsite.Util.SingleFlight;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.transaction.support.TransactionOperations;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* 图片索引(gid 到每页 imgkey)的唯一读写入口。
*
* <p>这个类替换掉原先「读缓存 / 写缓存 / 回滚」分散在业务服务里、并且逐页 INSERT
* 的写法。三件事在这里被固定下来:
*
* <ol>
* <li><b>同 gid 合并</b>:并发请求同一本画廊时只有一个线程真的去抓上游,
* 其余线程等待同一个结果(含失败),因此不会出现两个构建者互相删除对方写入的索引。</li>
* <li><b>抓取在事务外</b>:上游 HTTP 可能耗时十几秒,绝不能占着 SQLite 写事务,
* 否则整个缓存库的写入都会排队。</li>
* <li><b>整组短事务发布</b>:一个事务里先删该 gid 的旧索引,再批量写入新索引。
* 失败整体回滚,磁盘上仍保留旧的完整索引;只有写入成功这一个状态是对外可见的。</li>
* </ol>
*
* <p>依赖全部由构造函数注入:mapper 负责落库,operations 负责事务边界,
* upstream 负责把 mpv 页解析成页 key 列表。测试可以直接给假的实现,
* 不需要起 Spring 上下文,也不需要网络。
*/
public class GalleryIndexService {
private static final Logger log = LoggerFactory.getLogger(GalleryIndexService.class);
/** 单条 INSERT 携带的最大行数,避免 SQL 过长。 */
static final int BATCH_SIZE = 200;
private final ImageCacheMapper mapper;
private final TransactionOperations operations;
private final TransactionOperations readOperations;
private final GalleryKeyUpstream upstream;
private final SingleFlight<String, GidToKey> builds = new SingleFlight<>();
public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations,
GalleryKeyUpstream upstream) {
this(mapper, operations, operations, upstream);
}
/**
* @param operations 发布索引的写事务(删除加批量写入必须整体生效)
* @param readOperations 读取索引的一致性快照事务;传 null 表示不额外开启读事务
*/
public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations,
TransactionOperations readOperations, GalleryKeyUpstream upstream) {
if (mapper == null || operations == null || upstream == null)
throw new IllegalArgumentException("mapper/operations/upstream 不能为空");
this.mapper = mapper;
this.operations = operations;
this.readOperations = readOperations;
this.upstream = upstream;
}
/**
* 保证该画廊的索引可用,返回 gid 到 key 的映射。
*
* @param url 画廊链接,形如 https://exhentai.org/g/1596929/f08534d87d/
* @throws GalleryIndexException 链接错误 / 画廊不可用 / 暂时失败,见 GalleryIndexException.Reason
*/
public GidToKey ensureIndex(String url) throws GalleryIndexException {
return ensureIndex(url, this::buildAndPublish);
}
/**
* 同 ensureIndex,但允许调用方提供构建动作。
*
* <p>离线重建需要「先检查再决定是否重建」的不同策略,因此把构建步骤作为参数暴露出来;
* 生产调用使用默认实现即可。
*/
GidToKey ensureIndex(String url, IndexBuilder builder) throws GalleryIndexException {
GalleryLink link = GalleryLink.parse(url);
GidToKey cached = lookup(link);
if (cached != null)
return cached;
try {
return builds.run(link.gid(), () -> {
// 进入临界区后必须再看一眼:等待期间别的线程可能已经把索引建好了。
GidToKey existing = lookup(link);
return existing != null ? existing : builder.build(url);
});
} catch (GalleryIndexException e) {
throw e;
} catch (Exception e) {
throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT,
"构建图片索引失败 gid=" + link.gid(), e);
}
}
/**
* 已缓存且完整则返回索引,否则返回 null。
*
* <p>完整性由页码集合判定,而不是行数:历史重复行与半截写入都会让行数与页数不符。
*/
GidToKey lookup(GalleryLink link) {
if (readOperations != null)
return readOperations.execute(status -> lookupInTransaction(link));
return lookupInTransaction(link);
}
/**
* 在同一个读事务里读头部与页码,得到一个一致的快照。
*
* <p>两个独立的 SELECT 之间可能夹着另一个线程的整组替换,于是头部是旧的、页码是新的
* (或相反),完整性判断会得出错误结论。SQLite 的读事务能看到固定快照,避免这种交错。
*/
private GidToKey lookupInTransaction(GalleryLink link) {
GidToKey header = mapper.selectKeyByGid(link.gid());
if (header == null)
return null;
if (header.getPages() <= 0 || header.getKey() == null || header.getKey().isBlank())
return null;
if (!isComplete(header))
return null;
return header;
}
private boolean isComplete(GidToKey header) {
List<Integer> pages = mapper.selectPageNumbersByGid(header.getGid());
if (pages == null || pages.size() != header.getPages())
return false;
if (mapper.countInvalidImageKeysByGid(header.getGid()) != 0)
return false;
// 页码必须是严格递增的 1..pages,重复或跳号都视为不完整。
for (int i = 0; i < pages.size(); i++) {
Integer page = pages.get(i);
if (page == null || page != i + 1)
return false;
}
return true;
}
/** 抓上游(事务外)后短事务整组替换。 */
private GidToKey buildAndPublish(String url) throws GalleryIndexException {
GalleryLink link = GalleryLink.parse(url);
List<ImageKeyCache> pageKeys;
try {
// 抓取必须发生在事务之外:上游可能耗时十几秒,占着 SQLite 写事务会拖垮整个缓存库。
pageKeys = upstream.fetchPageKeys(url);
} catch (Exception e) {
throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT,
"抓取图片索引失败 gid=" + link.gid(), e);
}
if (pageKeys == null || pageKeys.isEmpty())
throw new GalleryIndexException(GalleryIndexException.Reason.UNAVAILABLE,
"该图片已下架或已被删除 gid=" + link.gid());
// 落库前先校验上游数据自洽:整组替换一旦开始就没有回头路,
// 与其把畸形页码写进唯一约束再回滚,不如在此之前就判为失败。
validatePageKeys(link, pageKeys);
GidToKey header = new GidToKey();
header.setGid(link.gid());
header.setKey(link.key());
header.setPages(pageKeys.size());
publish(header, pageKeys);
return header;
}
/**
* 校验上游返回的页 key 是否自洽。
*
* <p>四条要求缺一不可,否则索引会以「看起来完整」的样子写进库:
* gid 属于请求的那本画廊、页码恰好是 1..N、页码不重复、imgkey 非空。
* 任一不满足都抛出 {@link GalleryIndexException.Reason#MALFORMED},此时尚未开启写事务,
* 库中旧索引保持原样。
*/
void validatePageKeys(GalleryLink link, List<ImageKeyCache> pageKeys) throws GalleryIndexException {
int expected = pageKeys.size();
Set<Integer> seen = new HashSet<>(expected * 2);
for (ImageKeyCache row : pageKeys) {
if (row == null || row.getGid() == null || !row.getGid().equals(link.gid()))
throw malformed(link, "上游页 key 的 gid 与请求不一致");
if (row.getImgkey() == null || row.getImgkey().isBlank())
throw malformed(link, "上游页 key 的 imgkey 为空");
int page = row.getPage();
if (page < 1 || page > expected)
throw malformed(link, "上游页码越界: " + page + "(共 " + expected + " 页)");
if (!seen.add(page))
throw malformed(link, "上游页码重复: " + page);
}
if (seen.size() != expected)
throw malformed(link, "上游页码不连续,去重后 " + seen.size() + " 个,期望 " + expected + " 个");
}
private GalleryIndexException malformed(GalleryLink link, String detail) {
log.warn("上游图片索引非法 gid={} detail={}", link.gid(), detail);
return new GalleryIndexException(GalleryIndexException.Reason.MALFORMED,
detail + " gid=" + link.gid());
}
/**
* 在单个事务里整组替换索引。
*
* <p>失败时事务整体回滚:要么看到旧的完整索引,要么看到新的完整索引,
* 不存在「头部已更新、页 key 缺失」的中间态。
*/
void publish(GidToKey header, List<ImageKeyCache> pageKeys) {
operations.executeWithoutResult(status -> {
mapper.deleteImageKeyCacheByGid(header.getGid());
mapper.deleteGidToKey(header.getGid());
mapper.insertGidToKey(header);
writeBatches(pageKeys);
});
log.info("图片索引已整体替换 gid={} pages={}", header.getGid(), header.getPages());
}
private void writeBatches(List<ImageKeyCache> pageKeys) {
for (int start = 0; start < pageKeys.size(); start += BATCH_SIZE) {
int end = Math.min(start + BATCH_SIZE, pageKeys.size());
mapper.insertImageKeyCacheBatch(new ArrayList<>(pageKeys.subList(start, end)));
}
}
/** 便于日志与错误信息使用的解析结果。 */
record GalleryLink(String gid, String key) {
static GalleryLink parse(String url) throws GalleryIndexException {
String[] segments = url == null ? null : url.split("/");
Integer gid = GalleryUtil.parseGid(url);
if (gid == null || segments == null || segments.length <= 5 || segments[5].isBlank())
throw new GalleryIndexException(GalleryIndexException.Reason.INVALID_LINK,
"链接错误");
return new GalleryLink(String.valueOf(gid), segments[5].strip());
}
}
/**
* 上游取页 key 的端口。
*
* <p>生产实现只是把 URL 转交给 GalleryUtil.parseImageKeys;
* 单独抽出来是为了让本服务在测试里完全脱离网络与静态工具类。
*/
@FunctionalInterface
public interface GalleryKeyUpstream {
List<ImageKeyCache> fetchPageKeys(String url) throws Exception;
}
/**
* 构建索引的动作。允许抛出业务异常,因此不用 java.util.function.Function
* (后者不能抛受检异常,会让调用方被迫包装成运行时异常)。
*/
@FunctionalInterface
interface IndexBuilder {
GidToKey build(String url) throws GalleryIndexException;
}
}
@@ -46,6 +46,16 @@ public class GalleryManageService {
final PushService pushService;
/**
* 任务生命周期(协议 v2)。创建/删除都先走它做持久化意图,再产生外部副作用。
* 用字段注入是为了兼容现有构造式 mock 测试与旧调用点。
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
GalleryTaskLifecycleService lifecycleService;
@org.springframework.beans.factory.annotation.Autowired
GalleryIndexService galleryIndexService;
/**
* 创建任务
* @param link 任务链接
@@ -79,6 +89,20 @@ public class GalleryManageService {
pushService.taskCreateReport(user.getUsername(), taskName, response);
return response.toJSONString();
}
// 协议 v2 生命周期:gid 唯一意图先落库(占位行「提交中」),提交前持久化 SUBMITTING,
// 上游结果不确定时报 UNKNOWN 而不是自动重发;差额额度与任务行在同一短事务里写入。
if (lifecycleService != null) {
response = lifecycleService.createTask(gid, taskName, link, targetResolution, user.getId());
Gallery current = galleryMapper.selectGalleryByGid(gid);
taskName = current == null || current.getName() == null ? taskName : current.getName();
// 业务结果由生命周期服务给出,data 用结构化对象下发(前端解码器兼容旧字符串)。
if (response.isSuccess() && current != null)
response.success(objectMapper.valueToTree(current));
pushService.taskCreateReport(user.getUsername(), taskName, response);
return response.toJSONString();
}
Gallery gallery;
//判断数据库中是否有这个任务,有则返回状态
@@ -124,7 +148,7 @@ public class GalleryManageService {
// Do not overwrite an immediate node status with the original submitted state.
Gallery current = galleryMapper.selectGalleryByGid(gallery.getGid());
response.success((current == null ? gallery : current).toString());
response.success(objectMapper.valueToTree(current == null ? gallery : current));
pushService.taskCreateReport(user.getUsername(), taskName, response);
return response.toJSONString();
@@ -184,7 +208,7 @@ public class GalleryManageService {
try {
gallery = GalleryUtil.parse(link, false, null);
if (gallery != null)
response.success(objectMapper.valueToTree(gallery).toString());
response.success(objectMapper.valueToTree(gallery));
else
response.failure("查询失败");
} catch (Exception e) {
@@ -193,7 +217,7 @@ public class GalleryManageService {
}
else
response.success(objectMapper.valueToTree(gallery).toString());
response.success(objectMapper.valueToTree(gallery));
return response.toJSONString();
}
@@ -212,7 +236,7 @@ public class GalleryManageService {
response.failure("未找到该图片,请使用链接");
else
response.success(gallery.toString());
response.success(objectMapper.valueToTree(gallery));
return response.toJSONString();
}
@@ -245,7 +269,7 @@ public class GalleryManageService {
if (userId == UserService.ADMIN_USER_ID)
fillDownloaderNames(galleries);
response.success(objectMapper.valueToTree(galleries).toString());
response.success(objectMapper.valueToTree(galleries));
return response.toJSONString();
}
@@ -302,7 +326,7 @@ public class GalleryManageService {
Gallery gallery = galleryMapper.selectGalleryByName("%" + name + "%");
if (gallery != null)
response.success(objectMapper.valueToTree(gallery).toString());
response.success(objectMapper.valueToTree(gallery));
else
response.failure("没有找到该名字的图片");
@@ -320,7 +344,7 @@ public class GalleryManageService {
Gallery[] galleries = galleryMapper.selectGalleryByDownloader(userMapper.selectUserByAuthCode(AuthCode).getId());
if (galleries.length > 0)
response.success(objectMapper.valueToTree(galleries).toString());
response.success(objectMapper.valueToTree(galleries));
else
response.failure("您未下载图片");
@@ -358,8 +382,46 @@ public class GalleryManageService {
return response.toJSONString();
}
// 通过授权后才落库并通知节点,避免被拒请求仍删除节点文件。
// 通过授权后才产生外部副作用。协议 v2 的删除顺序:
// 先落「待删除」意图 → 节点确认(或文件已不存在)→ 才在同一事务删任务行并留墓碑。
// 失败时任务行与意图都保留,可沿原记录重试并复用同一个删除 operationId。
log.info("删除图片{}", gallery.getName());
if (lifecycleService != null) {
// 被拒绝(例如任务仍在提交中)时必须回业务失败:
// 绝不能退避到裸删除,否则会与并发的创建 POST 同时执行并写回已删/重建的 gid。
GalleryTaskLifecycleService.DeleteIntent intent = lifecycleService.beginDelete(gallery.getGid());
if (!intent.accepted()) {
response.failure(intent.message());
log.info("拒绝删除 gid={}:{}", gid, intent.message());
return response.toJSONString();
}
byte result = remoteService.sendDeleteCommand(intent.deleteOperationId(),
gallery.getGid(), gallery.getName());
switch (result) {
case 0 -> {
lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId());
response.success();
}
// 文件已不存在等于目标达成:同样完成本地清理,避免留下永远删不掉的任务行。
case ErrorCode.FILE_NOT_FOUND -> {
lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId());
response.success();
}
case ErrorCode.IO_ERROR -> {
lifecycleService.failDelete(gallery.getGid());
response.failure("图片:" + gallery.getName() + "删除失败,IO错误");
}
// 节点无响应/超时:保留记录与待删除意图,可重试。
default -> {
lifecycleService.failDelete(gallery.getGid());
response.failure("图片:" + gallery.getName() + "删除失败,节点无响应");
}
}
if (!response.isSuccess())
log.info(response.getData());
return response.toJSONString();
}
galleryMapper.deleteGalleryByGid(gallery.getGid());
switch (remoteService.deleteGallery(gallery)) {
case ErrorCode.IO_ERROR -> response.failure("图片:" + gallery.getName() + "删除失败,IO错误");
@@ -399,76 +461,25 @@ public class GalleryManageService {
lastResetAmountTime == null || lastResetAmountTime.getValue() == null
? "" : lastResetAmountTime.getValue());
response.success(objectMapper.valueToTree(data).toString());
response.success(objectMapper.valueToTree(data));
return response.toJSONString();
}
public String cacheImagesKey(String url) {
Response response = Response.generateResponse();
// 畸形链接在这里就拦掉:段数不足会让 url.split("/")[5] 抛 ArrayIndexOutOfBoundsException,
// 而 parseGid 也会对无 /g/ 的链接返回 null,两者都不该变成 500。
String[] segments = url == null ? null : url.split("/");
Integer parsedGid = parseGidFromLink(url);
if (parsedGid == null || segments == null || segments.length <= 5) {
response.failure("链接错误");
return response.toJSONString();
}
String gid = String.valueOf(parsedGid);
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
//已缓存过且页 key 完整,直接返回
if(gidToKey != null && isIndexComplete(gidToKey)) {
return response.success(objectMapper.valueToTree(gidToKey)).toJSONString();
}
// 半截缓存(历史崩溃或旧版本写入中断)先清干净再整体重建,否则残留的页 key 会重复累积。
if (gidToKey != null) {
log.warn("图片索引缓存不完整,重建 gid={} 记录页数={}", gid, gidToKey.getPages());
rollbackImageIndex(gid);
}
boolean inserted = false;
try {
gidToKey = new GidToKey();
gidToKey.setGid(gid);
gidToKey.setKey(segments[5].strip());
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
if(imageKeyCaches == null)
return response.failure("该图片已下架或已被删除").toJSONString();
gidToKey.setPages(imageKeyCaches.size());
imageCacheMapper.insertGidToKey(gidToKey);
inserted = true;
for (ImageKeyCache imageKeyCache : imageKeyCaches)
imageCacheMapper.insertImageKeyCache(imageKeyCache);
response.success(objectMapper.valueToTree(gidToKey));
}catch (Exception e){
// 索引必须整体生效:先写 gidToKey 再逐页写 key,中途失败会留下「gidToKey 命中、
// 页 key 缺失」的半截缓存,后续请求直接返回已缓存而永远取不到图。
// 因此任何异常都回滚已写入的部分再回业务失败。Jackson 3 的解析异常继承
// RuntimeException 而非 IOException,只用 IOException 会漏掉,这里统一兜住。
if (inserted)
rollbackImageIndex(gid);
log.warn("缓存图片索引失败 gid={} errorType={}", gid, e.getClass().getSimpleName());
response.failure("网络波动或其他异常");
}
return response.toJSONString();
}
/** 页 key 行数与 gidToKey 记录不一致时视为半截缓存,需要重建。 */
private boolean isIndexComplete(GidToKey gidToKey) {
Integer pages = gidToKey.getPages();
if (pages == null || pages <= 0)
return false;
return imageCacheMapper.countImageKeyCacheByGid(gidToKey.getGid()) == pages;
}
/** 删除某个 gid 已写入的索引(页 key 与 gidToKey),用于回滚或重建。 */
private void rollbackImageIndex(String gid) {
try {
imageCacheMapper.deleteImageKeyCacheByGid(gid);
imageCacheMapper.deleteGidToKey(gid);
} catch (Exception e) {
log.warn("回滚图片索引缓存失败 gid={}", gid, e);
GidToKey index = galleryIndexService.ensureIndex(url);
return Response.generateResponse().success(objectMapper.valueToTree(index)).toJSONString();
} catch (com.lion.lionwebsite.Exception.GalleryIndexException error) {
String message = switch (error.getReason()) {
case INVALID_LINK -> "链接错误";
case UNAVAILABLE -> "该图片已下架或已被删除";
case MALFORMED, TRANSIENT -> "网络波动或其他异常";
};
log.warn("缓存图片索引失败 reason={}", error.getReason());
return Response._failure(message);
} catch (RuntimeException error) {
log.warn("缓存图片索引失败 errorType={}", error.getClass().getSimpleName());
return Response._failure("网络波动或其他异常");
}
}
@@ -517,10 +528,24 @@ public class GalleryManageService {
Gallery[] galleries = galleryMapper.selectUnDoneGalleries();
if(galleries != null && galleries.length != 0) {
log.info("发送未下载完成图片至节点,{}本", galleries.length);
for (Gallery gallery : galleries)
// 只重投「确实已提交上游」的任务:提交未知/预占中/待删除的任务重投会变成
// 未计量副作用(节点侧显示下载完成但没有任何额度记录)。
int sent = 0;
int skipped = 0;
for (Gallery gallery : galleries) {
if (lifecycleService != null
&& !lifecycleService.resolveDispatch(gallery.getGid()).dispatchable()) {
skipped++;
continue;
}
remoteService.addGalleryToQueue(gallery);
response.success(String.format("发送未下载完成图片至节点,%s本", galleries.length));
sent++;
}
log.info("发送未下载完成图片至节点,{}本(跳过需要对账的{}本)", sent, skipped);
if (sent == 0)
response.failure("没有可安全重投的任务,需先人工对账提交未知的任务");
else
response.success(String.format("发送未下载完成图片至节点,%s本", sent));
}else{
response.failure("当前没有未下载完成的图片");
}
@@ -536,6 +561,13 @@ public class GalleryManageService {
return Response._success("下载完成");
if(remoteService.isDead())
return Response._failure("节点不在线,无法重试");
// 提交结果未知(UNKNOWN)或正在删除的任务不能重试:上游不支持幂等,重发会让
// 「可能已受理但未计量」的任务在节点侧变成下载完成却 0 额度。这类任务需人工对账。
if (lifecycleService != null) {
var decision = lifecycleService.resolveDispatch(gid);
if (decision.kind() == GalleryTaskLifecycleService.DispatchDecision.Kind.REFUSE)
return Response._failure("该任务提交结果未知或正在删除,请先人工对账后再重试");
}
RemoteService.RetryResult retryResult = remoteService.retryGallery(gallery);
if(retryResult.success())
@@ -0,0 +1,783 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Dao.normal.*;
import com.lion.lionwebsite.Domain.*;
import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
import com.lion.lionwebsite.Message.TaskCommandMessage;
import com.lion.lionwebsite.Message.TaskReportMessage;
import com.lion.lionwebsite.Util.PreparedGallery;
import com.lion.lionwebsite.Util.Response;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
/**
* 下载任务的持久化生命周期(协议 v2)。
*
* <p>核心约束:
* <ol>
* <li><b>先保存意图,再产生外部副作用</b>:同一 gid 的唯一 CREATE 意图先落库(占位任务行 + RESERVED 操作记录),
* 通过 gid 的主键唯一约束把并发创建收敛成「一个赢家」;</li>
* <li><b>SUBMITTING 之后绝不自动重发</b>:上游不支持幂等,提交结果不确定时只能落 UNKNOWN,等显式对账/解决;</li>
* <li><b>提交结果与额度用一个短事务写入</b>:Gallery 与 weekUsedAmount 要么一起成功,要么一起回滚,
* 事务提交之后才把 DOWNLOAD 命令下发给节点;</li>
* <li><b>删除失败保留记录</b>:先落待删除意图,节点确认(或文件已不存在)后才在同一事务里删任务行并留墓碑;
* 失败则保留 Gallery 与 DELETE_PENDING 意图,可沿原记录重试且复用同一删除 operationId。</li>
* </ol>
*
* <p>所有网络调用都发生在事务之外;事务回调内部不做 I/O,也不吞异常(吞掉会错误提交)。
*/
@Service
@Slf4j
public class GalleryTaskLifecycleService implements TaskReportHandler {
final GalleryMapper galleryMapper;
final CollectMapper collectMapper;
final CustomConfigurationMapper configurationMapper;
final UserMapper userMapper;
final TaskOperationRepository taskOperationRepository;
final UpstreamGateway upstreamGateway;
final RemoteService remoteService;
/**
* 短全局创建预占锁。
*
* <p>只有「登记唯一 CREATE 意图」这一小段数据库写入在锁内,网络调用一律在锁外。
* 单进程内它把并发同 gid 创建直接收敛成一个赢家;跨进程的正确性仍由
* {@code task_operation}/{@code gallery} 的主键唯一约束兜底。
*/
private final Object reservationLock = new Object();
/**
* 测试钩子:进入上游只读解析之前执行。
* 用来让并发删除发生在预占之后、SUBMITTING 之前。生产环境是空实现。
*/
volatile Runnable beforePrepare = () -> { };
/**
* 测试钩子:SUBMITTING 已持久化之后、真正 POST 上游之前执行。
* 用来制造「提交瞬间任务被删除」的 late generation 场景。生产环境是空实现。
*/
volatile Runnable afterMarkSubmitting = () -> { };
/** 主库事务模板;测试可注入内存 sqlite 的事务模板。 */
TransactionTemplate transactions;
public GalleryTaskLifecycleService(GalleryMapper galleryMapper,
CollectMapper collectMapper,
CustomConfigurationMapper configurationMapper,
UserMapper userMapper,
TaskOperationRepository taskOperationRepository,
UpstreamGateway upstreamGateway,
RemoteService remoteService,
@Qualifier("mainTransactionTemplate") TransactionTemplate transactions) {
this.galleryMapper = galleryMapper;
this.collectMapper = collectMapper;
this.configurationMapper = configurationMapper;
this.userMapper = userMapper;
this.taskOperationRepository = taskOperationRepository;
this.upstreamGateway = upstreamGateway;
this.remoteService = remoteService;
this.transactions = transactions;
}
/** 测试用:替换事务模板或在没有 Spring 事务时退化为直连执行。 */
void setTransactions(TransactionTemplate transactions) {
this.transactions = transactions;
}
/**
* 一次「登记创建意图」的结果。
*
* <p>只有赢家能继续做外部副作用;输家(已存在有效意图 / 数据库竞争失败)
* 一律转成业务失败,绝不再 POST 上游。
*/
private enum ReserveOutcome {
/** 已成功登记唯一 CREATE 意图,可以做只读解析与提交。 */
RESERVED,
/** 同一 gid 已存在有效意图(或任务行)。 */
DUPLICATE,
/** 登记时的数据库竞争/失败:没有产生任何副作用,必须转业务失败。 */
CONTENDED
}
/** 一次「登记创建意图」的返回:结果 + 赢家持有的操作记录。 */
private record Reservation(ReserveOutcome outcome, TaskOperation operation) {
}
/**
* 单次下发决策。
*
* <p>把「能不能把这个任务交给节点」从「下载是否已提交」里拆出来:
* 未确认上游提交结果(UNKNOWN / 预占 / 待删除 / 删除墓碑)绝不能下发,
* 否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。
*/
public static final class DispatchDecision {
public enum Kind {
/** 当前 operation 已是 SUBMITTED:可以按稳定 operationId 下发。 */
READY,
/** 迁移前的旧任务行:没有操作记录但历史上确实提交过下载。 */
LEGACY,
/** 不可下发:状态不明或被删除,下发会产生未计量副作用。 */
REFUSE
}
private final Kind kind;
private final String operationId;
private DispatchDecision(Kind kind, String operationId) {
this.kind = kind;
this.operationId = operationId;
}
public Kind kind() {
return kind;
}
public String operationId() {
return operationId;
}
/** 只有 READY/LEGACY 允许真正下发到节点。 */
public boolean dispatchable() {
return kind != Kind.REFUSE && operationId != null;
}
/** 拒绝下发(供调用方与测试构造)。 */
public static DispatchDecision refuse() {
return REFUSE;
}
/** 可直接下发(供测试构造)。 */
public static DispatchDecision ready(String operationId) {
return new DispatchDecision(Kind.READY, operationId);
}
}
private static final DispatchDecision REFUSE = new DispatchDecision(DispatchDecision.Kind.REFUSE, null);
/**
* 一次删除登记的返回。
*
* <p>用显式结果替代「null 即退避到裸删除」的旧语义:
* 被拒绝时调用方必须回业务失败,不能绕过待删除意图直接删节点文件。
*/
public record DeleteIntent(boolean accepted, String deleteOperationId, String message) {
static DeleteIntent refused(String message) {
return new DeleteIntent(false, null, message);
}
static DeleteIntent accepted(String deleteOperationId) {
return new DeleteIntent(true, deleteOperationId, null);
}
}
/** 在事务里执行;没有配置事务模板时候直接执行(仅便于构造 mock 测试)。 */
private <T> T inTransaction(java.util.function.Supplier<T> body) {
TransactionTemplate template = transactions;
if (template == null)
return body.get();
return template.execute(status -> {
try {
return body.get();
} catch (RuntimeException e) {
// 回调里必须把异常继续抛出,否则 Spring 会把只读了一半的事务当成正常提交。
status.setRollbackOnly();
throw e;
}
});
}
/**
* 创建任务的持久化前置:在同一个事务里登记唯一 CREATE 意图。
*
* <p>同时写入可见的占位任务行(状态「提交中」),这样提交期间任务列表就能解析该 gid,
* 不会出现「上游已受理但本地查不到」的空窗。
*
* @return 已登记意图(赢家)返回它;同一 gid 已有有效意图返回 {@code null}(输家)。
*/
private Reservation reserveCreateIntent(int gid, String placeholderName, int downloaderId) {
// 锁内只做数据库写入:把并发同 gid 创建串行化,网络调用留在锁外。
synchronized (reservationLock) {
try {
TaskOperation operation = inTransaction(() -> {
TaskOperation existingOperation = taskOperationRepository.selectByGid(gid);
Gallery existingGallery = galleryMapper.selectGalleryByGid(gid);
if (existingOperation != null) {
// 只有「删除已完成」的墓碑才允许用全新 operationId 重新创建(显式重建)。
if (!TaskOperation.STATE_DELETED.equals(existingOperation.getState()))
return null;
if (existingGallery != null)
return null;
String operationId = TaskProtocol.newDownloadOperationId();
taskOperationRepository.reactivate(gid, operationId, TaskOperation.STATE_RESERVED, now());
insertPlaceholder(gid, placeholderName, downloaderId);
return taskOperationRepository.selectByGid(gid);
}
if (existingGallery != null)
return null;
TaskOperation created = new TaskOperation();
created.setGid(gid);
created.setOperationId(TaskProtocol.newDownloadOperationId());
created.setState(TaskOperation.STATE_RESERVED);
created.setUpdateTime(now());
taskOperationRepository.insert(created);
insertPlaceholder(gid, placeholderName, downloaderId);
return created;
});
return operation == null
? new Reservation(ReserveOutcome.DUPLICATE, null)
: new Reservation(ReserveOutcome.RESERVED, operation);
} catch (RuntimeException e) {
// 数据库竞争/失败发生在任何外部副作用之前,绝不能穿透成 500,也不能继续提交。
log.warn("登记创建意图失败,按业务失败处理 gid={} error={}", gid, e.getClass().getSimpleName());
return new Reservation(ReserveOutcome.CONTENDED, null);
}
}
}
/** 写入占位任务行:让提交期间的任务在列表里可见。 */
private void insertPlaceholder(int gid, String name, int downloaderId) {
Gallery placeholder = new Gallery();
placeholder.setGid(gid);
placeholder.setName(name);
placeholder.setStatus(TaskStatus.PENDING_SUBMIT);
placeholder.setDownloader(downloaderId);
placeholder.setCreateTime(System.currentTimeMillis() / 1000);
placeholder.setProceeding(0);
galleryMapper.insertGallery(placeholder);
}
/**
* 标记为正在提交,并用 CAS 确认这一代意图仍然有效。
*
* <p>只有当前状态与 operationId 都匹配(RESERVED → SUBMITTING)才返回 true;
* 期间被删除或重建返回 false,调用方必须停下,绝不能再 POST 上游。
*
* @return true 表示已持久化 SUBMITTING,可以安全产生上游副作用
*/
boolean markSubmitting(String operationId, int gid) {
return inTransaction(() -> taskOperationRepository.compareAndSetCreateState(gid, operationId,
TaskOperation.STATE_RESERVED, TaskOperation.STATE_SUBMITTING, now()) == 1);
}
/**
* 提交结果未知(超时/断连/中断)。保留意图,等待人工对账,不自动重发。
*
* <p>TaskOperation 与 Gallery 状态在同一个短事务里更新:
* 操作落 UNKNOWN 而任务行仍显示「提交中」会让运营侧误以为还能自动重试,
* 因此两边必须一致地表达「不确定,需要人工对账」。
*/
void markUnknown(String operationId, int gid) {
inTransaction(() -> {
int advanced = taskOperationRepository.compareAndSetCreateState(gid, operationId,
TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now());
// 只有本次这一代确实从 SUBMITTING 推进到 UNKNOWN 时才同步任务行:
// 若期间已被删除/重建(CAS 返回 0),绝不能再改那条记录。
if (advanced != 1)
return null;
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus()))
galleryMapper.updateGalleryStatus(gid, TaskStatus.UNKNOWN);
return null;
});
}
/**
* 提交成功后的「一次性短事务」:写入 Gallery 行、累计周用量、把操作推进为 SUBMITTED。
* 三者同一个事务,任何一个失败都一起回滚,避免出现「任务存在但用量未计」这类不一致。
*/
/**
* 提交成功后的落库(带世代校验)。
*
* @param expectedOperationId 本次提交所归属的 operationId;只有它仍是当前有效意图时才能落库,
* 否则(被删除/重建)必须抛错,避免复活已删 gid 或污染新世代
*/
int persistSubmitted(Gallery gallery, int downloaderId, String expectedOperationId) {
return inTransaction(() -> {
TaskOperation operation = taskOperationRepository.selectByGid(gallery.getGid());
// 严格匹配本次世代:任务被删除、被重建或换了 operationId 时都不能落库。
if (operation == null || TaskOperation.STATE_DELETED.equals(operation.getState())
|| TaskOperation.STATE_DELETE_PENDING.equals(operation.getState()))
throw new IllegalStateException("任务已删除或没有有效创建意图,禁止落库复活 gid=" + gallery.getGid());
if (expectedOperationId == null || !expectedOperationId.equals(operation.getOperationId()))
throw new IllegalStateException("创建意图世代不匹配,禁止落库 gid=" + gallery.getGid());
// 幂等:同一意图重复落库(例如上游已提交后重跑)不得把用量累计两次。
boolean alreadyCounted = TaskOperation.STATE_SUBMITTED.equals(operation.getState());
gallery.setDownloader(downloaderId);
gallery.set_download(true);
if (!TaskStatus.SUBMITTED.equals(gallery.getStatus()))
gallery.setStatus(TaskStatus.SUBMITTED);
Gallery existing = galleryMapper.selectGalleryByGid(gallery.getGid());
if (existing == null)
galleryMapper.insertGallery(gallery);
else
galleryMapper.updateGallery(gallery);
if (!alreadyCounted)
configurationMapper.incrementConfiguration(CustomConfiguration.WEEK_USED_AMOUNT, gallery.getFileSize());
taskOperationRepository.updateState(gallery.getGid(), TaskOperation.STATE_SUBMITTED, now());
return 1;
});
}
/**
* 预占式创建:登记 gid 唯一意图(提交任务行占位)→ 只读解析上游 → 标 SUBMITTING → 真正提交。
*
* <p>上游提交发生在事务之外,且只在 SUBMITTING 已持久化之后执行一次。网络异常一律落 UNKNOWN。
*/
public Response createTask(int gid, String placeholderName, String link, String targetResolution, int downloaderId) {
Response response = Response.generateResponse();
Reservation reservation = reserveCreateIntent(gid, placeholderName, downloaderId);
if (reservation.outcome() == ReserveOutcome.DUPLICATE) {
response.failure("任务队列已有此任务,请点击查找任务");
return response;
}
if (reservation.outcome() == ReserveOutcome.CONTENDED) {
// 预占阶段的数据库竞争不产生任何副作用;转业务失败,绝不下发上游、绝不穿透成 500。
response.failure("任务提交失败,请稍后重试");
return response;
}
// 解析上游是只读步骤;真正的 POST 只在 SUBMITTING 持久化之后发生。
// 生命周期约束:RESERVED 阶段被中断是安全的(可释放重来),SUBMITTING 之后一律按 UNKNOWN 处理。
return submitPreparedIntent(gid, reservation.operation(), link, targetResolution, downloaderId);
}
private Response submitPreparedIntent(int gid, TaskOperation operation, String link, String targetResolution,
int downloaderId) {
Response response = Response.generateResponse();
beforePrepare.run();
PreparedGallery prepared;
try {
prepared = upstreamGateway.prepare(link);
} catch (IOException e) {
log.warn("解析任务链接失败 gid={}", gid, e);
releaseReservation(gid, operation.getOperationId());
response.failure("IO错误,可能是网络波动");
return response;
} catch (RuntimeException e) {
log.warn("解析任务链接异常 gid={}", gid, e);
releaseReservation(gid, operation.getOperationId());
response.failure("提交任务失败,未知原因,请检查链接是否正常");
return response;
}
if (prepared == null || prepared.gallery() == null) {
releaseReservation(gid, operation.getOperationId());
response.failure("提交任务失败,未知原因,请检查链接是否正常");
return response;
}
Gallery gallery = prepared.gallery();
gallery.setGid(gid);
// CAS 确认这一代意图仍然有效(未被并发删除/重建),并把状态持久化为 SUBMITTING。
// 只有成功之后才允许真正 POST;失败说明任务已不在当前生命周期里,直接放弃。
if (!markSubmitting(operation.getOperationId(), gid)) {
log.warn("创建意图已失效(可能被并发删除),放弃提交 gid={} operationId={}",
gid, operation.getOperationId());
response.failure("任务已被删除或状态已变化,请刷新任务列表");
return response;
}
afterMarkSubmitting.run();
Gallery submitted;
try {
submitted = upstreamGateway.submit(prepared, targetResolution);
} catch (ResolutionNotMatchException e) {
releaseReservation(gid, operation.getOperationId());
response.failure("提交失败,分辨率不存在");
return response;
} catch (IOException | RuntimeException e) {
// 超时或断连时上游可能已经受理,无法确认:只能记 UNKNOWN,绝不自动重发。
log.warn("上游提交结果未知 gid={} operationId={}", gid, operation.getOperationId(), e);
markUnknown(operation.getOperationId(), gid);
response.failure("上游提交结果未知,请稍后在任务列表中确认,不要重复提交");
return response;
}
if (submitted == null || !TaskStatus.SUBMITTED.equals(submitted.getStatus())) {
markUnknown(operation.getOperationId(), gid);
response.failure("上游提交结果未知,请稍后在任务列表中确认,不要重复提交");
return response;
}
submitted.setGid(gid);
try {
persistSubmitted(submitted, downloaderId, operation.getOperationId());
} catch (RuntimeException e) {
// 上游已受理但本地落库失败:保留 SUBMITTING 意图并标 UNKNOWN,任务行仍在列表里可对账。
log.error("提交成功后落库失败 gid={} operationId={}", gid, operation.getOperationId(), e);
markUnknown(operation.getOperationId(), gid);
response.failure("任务已提交,但本地保存失败,请在任务列表中确认");
return response;
}
// 只有事务提交之后才向节点下发命令;重复下发由节点按 operationId 去重。
byte dispatchResult = dispatchDownload(gid, submitted.getName());
Gallery current = galleryMapper.selectGalleryByGid(gid);
Gallery payload = current == null ? submitted : current;
if (dispatchResult == 0)
response.success(objectMapper.valueToTree(payload));
else
response.failure("任务已保存,但节点未确认接收;请刷新任务列表后重试");
return response;
}
/**
* 提交成功后的落库 + 下发。落库(含用量)在一个短事务里完成,事务提交后才向节点发命令。
*
* <p>先解析当前世代再落库:世代不存在(未提交过 / 已删除)时不落库也不下发。
*
* @return 已持久化的 Gallery;数据库失败或世代无效时抛出异常,调用方回业务失败且不下发节点。
*/
public Gallery persistAndDispatch(Gallery gallery, int downloaderId) {
DispatchDecision decision = resolveDispatch(gallery.getGid());
if (!decision.dispatchable())
throw new IllegalStateException("任务没有可下发的有效世代 gid=" + gallery.getGid());
persistSubmitted(gallery, downloaderId, decision.operationId());
dispatchDownload(gallery.getGid(), gallery.getName());
return gallery;
}
/** 向节点下发 DOWNLOAD 命令(复用当前有效 operationId),返回节点结果码。 */
public byte dispatchDownload(int gid, String name) {
DispatchDecision decision = resolveDispatch(gid);
if (!decision.dispatchable())
return -1;
return remoteService.sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(), gid, name);
}
/**
* 判定一个 gid 能否下发给节点,并为迁移前的旧任务补齐稳定 operationId。
*
* <p>规则:
* <ul>
* <li>v2 任务只有 {@code SUBMITTED} 可下发;{@code UNKNOWN} 必须人工对账,
* 否则未计量的未知任务会在节点侧变成「下载完成却 0 额度」;</li>
* <li>预占中的 {@code RESERVED}/{@code SUBMITTING}、待删除与删除墓碑都不下发;</li>
* <li>没有任何操作记录、但任务行确实存在且状态已是「已提交」及之后的历史任务,
* 补建一个稳定 operationId 后再下发(这是唯一的 legacy 兼容路径)。</li>
* </ul>
*/
public DispatchDecision resolveDispatch(int gid) {
TaskOperation operation = taskOperationRepository.selectByGid(gid);
if (operation != null && TaskOperation.STATE_SUBMITTED.equals(operation.getState())
&& operation.getOperationId() != null)
return new DispatchDecision(DispatchDecision.Kind.READY, operation.getOperationId());
// 没有操作记录,或只有「旧世代」记录(缺 operationId):必须先确认任务行确实已提交过,
// 再补一个稳定 operationId;UNKNOWN/预占/待删除一律拒绝。
if (operation != null && !TaskOperation.STATE_SUBMITTED.equals(operation.getState()))
return REFUSE;
if (!isLegacySubmitted(gid))
return REFUSE;
String freshId = ensureStableOperationId(gid);
return freshId == null ? REFUSE : new DispatchDecision(DispatchDecision.Kind.LEGACY, freshId);
}
/** 迁移前的旧任务行:没有操作记录,但状态表明上游确实已受理过下载,可安全重投。 */
private boolean isLegacySubmitted(int gid) {
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
if (gallery == null || gallery.getStatus() == null)
return false;
return switch (gallery.getStatus()) {
case TaskStatus.SUBMITTED, TaskStatus.DOWNLOADING, TaskStatus.WAITING_COMPRESS,
TaskStatus.COMPRESSING -> true;
// 提交未知 / 提交中 / 待删除 / 删除失败 一律不算已提交,不能下发。
default -> false;
};
}
/**
* 为一个已经真正提交过的旧任务持久化稳定 operationId。
*
* <p>必须是短事务写入、先落库后返回,之后重投/重连才复用同一个 ID;
* 用 {@code insert or ignore} + 回读,避免与并发创建互相覆盖。
*
* @return 稳定 operationId;无法安全建立时返回 null(调用方不下发)
*/
String ensureStableOperationId(int gid) {
// 与创建预占共用同一把短锁(锁内只有数据库写),并发为同一 gid 补 ID 时只能有一个赢家。
synchronized (reservationLock) {
String candidate = TaskProtocol.newDownloadOperationId();
return inTransaction(() -> {
if (taskOperationRepository.selectByGid(gid) == null)
taskOperationRepository.insertIfAbsent(gid, candidate, TaskOperation.STATE_SUBMITTED, now());
else
taskOperationRepository.assignOperationIdIfMissing(gid, candidate, now());
TaskOperation current = taskOperationRepository.selectByGid(gid);
if (current == null || current.getOperationId() == null)
return null;
// 只有确实处于 SUBMITTED 的世代才可下发;并发写入的其他状态一律拒绝。
return TaskOperation.STATE_SUBMITTED.equals(current.getState()) ? current.getOperationId() : null;
});
}
}
/**
* 取消预占(解析失败、分辨率不存在等:还没有产生任何上游副作用)。
*
* <p>只删除与本次创建 intent 匹配的 RESERVED 记录:若期间该 gid 已被删除并重建
* (新的 operationId / 墓碑),这里绝不能把新世代记录抹掉。占位行同理只删「提交中」。
*/
private void releaseReservation(int gid, String operationId) {
inTransaction(() -> {
// 只删与本代 operationId 匹配且仍停在 RESERVED/SUBMITTING 的记录。
int removed = taskOperationRepository.deleteReservationIfSameGeneration(gid, operationId);
// 只有确实删掉了本代预占记录,才能删对应的占位行:
// 若期间已出现新的 RESERVED 代,删除返回 0,此时绝不能误删新占位。
if (removed == 1) {
Gallery existing = galleryMapper.selectGalleryByGid(gid);
if (existing != null && TaskStatus.PENDING_SUBMIT.equals(existing.getStatus()))
galleryMapper.deleteGalleryByGid(gid);
}
return null;
});
}
/**
* 删除的持久化部分。
*
* <p>正在创建意图(RESERVED/SUBMITTING)的任务不能删除:此刻 DELETE 与创建 POST 会同时发生,
* 随后创建落库还会把已删/重建的 gid 写回。这里直接拒绝,由用户在提交结果明确后再操作。
*
* @return 显式结果:被拒时调用方必须回业务失败,不能退避到裸删除
*/
public DeleteIntent beginDelete(int gid) {
return inTransaction(() -> {
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
if (gallery == null)
return DeleteIntent.refused("删除失败,该图片不存在");
TaskOperation operation = taskOperationRepository.selectByGid(gid);
if (operation != null && (TaskOperation.STATE_RESERVED.equals(operation.getState())
|| TaskOperation.STATE_SUBMITTING.equals(operation.getState())))
return DeleteIntent.refused("任务正在提交中,请等待提交结果明确后再删除");
if (operation == null) {
// 历史任务行没有操作记录:直接落一条 DELETE_PENDING 意图,避免中间态缺字段。
TaskOperation fresh = new TaskOperation();
fresh.setGid(gid);
fresh.setOperationId(TaskProtocol.newDownloadOperationId());
fresh.setDeleteOperationId(TaskProtocol.newDeleteOperationId());
fresh.setState(TaskOperation.STATE_DELETE_PENDING);
fresh.setUpdateTime(now());
taskOperationRepository.insert(fresh);
galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING);
return DeleteIntent.accepted(fresh.getDeleteOperationId());
}
if (operation.getDeleteOperationId() == null) {
operation.setDeleteOperationId(TaskProtocol.newDeleteOperationId());
operation.setUpdateTime(now());
}
taskOperationRepository.updateDeleteIntent(gid, operation.getDeleteOperationId(),
TaskOperation.STATE_DELETE_PENDING, now());
galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING);
return DeleteIntent.accepted(operation.getDeleteOperationId());
});
}
/**
* 删除的完成部分:节点已确认(或文件已不存在)后,在一个短事务里删任务行、收藏并留墓碑。
*
* <p>事务提交前若失败,任务行与 DELETE_PENDING 意图都保留,可以继续重试。
* 删除意图 ID 必须匹配(防止并发删除/重建时抹掉别的一代),随后只把 DELETE_PENDING
* 世代推进为 DELETED;墓碑(DELETED 且无删除意图)不允许在这里被重新激活。
*
* @return true 表示本次删除意图已完成并留下墓碑
*/
public boolean completeDelete(int gid, String deleteOperationId) {
return inTransaction(() -> {
TaskOperation operation = taskOperationRepository.selectByGid(gid);
if (operation == null || operation.getDeleteOperationId() == null
|| !operation.getDeleteOperationId().equals(deleteOperationId))
return false;
// 先清收藏再删任务行:收藏一侧在部分历史库上带有指向 gallery 的外键。
collectMapper.deleteByGid(gid);
galleryMapper.deleteGalleryByGid(gid);
int updated = taskOperationRepository.completeDeleteIfSameGeneration(gid, deleteOperationId,
TaskOperation.STATE_DELETED, now());
return updated == 1;
});
}
/** 删除失败:保留任务行并记一条可重试的失败状态。 */
public void failDelete(int gid) {
inTransaction(() -> {
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
if (gallery == null)
return null;
gallery.setStatus(TaskStatus.DELETE_FAILED);
galleryMapper.updateGallery(gallery);
return null;
});
}
/**
* 启动恢复:把上次中断在 SUBMITTING 的意图判为 UNKNOWN,并安全释放还停在 RESERVED 的预占。
*
* <p>这段恢复必须发生在任何自动重投之前:
* <ul>
* <li>{@code SUBMITTING}:上游可能已经受理但本地无法确认,只能落 UNKNOWN(操作与任务行同时
* 更新为「提交未知」),交人工对账,绝不自动再 POST;</li>
* <li>{@code RESERVED}:还没有产生任何上游副作用,重启后可以直接释放预占
* (删掉匹配的 RESERVED 记录与「提交中」占位行),让用户重新提交。</li>
* </ul>
*
* @return 本次恢复(判未知 + 释放预占)处理的任务数
*/
public int recoverInterruptedSubmissions() {
return inTransaction(() -> {
int recovered = 0;
for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_SUBMITTING)) {
int advanced = taskOperationRepository.compareAndSetCreateState(operation.getGid(), operation.getOperationId(),
TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now());
if (advanced != 1)
continue;
Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid());
if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus()))
galleryMapper.updateGalleryStatus(operation.getGid(), TaskStatus.UNKNOWN);
recovered++;
}
for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_RESERVED)) {
// 还没 POST:释放匹配的预占记录与占位行,允许重启后重新提交。
// 只有确实删掉了这条本代 RESERVED 记录,才删其占位行;否则不动(可能已换代)。
int removed = taskOperationRepository.deleteReservationIfSameGeneration(operation.getGid(),
operation.getOperationId());
if (removed == 1) {
Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid());
if (gallery != null && TaskStatus.PENDING_SUBMIT.equals(gallery.getStatus()))
galleryMapper.deleteGalleryByGid(operation.getGid());
}
recovered++;
}
return recovered;
});
}
/** 已完成状态上报的处理:丢弃旧 revision、旧 operation、待删除任务与重复上报。 */
@Override
public CommittedReport apply(TaskReportMessage report) {
if (report == null || report.getReportId() == null || report.getGalleryTasks() == null)
return null;
Applied applied;
try {
applied = inTransaction(() -> applyInTransaction(report));
} catch (RuntimeException e) {
// 数据库失败:不返回可回执结果,节点会保留快照并稍后重投;失败写入绝不清任务。
log.warn("应用节点上报失败,未回执 reportId={}", report.getReportId(), e);
return null;
}
if (applied == null)
return null;
return new CommittedReport(report.getReportId(), applied.newlyCompleted(), applied.acceptedTasks());
}
/** 事务内应用上报的结果:新完成的任务与真正被接受的进度更新。 */
private record Applied(List<Gallery> newlyCompleted, List<GalleryTask> acceptedTasks) {
}
/**
* 在一个事务里应用上报。
*
* @return 非 null 表示已经提交(可以回 ACK,重复上报也会返回结果以停止重放)。
*/
private Applied applyInTransaction(TaskReportMessage report) {
long revision = report.getRevision();
List<Gallery> newlyCompleted = new ArrayList<>();
List<GalleryTask> acceptedTasks = new ArrayList<>();
for (GalleryTask task : report.getGalleryTasks()) {
if (task == null)
continue;
TaskOperation operation = taskOperationRepository.selectByGid(task.getGid());
Gallery gallery = galleryMapper.selectGalleryByGid(task.getGid());
if (gallery == null) {
// 未知 gid(任务已删除):忽略但 ACK,让节点停止重放。
log.debug("忽略节点上报的未知任务 gid={}", task.getGid());
continue;
}
String expectedOperationId = report.getOperationIds() == null
? null : report.getOperationIds().get(task.getGid());
if (operation == null) {
// 旧任务行(迁移前创建)没有操作记录:按显式生成策略补建并采纳上报里的 operationId。
operation = adoptGeneration(task.getGid(), expectedOperationId);
} else if (TaskOperation.STATE_DELETED.equals(operation.getState())) {
// 墓碑必须永久忽略旧上报:删除已完成的任务只能由显式 CREATE 重新建立新世代。
// (gallery 已被删除时上面的 null 分支已经提前忽略,这里防御性兜底。)
log.debug("忽略已删除 gid 的迟到上报 gid={}", task.getGid());
continue;
} else if (TaskOperation.STATE_DELETE_PENDING.equals(operation.getState())) {
log.info("忽略待删除任务的迟到上报 gid={}", task.getGid());
continue;
} else if (operation.getLastRevision() > 0 && revision <= operation.getLastRevision()) {
log.debug("丢弃过期上报 gid={} revision={} lastRevision={}",
task.getGid(), revision, operation.getLastRevision());
continue;
} else if (operation.getOperationId() != null) {
// v2 任务必须有归属:缺 operationId 或与当前世代不符都不能改当前任务,
// 否则「删除后重建」的新世代会被旧上报污染。
if (expectedOperationId == null || !operation.getOperationId().equals(expectedOperationId)) {
log.info("丢弃无法归属/旧世代的迟到上报 gid={} 上报={} 当前={}",
task.getGid(), expectedOperationId, operation.getOperationId());
continue;
}
}
String status = TaskStatus.fromReport(task.getStatus());
if (status == null) {
log.warn("未知上报状态码 gid={} status={}", task.getGid(), task.getStatus());
continue;
}
gallery.setProceeding(task.getProceeding());
if (task.getName() != null)
gallery.setName(task.getName());
boolean justCompleted = TaskStatus.COMPLETE.equals(status)
&& !TaskStatus.COMPLETE.equals(gallery.getStatus());
gallery.setStatus(status);
galleryMapper.updateGallery(gallery);
taskOperationRepository.updateReportReceipt(task.getGid(), revision, report.getReportId(), now());
acceptedTasks.add(task);
if (justCompleted)
newlyCompleted.add(gallery);
}
return new Applied(newlyCompleted, acceptedTasks);
}
/**
* 为迁移前创建、尚无操作记录的历史任务建立一次「世代」:优先采纳上报里的 operationId;
* 旧节点不带 operationId 时保留 null,让后续同世代的进度上报仍能被接受。
*
* <p>不在这里凭空生成随机 ID:那会让第一份进度上报之后的所有旧上报都因「无法归属」被丢弃。
* 真正的稳定 operationId 会在首次自动下发时用 {@code ensureStableOperationId} 落库,
* 之后该世代即可被严格匹配。不会覆盖任何已有任务状态。
*/
private TaskOperation adoptGeneration(int gid, String reportedOperationId) {
TaskOperation operation = new TaskOperation();
operation.setGid(gid);
operation.setOperationId(reportedOperationId);
operation.setState(TaskOperation.STATE_SUBMITTED);
operation.setUpdateTime(now());
taskOperationRepository.insert(operation);
return operation;
}
private static long now() {
return System.currentTimeMillis() / 1000;
}
/** 供装配层解析当前有效 operationId(避免装配代码访问包私有字段)。 */
public String currentOperationId(int gid) {
TaskOperation operation = taskOperationRepository.selectByGid(gid);
return operation == null ? null : operation.getOperationId();
}
}
@@ -54,35 +54,16 @@ public class QueryService {
}
Document parse = Jsoup.parse(result);
Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr");
ArrayList<GalleryForQuery> galleries = new ArrayList<>();
ListIterator<Element> elementListIterator = elements.listIterator();
java.util.List<GalleryForQuery> galleries;
try {
elementListIterator.next(); //skip firstlink
}catch (NullPointerException | NoSuchElementException e){
response.failure("没有搜索到结果");
return response.toJSONString();
galleries = com.lion.lionwebsite.Util.GallerySearchParser.parse(parse);
} catch (IOException | RuntimeException error) {
log.warn("上游搜索页面格式不支持: {}", error.getClass().getSimpleName());
return response.failure("上游搜索页面格式不支持").toJSONString();
}
if (galleries.isEmpty()) return response.failure("没有搜索到结果").toJSONString();
while (elementListIterator.hasNext()){
Element element = elementListIterator.next();
GalleryForQuery gallery = new GalleryForQuery();
gallery.setType(element.child(0).text()); //type
String src = element.child(1).select("img").attr("src");
if (src.startsWith("data"))
src = element.child(1).select("img").attr("data-src");
src = src.replace("https://s.exhentai.org", "");
gallery.setThumbnailUrl(src); //thumbnailSrc
gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime
gallery.setLink(element.child(2).child(0).attr("href")); //link
gallery.setGid(GalleryUtil.parseGid(gallery.getLink()) + ""); //gid
gallery.setName(element.child(2).child(0).child(0).text()); //name
gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page
galleries.add(gallery);
}
response.success(objectMapper.valueToTree(galleries).toString());
response.success(objectMapper.valueToTree(galleries));
Elements nextLink = parse.select("#unext");
if(nextLink.hasAttr("href"))
response.set("next", nextLink.attr("href"));
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.List;
@Service
@Data
@@ -76,12 +77,131 @@ public class RemoteService {
final SubscriptionStandbySnapshotService subscriptionStandbySnapshotService;
/**
* 协议 v2 上报处理器。用 setter 注入而不是构造参数,避免 RemoteService 与
* GalleryTaskLifecycleService 之间的依赖环(后者需要 RemoteService 下发命令)。
*/
volatile TaskReportHandler taskReportHandler;
/**
* gid → 当前有效的下载 operationId。由生命周期服务在装配期注册(保持依赖方向单向)。
* 未注册或返回 null 时,重投/重试退化为旧的无 operationId 命令。
*/
volatile java.util.function.IntFunction<String> downloadOperationIdResolver;
/**
* gid → 下发判定(是否允许下发 + 用哪个稳定 operationId)。
*
* <p>替代「能解析出 operationId 就下发」的旧语义:UNKNOWN / 预占中 / 待删除的任务
* 必须被明确拒绝,否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。
*/
volatile java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> dispatchResolver;
/**
* 生命周期装配是否已经完成(handler/resolver 已绑定 + 启动恢复已执行)。
*
* <p>在它为 true 之前,绝不自动重投未完成任务:启动早期还不知道哪些任务是
* SUBMITTED、哪些是 UNKNOWN,盲目下发会制造未计量副作用。
*/
volatile boolean lifecycleReady;
/**
* 启动恢复是否安全完成。恢复失败时拒绝自动重投,避免在状态不明的库上继续派发。
*/
volatile boolean recoverySafe;
/**
* 节点是否声明了 v2 可靠任务能力。只有协商成功之后才发送类型 11..13,
* 否则继续走旧的无确认上报,保持向后兼容。
*/
volatile boolean v2Negotiated;
/** 本次连接的 Hello 协商结果;收到节点 Hello(或超时)后完成。 */
volatile CompletableFuture<Boolean> helloNegotiation;
/** 明确「未下发」的结果码:与 -1(发送失败/超时)区分开。 */
public static final byte REFUSED = -3;
/** 事务提交之后的通知/推送执行器:外部延迟(Telegram、慢浏览器)不得阻塞状态提交。 */
final ExecutorService notificationExecutor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r, "gallery-notify");
thread.setDaemon(true);
return thread;
});
/**
* 协议握手的专用线程。hello 等待响应是阻塞调用,绝不能放在 Netty 事件循环上,
* 否则响应本身要由同一个事件循环投递,会直接死锁。
*/
final ExecutorService handshakeExecutor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r, "node-protocol-handshake");
thread.setDaemon(true);
return thread;
});
final ExecutorService subscriptionSyncExecutor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r, "subscription-standby-sync");
thread.setDaemon(true);
return thread;
});
/** 测试/装配用:绑定上报处理器。 */
public void setTaskReportHandler(TaskReportHandler taskReportHandler) {
this.taskReportHandler = taskReportHandler;
}
public void setDownloadOperationIdResolver(java.util.function.IntFunction<String> resolver) {
this.downloadOperationIdResolver = resolver;
}
public void setDispatchResolver(java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> resolver) {
this.dispatchResolver = resolver;
}
/**
* 由装配层在「绑定 handler/resolver + 完成启动恢复」之后调用:标记可以安全派发并建立连接。
*
* <p>初始连接不再放在 {@code @PostConstruct}:那时 handler/resolver 都还没绑定,
* 连接成功会把尚未确认的 UNKNOWN/预占任务当作普通任务重投。
*
* @param recoverySafe 启动恢复是否成功;false 时只允许连接,不允许自动重投
*/
public void startAfterWiring(boolean recoverySafe) {
this.recoverySafe = recoverySafe;
this.lifecycleReady = true;
initChannel();
}
/** 把 gid 解析成下发判定;未注册解析器或解析失败时按「拒绝下发」处理。 */
private GalleryTaskLifecycleService.DispatchDecision resolveDispatchDecision(int gid) {
java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> resolver = dispatchResolver;
if (resolver == null)
return null;
try {
return resolver.apply(gid);
} catch (Exception e) {
log.warn("解析任务下发判定失败 gid={}", gid, e);
return null;
}
}
private String resolveDownloadOperationId(int gid) {
java.util.function.IntFunction<String> resolver = downloadOperationIdResolver;
if (resolver == null)
return null;
try {
return resolver.apply(gid);
} catch (Exception e) {
log.warn("解析任务 operationId 失败 gid={}", gid, e);
return null;
}
}
/** 节点是否已完成 v2 能力协商(供健康检查/测试观察)。 */
public boolean isReliableProtocolReady() {
return v2Negotiated;
}
/**
* 处理节点上报的任务状态。原先这些 JDBC 读写直接跑在 Netty 的 IO 线程上,
* N 条上报会阻塞该 IO 线程,连带拖慢心跳与请求响应;改为单线程顺序执行,
@@ -120,10 +240,14 @@ public class RemoteService {
@Value("${subscription.standby.sync-enabled:false}")
boolean subscriptionSyncEnabled;
@PostConstruct
void init() {
initChannel();
}
/**
* 这里刻意不再自动连接。
*
* <p>原实现在 PostConstruct 里立即 initChannel(),而那一刻 TaskLifecycleWiring
* 还没绑定上报处理器、operationId 解析器,也没执行启动恢复:连接成功后会马上把
* 未完成任务(含 UNKNOWN/预占)当作普通任务重投给节点。
* 现在改为由装配层在完成绑定与恢复后调用 startAfterWiring(boolean)。
*/
public boolean initChannel(){
if (stopping || !connecting.compareAndSet(false, true))
@@ -175,12 +299,32 @@ public class RemoteService {
channel = channelFuture.channel();
closeMonitorSocket();
channel.writeAndFlush(new IdentityMessage("lionwebsite"));
//子节点上线时,发送未完成的任务
resetUndone();
// 节点刚上线时可能没有任何快照(例如刚重启),必须推一次;
// 这同时会刷新节点「最近收到快照」的时刻,避免内容未变时被判过期。
requestSubscriptionSync();
// 协议 v2:先发旧的 Identity,再发 hello 做能力协商。
// 协商完成之前不允许下发任何任务,也不允许自动重投未完成任务。
CompletableFuture<Boolean> negotiation = queueHelloHandshake();
// 在协商线程上等待结果,再决定重投;绝不阻塞 Netty 事件循环。
handshakeExecutor.execute(() -> {
boolean reliable = awaitNegotiation(negotiation);
// 订阅快照与任务重投都要求生命周期装配已完成;装配前只保持连接。
requestSubscriptionSync();
if (!reliable) {
log.info("节点未确认 v2 能力,跳过自动重投未完成任务");
return;
}
if (!lifecycleReady) {
log.info("生命周期装配未就绪,跳过自动重投未完成任务");
return;
}
if (!recoverySafe) {
log.error("启动恢复未安全完成,拒绝自动重投未完成任务");
return;
}
try {
resetUndone();
} catch (RuntimeException e) {
log.warn("重投未完成任务失败", e);
}
});
return true;
}catch (Exception e){
log.error("connect node failed, wait for node back online", e);
@@ -297,6 +441,8 @@ public class RemoteService {
failPendingRequests();
if (channel != null) channel.close();
statusApplyExecutor.shutdownNow();
notificationExecutor.shutdownNow();
handshakeExecutor.shutdownNow();
subscriptionSyncExecutor.shutdownNow();
networkGroup.shutdownGracefully();
eventLoopGroup.shutdownGracefully();
@@ -315,7 +461,11 @@ public class RemoteService {
}
public void resetUndone(){
if (channelFuture.channel() == null || !channelFuture.channel().isActive())
// 只有「连接在线 + v2 已协商 + 生命周期装配完成 + 启动恢复安全完成」才允许批量重投;
// 任何一项不满足都说明还不知道任务真实状态,盲目下发会制造未计量副作用。
if (channelFuture == null || channelFuture.channel() == null || !channelFuture.channel().isActive())
return;
if (!v2Negotiated || !lifecycleReady || !recoverySafe)
return;
Gallery[] galleries = galleryMapper.selectUnDoneGalleries();
@@ -328,15 +478,135 @@ public class RemoteService {
}
public byte addGalleryToQueue(Gallery gallery){
GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid());
if (decision != null) {
// 明确拒绝(UNKNOWN/预占/待删除等):绝不能退化为无 operationId 的旧命令。
if (!decision.dispatchable())
return REFUSED;
return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(),
gallery.getGid(), gallery.getName());
}
// 兼容:装配层未注册判定解析器时(如单元测试),退回只按 operationId 解析的旧行为。
String operationId = resolveDownloadOperationId(gallery.getGid());
if (operationId != null)
return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, operationId, gallery.getGid(), gallery.getName());
return legacyDownloadCommand(gallery.getGid(), gallery.getName());
}
/** 旧的无 operationId 下载命令,供未启用 v2 或无法解析 operationId 时退避使用。 */
private byte legacyDownloadCommand(int gid, String name) {
GalleryTask galleryTask = new GalleryTask();
galleryTask.setGid(gallery.getGid());
galleryTask.setName(gallery.getName());
galleryTask.setGid(gid);
galleryTask.setName(name);
DownloadPostMessage message = new DownloadPostMessage();
message.setGalleryTask(galleryTask);
return sendRequest(message, 10, TimeUnit.SECONDS);
}
/**
* 在专用线程上执行 hello 协商,并返回一个在「收到节点 Hello」或「等待超时」后完成的 future。
*
* <p>节点的应答是 Hello 而不是 ResponseMessage,因此绝不能走 sendRequest 的响应等待器
* (那会白等 10 秒超时,排在后面的自动重投也只能 10 秒后才开始)。
* 这里直接 writeAndFlush,并让 channelRead 的 Hello 分支完成 future;
* 超时用 completeOnTimeout 处理,写失败立刻完成。
*/
private CompletableFuture<Boolean> queueHelloHandshake() {
// 10 秒内没等到节点 Hello 就按「未协商」完成,避免后续重投被无限期挂起。
CompletableFuture<Boolean> result = new CompletableFuture<Boolean>()
.completeOnTimeout(false, 10, TimeUnit.SECONDS);
helloNegotiation = result;
if (stopping) {
result.complete(false);
return result;
}
try {
handshakeExecutor.execute(() -> {
v2Negotiated = false;
try {
Channel target = channel;
if (target == null || !target.isActive()) {
result.complete(false);
return;
}
// Hello 不写入 promiseHashMap:节点回的是 Hello,没有 ResponseMessage 等待者。
target.writeAndFlush(new ProtocolHelloMessage()).addListener(future -> {
if (!future.isSuccess()) {
if (!stopping)
log.warn("协议协商发送失败: {}", future.cause() == null
? "unknown" : future.cause().getClass().getSimpleName());
result.complete(false);
}
});
} catch (RuntimeException e) {
log.warn("协议协商失败", e);
result.complete(false);
}
});
} catch (java.util.concurrent.RejectedExecutionException e) {
if (!stopping) log.warn("安排协议协商失败", e);
result.complete(false);
}
return result;
}
/** 等待协商结果;超时或中断都按「未协商」处理(宁可不下发,也不盲目重投)。 */
boolean awaitNegotiation(CompletableFuture<Boolean> negotiation) {
if (negotiation == null)
return v2Negotiated;
try {
// future 自身已在 10 秒时按「未协商」完成,这里留少量余量即可。
return negotiation.get(12, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (Exception e) {
return false;
}
}
/**
* 下发一条带稳定 operationId 的任务命令;未协商 v2 时退化为旧的 DownloadPostMessage,
* 保证新旧节点组合仍可用(旧节点上无法保证可靠 ACK,这是已知的兼容边界)。
*/
public byte sendTaskCommand(String action, String operationId, int gid, String name) {
if (!v2Negotiated)
return TaskCommandMessage.ACTION_DELETE.equals(action)
? legacyDeleteCommand(gid, name)
: legacyDownloadCommand(gid, name);
TaskCommandMessage command = new TaskCommandMessage();
command.setOperationId(operationId);
command.setAction(action);
command.setGid(gid);
command.setName(name);
return sendRequest(command, 10, TimeUnit.SECONDS);
}
/** 删除命令:带独立稳定 operationId,重试复用同一个 ID。 */
public byte sendDeleteCommand(String operationId, int gid, String name) {
if (!v2Negotiated)
return legacyDeleteCommand(gid, name);
TaskCommandMessage command = new TaskCommandMessage();
command.setOperationId(operationId);
command.setAction(TaskCommandMessage.ACTION_DELETE);
command.setGid(gid);
command.setName(name);
return sendRequest(command, 10, TimeUnit.SECONDS);
}
/** 旧的无 operationId 删除命令,供未协商 v2 时退避使用。 */
private byte legacyDeleteCommand(int gid, String name) {
Gallery gallery = new Gallery();
gallery.setGid(gid);
gallery.setName(name);
return deleteGallery(gallery);
}
public RetryResult retryGallery(Gallery gallery){
// 提交结果未知或被删除的任务不能重试:节点可能已经把未计量任务当成新下载。
GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid());
if (decision != null && !decision.dispatchable())
return new RetryResult(false, "该任务提交结果未知或正在删除,需要人工对账后再重试");
CompletableFuture<String> statusFuture = new CompletableFuture<>();
retryStatusWaiters.computeIfAbsent(gallery.getGid(), ignored -> new CopyOnWriteArrayList<>())
.add(statusFuture);
@@ -447,6 +717,18 @@ public class RemoteService {
}
});
}
else if(msg instanceof ProtocolHelloMessage hello) {
// 节点声明了 v2 可靠任务能力;此后才允许使用类型 11..13。
v2Negotiated = hello.supportsReliableTasks();
log.info("节点协议协商完成 v2={} version={}", v2Negotiated, hello.getProtocolVersion());
CompletableFuture<Boolean> negotiation = helloNegotiation;
if (negotiation != null)
negotiation.complete(v2Negotiated);
}
else if(msg instanceof TaskReportMessage report) {
// 事务提交与 ACK 都放在单线程消费队列里顺序执行,避免并发写库。
statusApplyExecutor.execute(() -> consumeV2Report(report));
}
else if(msg instanceof ResponseMessage rsm) {
Promise<AbstractMessage> promise = promiseHashMap.remove(rsm.messageId);
if(promise != null)
@@ -456,6 +738,55 @@ public class RemoteService {
}
}
/**
* 消费协议 v2 上报:短事务提交 → 事务成功后回 ACK → 再用独立执行器做通知/推送。
*
* <p>顺序很关键:数据库失败时既不能回 ACK,也不能清任务;慢通知不能阻塞这里,
* 否则会拖住后续上报的提交。
*/
private void consumeV2Report(TaskReportMessage report) {
try {
TaskReportHandler handler = taskReportHandler;
if (handler == null) {
log.warn("收到 v2 上报但未装配处理器,暂不回执 reportId={}", report.getReportId());
return;
}
TaskReportHandler.CommittedReport committed = handler.apply(report);
if (committed == null) {
// 提交失败:不回 ACK,节点保留快照后重投。
log.warn("v2 上报未提交,等待节点重投 reportId={}", report.getReportId());
return;
}
// 提交成功之后才回执,让节点清理快照。
Channel target = channel;
if (target != null && target.isActive())
target.writeAndFlush(new TaskReportAckMessage(committed.reportId(), TaskReportAckMessage.RESULT_SUCCESS));
dispatchNotifications(report, committed);
} catch (Exception e) {
log.warn("处理 v2 上报失败 reportId={}", report.getReportId(), e);
}
}
/** 通知/推送在独立执行器里执行,外部延迟不会阻塞状态提交。 */
private void dispatchNotifications(TaskReportMessage report, TaskReportHandler.CommittedReport committed) {
// 只推送事务里真正被接受的更新(含下载中/压缩中等中间状态);
// 过期、旧世代、待删除任务已被过滤掉,不会把它们带到前端。
GalleryTask[] accepted = committed.acceptedTasks() == null
? new GalleryTask[0]
: committed.acceptedTasks().toArray(new GalleryTask[0]);
List<Gallery> newlyCompleted = committed.newlyCompleted();
notificationExecutor.execute(() -> {
try {
for (Gallery gallery : newlyCompleted)
pushService.downloadComplete(gallery);
if (accepted.length > 0)
webSocketService.updateTaskProcessing(accepted);
} catch (Exception e) {
log.warn("任务完成通知/推送失败", e);
}
});
}
/** 把节点上报的任务状态写回库里并推送给前端,按上报顺序逐个处理。 */
private void applyReportedStatus(GalleryTask[] galleryTasks) {
for (GalleryTask galleryTask : galleryTasks) {
@@ -17,6 +17,10 @@ import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.transaction.support.TransactionTemplate;
import jakarta.annotation.PreDestroy;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -39,6 +43,10 @@ import java.util.function.Supplier;
final RemoteService remoteService;
final SubscriptionStateCoordinator stateCoordinator;
@Autowired
@Qualifier("mainTransactionTemplate")
TransactionTemplate transactions;
/**
* 归属地查询是同步外呼(最长十几秒),且失败只影响记录里的一列展示。
* 若放在订阅分发路径上,用户取一次订阅就得先等它完成,因此改为后台补齐。
@@ -48,11 +56,12 @@ import java.util.function.Supplier;
Executor locationExecutor = newFixedLocationExecutor();
private static ExecutorService newFixedLocationExecutor() {
return Executors.newFixedThreadPool(2, r -> {
return new java.util.concurrent.ThreadPoolExecutor(2, 2, 0, java.util.concurrent.TimeUnit.MILLISECONDS,
new java.util.concurrent.ArrayBlockingQueue<>(128), r -> {
Thread thread = new Thread(r, "sub-location-resolver");
thread.setDaemon(true);
return thread;
});
}, new java.util.concurrent.ThreadPoolExecutor.AbortPolicy());
}
public String insertSubscriptionAccount(String name, String upstreamKey, boolean filterHighMultiplier, boolean enabled) {
@@ -211,6 +220,16 @@ import java.util.function.Supplier;
}
public void updateSub(HttpServletResponse response, HttpServletRequest request, String client, String key) {
Lock lock = stateCoordinator.readLock();
lock.lock();
try {
distributeSubscription(response, request, client, key);
} finally {
lock.unlock();
}
}
private void distributeSubscription(HttpServletResponse response, HttpServletRequest request, String client, String key) {
if (key == null || client == null)
return;
SubBind subBind = subMapper.selectSubBind(key);
@@ -221,7 +240,7 @@ import java.util.function.Supplier;
String ip = resolveClientIp(request);
String ua = request.getHeader("User-Agent");
if (ua == null) return;
recordUpdate(subBind.getUser(), ip, ua);
recordUpdate(subBind, ip, ua);
if (!"v2".equals(client) && !"cat".equals(client)) {
sendStatus(response, HttpServletResponse.SC_BAD_REQUEST, "client error");
return;
@@ -259,24 +278,34 @@ import java.util.function.Supplier;
|| "0:0:0:0:0:0:0:1".equals(address);
}
private void recordUpdate(String user, String ip, String ua) {
private void recordUpdate(SubBind binding, String ip, String ua) {
String user = binding.getUser();
// 先落一条 unknown 记录并立即返回,归属地随后在后台补齐;
// 这样取订阅的延迟不再取决于第三方定位站点的响应时间。
subMapper.insertSubUpdateRecord(new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown"));
SubUpdateRecord record = new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown");
record.setUserId(binding.getUserId());
subMapper.insertSubUpdateRecord(record);
if (subMapper.selectUpdateRecordCount(user) > 10)
subMapper.deleteSubUpdateRecordById(subMapper.selectMinUpdateRecordId(user));
locationExecutor.execute(() -> enrichLocation(user, ip));
// Capture the generated row ID: another request may arrive before the lookup finishes.
if (record.getId() > 0) {
try {
locationExecutor.execute(() -> enrichLocation(record.getId(), ip));
} catch (java.util.concurrent.RejectedExecutionException busy) {
log.debug("位置查询队列已满,保留 unknown recordId={}", record.getId());
}
}
}
/** 后台补齐最近一条记录的归属地;任何失败都只记日志,不影响已落库的记录。 */
private void enrichLocation(String user, String ip) {
/** 后台按记录 ID 补齐归属地;任何失败都只记日志,不影响已落库的记录。 */
private void enrichLocation(int recordId, String ip) {
try {
String location = resolveLocation(ip);
if (location == null || location.isBlank() || "unknown".equals(location))
return;
subMapper.updateLatestSubUpdateRecordLocation(user, location);
subMapper.updateSubUpdateRecordLocation(recordId, location);
} catch (Exception e) {
log.debug("补齐订阅更新记录归属地失败 user={}", user, e);
log.debug("补齐订阅更新记录归属地失败 recordId={}", recordId, e);
}
}
@@ -310,20 +339,26 @@ import java.util.function.Supplier;
Lock stateLock = stateCoordinator.writeLock();
stateLock.lock();
try {
return action.get();
return transactions == null ? action.get() : transactions.execute(ignored -> action.get());
} finally {
stateLock.unlock();
}
}
@PreDestroy
void shutdownLocationResolver() {
if (locationExecutor instanceof ExecutorService service)
service.shutdownNow();
}
private SubscriptionAccount firstEnabledAccount() {
return subMapper.selectAllSubscriptionAccounts().stream().filter(SubscriptionAccount::isEnabled).findFirst().orElse(null);
}
private String accountJson(SubscriptionAccount account) {
private tools.jackson.databind.JsonNode accountJson(SubscriptionAccount account) {
account.setV2Url(refreshService.v2Url(account));
account.setClashUrl(refreshService.clashUrl(account));
return CustomUtil.objectMapper.valueToTree(account).toString();
return CustomUtil.objectMapper.valueToTree(account);
}
private static void sendStatus(HttpServletResponse response, int status, String message) {
@@ -2,7 +2,9 @@ package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Dao.normal.SubMapper;
import com.lion.lionwebsite.Domain.SubscriptionAccount;
import com.lion.lionwebsite.Util.ClashSubscriptionParser;
import com.lion.lionwebsite.Util.SubscriptionClientProfile;
import com.lion.lionwebsite.Util.SubscriptionCacheStore;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.classic.methods.HttpGet;
@@ -57,6 +59,12 @@ public class SubscriptionRefreshService {
/** 时间源,测试可替换以固定成功时刻。 */
Clock clock = Clock.systemDefaultZone();
/**
* 分代缓存存储。默认按 {@link #cacheRoot} 懒构造;测试可覆写为受控子类来模拟写入失败。
* 保持为普通字段而不是构造参数,是为了不改动既有两参构造器(Spring 与老测试都在用)。
*/
SubscriptionCacheStore cacheStore;
private long refreshSequence;
// Accessed only while holding the coordinator write lock.
private final Map<Integer, Long> latestRefresh = new HashMap<>();
@@ -78,18 +86,19 @@ public class SubscriptionRefreshService {
// Network access and parsing never hold the shared subscription lock.
try {
boolean filter = account.isFilterHighMultiplier();
String v2 = processV2(firstLine(download(v2Url(account), SubscriptionClientProfile.forAccount(accountId, false))),
account.isFilterHighMultiplier(), highMultiplierThreshold);
List<String> clash = processClash(download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)),
account.isFilterHighMultiplier(), highMultiplierThreshold);
filter, highMultiplierThreshold);
// 按 YAML 语义过滤高倍节点并同步清理 proxy-groups 引用,不再靠行缩进猜测。
String clash = ClashSubscriptionParser.transform(
download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)),
filter, highMultiplierThreshold);
stateLock.lock();
try {
if (!isCurrent(account, version))
return false;
Path dir = Paths.get(cacheRoot, String.valueOf(accountId));
Files.createDirectories(dir);
atomicWrite(dir.resolve("v2ray.txt"), v2.getBytes(StandardCharsets.UTF_8));
atomicWrite(dir.resolve("clash.yaml"), String.join("\n", clash).concat("\n").getBytes(StandardCharsets.UTF_8));
// 两份内容写进同一分代,校验通过后指针最后发布:任一步失败都保留上次完整版本。
store().publish(accountId, v2.getBytes(StandardCharsets.UTF_8), clash.getBytes(StandardCharsets.UTF_8));
subMapper.markSubscriptionRefreshSuccess(accountId, clock.millis());
return true;
} finally {
@@ -162,56 +171,6 @@ public class SubscriptionRefreshService {
return URLDecoder.decode(node.substring(hash + 1), StandardCharsets.UTF_8);
}
private List<String> processClash(List<String> source, boolean filter, double threshold) {
Set<String> removed = new HashSet<>();
List<String> result = new ArrayList<>();
boolean inProxies = false;
boolean skipNode = false;
for (String line : source) {
if (line.equals("proxies:")) {
inProxies = true;
skipNode = false;
result.add(line);
continue;
}
if (line.equals("proxy-groups:")) {
inProxies = false;
skipNode = false;
result.add(line);
continue;
}
if (inProxies && line.matches("^\\s{2}-\\s+name:.*")) {
String name = clashName(line);
skipNode = filter && isHigh(name, threshold);
if (skipNode)
removed.add(name);
else
result.add(line);
continue;
}
if (skipNode)
continue;
if (!inProxies && !removed.isEmpty() && line.trim().startsWith("- ")) {
String ref = line.trim().substring(2).trim();
if (removed.contains(unquote(ref)))
continue;
}
result.add(line);
}
return result;
}
private static String clashName(String line) {
int index = line.indexOf("name:");
return unquote(line.substring(index + 5).trim());
}
private static String unquote(String value) {
if (value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'"))))
return value.substring(1, value.length() - 1);
return value;
}
private static boolean isHigh(String name, double threshold) {
Matcher matcher = MULTIPLIER.matcher(name);
return matcher.find() && Double.parseDouble(matcher.group(1)) > threshold;
@@ -251,22 +210,17 @@ public class SubscriptionRefreshService {
}
}
private static void atomicWrite(Path target, byte[] data) throws IOException {
Path temp = target.resolveSibling(target.getFileName() + ".tmp");
Files.write(temp, data);
try {
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
}
}
/**
* 当前应分发的缓存文件。只在一整个版本有效时返回真实路径;否则返回一个保证不存在的
* 哨兵路径,使分发端按「订阅暂不可用」处理,而不是发出半份旧缓存。
*/
public Path cachedPath(Integer accountId, String client) {
return Paths.get(cacheRoot, String.valueOf(accountId), client.equals("v2") ? "v2ray.txt" : "clash.yaml");
SubscriptionCacheStore store = store();
return store.resolve(accountId, client).orElseGet(() -> store.unavailablePath(accountId, client));
}
public boolean hasCompleteCache(Integer accountId) {
return Files.isRegularFile(cachedPath(accountId, "v2")) && Files.isRegularFile(cachedPath(accountId, "cat"));
return store().hasComplete(accountId);
}
public void invalidateCache(Integer accountId) {
@@ -274,12 +228,15 @@ public class SubscriptionRefreshService {
stateLock.lock();
try {
latestRefresh.remove(accountId); // In-flight responses must not restore invalidated content.
Files.deleteIfExists(cachedPath(accountId, "v2"));
Files.deleteIfExists(cachedPath(accountId, "cat"));
} catch (IOException e) {
log.warn("清理失效订阅缓存失败 accountId={}", accountId, e);
store().invalidate(accountId);
} finally {
stateLock.unlock();
}
}
private synchronized SubscriptionCacheStore store() {
if (cacheStore == null)
cacheStore = new SubscriptionCacheStore(cacheRoot);
return cacheStore;
}
}
@@ -6,6 +6,7 @@ import com.lion.lionwebsite.Domain.SubBind;
import com.lion.lionwebsite.Domain.SubscriptionAccount;
import com.lion.lionwebsite.Message.*;
import com.lion.lionwebsite.Util.CustomUtil;
import com.lion.lionwebsite.Util.SubscriptionCacheStore;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -16,8 +17,6 @@ import javax.crypto.spec.SecretKeySpec;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.util.*;
import java.util.zip.GZIPOutputStream;
@@ -39,6 +38,12 @@ public class SubscriptionStandbySnapshotService {
@Value("${subscription.cache-root:sub/accounts}")
String cacheRoot;
/**
* 与 {@link SubscriptionRefreshService} 共用的分代缓存解析器。
* 测试可直接覆写;未设置时按 {@link #cacheRoot} 懒构造。
*/
SubscriptionCacheStore cacheStore;
@PostConstruct
void validateConfiguration() {
if (syncEnabled && (syncSecret == null || syncSecret.isBlank()))
@@ -51,12 +56,14 @@ public class SubscriptionStandbySnapshotService {
try {
Map<Integer, SubscriptionAccountSnapshot> accountMap = new HashMap<>();
for (SubscriptionAccount account : subMapper.selectAllSubscriptionAccounts()) {
Path v2Path = cachedPath(account.getId(), "v2");
Path clashPath = cachedPath(account.getId(), "cat");
if (!account.isEnabled() || !Files.isRegularFile(v2Path) || !Files.isRegularFile(clashPath))
if (!account.isEnabled())
continue;
byte[] v2 = Files.readAllBytes(v2Path);
byte[] clash = Files.readAllBytes(clashPath);
// 从同一分代读出成对内容,保证快照里两份文件同版本;无完整版本则跳过该账号。
Optional<SubscriptionCacheStore.Pair> pair = store().read(account.getId());
if (pair.isEmpty())
continue;
byte[] v2 = pair.get().v2();
byte[] clash = pair.get().clash();
SubscriptionAccountSnapshot snapshot = new SubscriptionAccountSnapshot();
snapshot.setAccountId(account.getId());
snapshot.setEnabled(true);
@@ -132,7 +139,9 @@ public class SubscriptionStandbySnapshotService {
private static String hex(byte[] input) { return HexFormat.of().formatHex(input); }
private Path cachedPath(Integer accountId, String client) {
return Path.of(cacheRoot, String.valueOf(accountId), "v2".equals(client) ? "v2ray.txt" : "clash.yaml");
private synchronized SubscriptionCacheStore store() {
if (cacheStore == null)
cacheStore = new SubscriptionCacheStore(cacheRoot);
return cacheStore;
}
}
@@ -0,0 +1,23 @@
package com.lion.lionwebsite.Service;
import java.util.UUID;
/** 协议 v2 的操作 ID 生成与解析。稳定 ID 是「节点重发/重试不重复执行副作用」的依据。 */
public final class TaskProtocol {
private TaskProtocol() {
}
/** 生成一个新的稳定操作 ID(下载或删除),形如 {@code op-<uuid>}。 */
public static String newOperationId(String action) {
return action + "-" + UUID.randomUUID();
}
public static String newDownloadOperationId() {
return newOperationId("dl");
}
public static String newDeleteOperationId() {
return newOperationId("del");
}
}
@@ -0,0 +1,37 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Domain.Gallery;
import com.lion.lionwebsite.Domain.GalleryTask;
import com.lion.lionwebsite.Message.TaskReportMessage;
import java.util.List;
/**
* 协议 v2 状态上报的消费入口。
*
* <p>定义成接口是为了断开依赖环:RemoteService 需要把上报交给生命周期服务,
* 而生命周期服务需要有 RemoteService 才能下发命令。RemoteService 通过可选的 setter
* 注入本接口(见 {@code RemoteService#setTaskReportHandler}),因此构造期没有环。
*/
public interface TaskReportHandler {
/**
* 提交一份上报。
*
* @return 已成功提交(可回 ACK)时返回提交结果;数据库失败等「不可回 ACK」的情况返回 null。
* 返回 null 时节点会保留快照并稍后重投,这正是「失败写入不得清任务」的要求。
*/
CommittedReport apply(TaskReportMessage report);
/**
* 已提交的结果。
*
* @param reportId 本次上报 ID(回执用)
* @param newlyCompleted 本次「首次」完成的任务,通知与 WebSocket 由独立执行器在事务之后处理,
* 因此慢通知不会阻塞数据库消费线程;重复上报不会重复出现在这里
* @param acceptedTasks 事务内实际被接受的进度更新(含下载中/压缩中等中间状态,已排除过期、
* 旧世代与待删除任务),WebSocket 只推这一批,避免前端看到被丢弃的迟到状态
*/
record CommittedReport(String reportId, List<Gallery> newlyCompleted, List<GalleryTask> acceptedTasks) {
}
}
@@ -0,0 +1,45 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Domain.GalleryTask;
/**
* 任务业务状态码到展示文案的唯一映射点。
*
* <p>状态以稳定代码表达,中文文案只出现在展示层(WebSocket/列表),
* 避免把「失败/待重试/结果未知」塞进成功失败提示字符串。
*/
public final class TaskStatus {
public static final String SUBMITTED = "已提交";
/** 已登记创建意图、尚未确认上游提交;用于让任务列表在提交期间也能看到占位任务。 */
public static final String PENDING_SUBMIT = "提交中";
public static final String DOWNLOADING = "下载中";
public static final String WAITING_COMPRESS = "等待压缩";
public static final String COMPRESSING = "压缩中";
public static final String COMPLETE = "下载完成";
public static final String UNKNOWN = "提交未知";
public static final String DELETE_PENDING = "待删除";
public static final String DELETE_FAILED = "删除失败";
private TaskStatus() {
}
/**
* 节点上报的进度状态映射为业务文案;未知状态码返回 null,由调用方忽略。
*
* <p>GalleryTask 里的状态码是可变 static byte,不是编译期常量,因此不能用 switch case;
* 这里用整数比较,避免为改常量引入风险。
*/
public static String fromReport(byte reportStatus) {
int status = reportStatus;
if (status == GalleryTask.DOWNLOADING)
return DOWNLOADING;
if (status == GalleryTask.DOWNLOAD_COMPLETE)
return WAITING_COMPRESS;
if (status == GalleryTask.COMPRESSING)
return COMPRESSING;
if (status == GalleryTask.COMPRESS_COMPLETE)
return COMPLETE;
return null;
}
}
@@ -0,0 +1,47 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Domain.Gallery;
import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
import com.lion.lionwebsite.Util.GalleryUtil;
import com.lion.lionwebsite.Util.PreparedGallery;
import org.springframework.stereotype.Component;
import java.io.IOException;
/**
* 上游图站的读写边界。
*
* <p>分成两步是协议 v2 的核心约束:
* <ol>
* <li>{@link #prepare} 只做读取(页面、分辨率),绝不产生下载副作用;</li>
* <li>调用方必须在持久化 SUBMITTING 之后才调用 {@link #submit},因为那一步才真正 POST。</li>
* </ol>
*
* <p>拆成接口是为了让生命周期服务可以在测试里注入假实现,也方便后续把抓取/解析/提交
* 进一步分开;默认实现 {@link GalleryUtilAdapter} 直接转发到现有 GalleryUtil。
*/
public interface UpstreamGateway {
/** 只读解析:失败返回 null 表示链接无效或页面不可用,不产生任何副作用。 */
PreparedGallery prepare(String link) throws IOException;
/** 真正提交下载请求。抛异常表示结果未知,由调用方落 UNKNOWN,不做自动重发。 */
Gallery submit(PreparedGallery prepared, String targetResolution)
throws IOException, ResolutionNotMatchException;
/** 默认实现:转发现有 GalleryUtil 的拆分方法。 */
@Component
class GalleryUtilAdapter implements UpstreamGateway {
@Override
public PreparedGallery prepare(String link) throws IOException {
return GalleryUtil.prepare(link);
}
@Override
public Gallery submit(PreparedGallery prepared, String targetResolution)
throws IOException, ResolutionNotMatchException {
return GalleryUtil.submit(prepared, targetResolution);
}
}
}
@@ -3,6 +3,7 @@ package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Dao.normal.CollectMapper;
import com.lion.lionwebsite.Dao.normal.GalleryMapper;
import com.lion.lionwebsite.Dao.normal.UserMapper;
import com.lion.lionwebsite.Dao.normal.SubMapper;
import com.lion.lionwebsite.Domain.User;
import com.lion.lionwebsite.Interceptor.TaskHandlerInterceptor;
import com.lion.lionwebsite.Util.CustomUtil;
@@ -10,8 +11,13 @@ import com.lion.lionwebsite.Util.Response;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.transaction.support.TransactionTemplate;
import java.util.ArrayList;
import java.util.concurrent.locks.Lock;
import java.util.function.Supplier;
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
@@ -28,6 +34,11 @@ public class UserService{
final TaskHandlerInterceptor taskHandlerInterceptor;
@Autowired SubMapper subMapper;
@Autowired SubscriptionStateCoordinator subscriptionState;
@Autowired RemoteService remoteService;
@Autowired @Qualifier("mainTransactionTemplate") TransactionTemplate transactions;
public String addAuthCode(String targetAuthCode, String people) {
Response response = Response.generateResponse();
@@ -66,12 +77,22 @@ public class UserService{
Response response = Response.generateResponse();
try{
if(userMapper.isExist(targetAuthCode) != 0){
userMapper.updateUsername(targetAuthCode, newUsername);
response.success("修改成功");
}else {
response.failure("授权码不存在");
}
return coordinated(() -> {
if (userMapper.isExist(targetAuthCode) == 0)
return response.failure("授权码不存在").toJSONString();
if (newUsername == null || newUsername.isBlank())
return response.failure("用户名不能为空").toJSONString();
String name = newUsername.trim();
User current = userMapper.selectUserByAuthCode(targetAuthCode);
User existing = userMapper.selectUserByUsername(name);
if (existing != null && (current == null || existing.getId() != current.getId()))
return response.failure("用户名已存在").toJSONString();
userMapper.updateUsername(targetAuthCode, name);
if (subMapper != null && current != null)
subMapper.renameBoundUser(current.getId(), current.getUsername(), name);
if (remoteService != null) remoteService.requestSubscriptionSync();
return response.success("修改成功").toJSONString();
});
}catch (Exception e){
log.error("修改用户名失败", e);
response.failure("修改失败");
@@ -85,17 +106,22 @@ public class UserService{
if(userMapper.isExist(targetAuthCode) != 0)
try{
User user = userMapper.selectUserByAuthCode(targetAuthCode);
//修改下载人以及取消收藏
ArrayList<Integer> galleries = collectMapper.selectGidByCollector(user.getId());
for(Integer gallery: galleries)
collectMapper.disCollect(gallery, user.getId());
galleryMapper.updateGalleryDownloader(3, user.getId());
userMapper.deleteUserByAuthCode(targetAuthCode);
coordinated(() -> {
User user = userMapper.selectUserByAuthCode(targetAuthCode);
ArrayList<Integer> galleries = collectMapper.selectGidByCollector(user.getId());
for(Integer gallery: galleries)
collectMapper.disCollect(gallery, user.getId());
galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, user.getId());
if (subMapper != null) {
subMapper.deleteSubUpdateRecord(user.getUsername());
subMapper.deleteSubBind(user.getUsername());
}
userMapper.deleteUserByAuthCode(targetAuthCode);
return "deleted";
});
response.success("删除成功");
taskHandlerInterceptor.updateAuthCodes();
if (remoteService != null) remoteService.requestSubscriptionSync();
}catch (Exception e){
log.error("删除授权码失败", e);
response.failure("删除失败");
@@ -108,27 +134,29 @@ public class UserService{
}
public String alterStatus(String AuthCode, boolean isEnable){
Response response = Response.generateResponse();
if(userMapper.isExist(AuthCode) == 1){
int id = userMapper.selectUserByAuthCode(AuthCode).getId();
userMapper.updateIsEnableById(id, isEnable);
galleryMapper.updateGalleryDownloader(3, id);
response.success("修改用户状态成功");
taskHandlerInterceptor.updateAuthCodes();
try {
String result = coordinated(() -> {
if (userMapper.isExist(AuthCode) != 1)
return Response._failure("该用户不存在");
int id = userMapper.selectUserByAuthCode(AuthCode).getId();
userMapper.updateIsEnableById(id, isEnable);
if (!isEnable) galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, id);
return Response._success("修改用户状态成功");
});
// Refresh the in-memory identity view only after the database commits.
if (result.contains("\"result\":\"success\"")) taskHandlerInterceptor.updateAuthCodes();
return result;
} catch (RuntimeException error) {
log.warn("修改用户状态失败: {}", error.getClass().getSimpleName());
return Response._failure("修改用户状态失败");
}
else{
response.failure("该用户不存在");
}
return response.toJSONString();
}
public String getAllUser() {
Response response = Response.generateResponse();
User[] users = userMapper.selectAllUser();
response.success(objectMapper.valueToTree(users).toString());
response.success(objectMapper.valueToTree(users));
return response.toJSONString();
}
@@ -150,4 +178,14 @@ public class UserService{
User user = userMapper.selectUserByAuthCode(AuthCode);
return user != null && user.getId() == ADMIN_USER_ID;
}
private String coordinated(Supplier<String> action) {
Lock lock = subscriptionState == null ? null : subscriptionState.writeLock();
if (lock != null) lock.lock();
try {
return transactions == null ? action.get() : transactions.execute(ignored -> action.get());
} finally {
if (lock != null) lock.unlock();
}
}
}
@@ -0,0 +1,45 @@
package com.lion.lionwebsite.Util;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.DumperOptions;
import java.util.*;
import java.util.regex.Pattern;
/** YAML semantics rather than indentation determine nodes and references. */
public final class ClashSubscriptionParser {
private static final Pattern MULTIPLIER = Pattern.compile("(\\d+(?:\\.\\d+)?)x\\s*$", Pattern.CASE_INSENSITIVE);
private ClashSubscriptionParser() { }
public static String transform(List<String> lines, boolean filter, double threshold) {
Object loaded = new Yaml().load(String.join("\n", lines));
if (!(loaded instanceof Map<?, ?> source)) throw new IllegalArgumentException("Clash 上游必须为 YAML 对象");
if (!(source.get("proxies") instanceof List<?> proxies)) throw new IllegalArgumentException("Clash 上游缺少 proxies 列表");
Map<Object, Object> output = new LinkedHashMap<>(source);
Set<String> removed = new HashSet<>();
List<Object> kept = new ArrayList<>();
for (Object node : proxies) {
if (!(node instanceof Map<?, ?> fields) || !(fields.get("name") instanceof String name))
throw new IllegalArgumentException("Clash 节点缺少名称");
var multiplier = MULTIPLIER.matcher(name);
if (filter && multiplier.find() && Double.parseDouble(multiplier.group(1)) > threshold)
removed.add(name);
else kept.add(node);
}
output.put("proxies", kept);
if (source.get("proxy-groups") instanceof List<?> groups) {
List<Object> result = new ArrayList<>();
for (Object group : groups) {
if (!(group instanceof Map<?, ?> fields)) throw new IllegalArgumentException("Clash 分组必须为对象");
Map<Object, Object> copy = new LinkedHashMap<>(fields);
if (fields.get("proxies") instanceof List<?> references)
copy.put("proxies", references.stream().filter(ref -> !removed.contains(ref)).toList());
result.add(copy);
}
output.put("proxy-groups", result);
}
DumperOptions options = new DumperOptions();
options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
options.setPrettyFlow(true);
return new Yaml(options).dump(output);
}
}
@@ -0,0 +1,72 @@
package com.lion.lionwebsite.Util;
import com.lion.lionwebsite.Domain.Gallery;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
/** Pure parsing: upstream requests and download submission live outside this class. */
public final class GalleryPageParser {
private GalleryPageParser() { }
public static Gallery metadata(Document document, String link, int gid) throws IOException {
String title = document.select("#gn").text();
if (title.isBlank()) title = document.select("#gj").text();
if (title.isBlank()) throw new IOException("上游页面缺少画廊名称,可能是错误页");
Map<String, String> fields = new HashMap<>();
for (Element row : document.select("#gdd tr")) {
var cells = row.select("td");
if (cells.size() < 2) continue;
String name = cells.get(0).text().replace(":", "").strip().toLowerCase(Locale.ROOT);
fields.put(name, cells.get(1).text().strip());
}
var pages = Pattern.compile("\\d+").matcher(fields.getOrDefault("length", ""));
if (!pages.find()) throw new IOException("上游页面缺少画廊页数");
Gallery result = new Gallery();
result.setGid(gid); result.setLink(link); result.setName(title + " [" + gid + "]");
result.setCreateTime(System.currentTimeMillis() / 1000); result.setProceeding(0);
result.setLanguage(fields.getOrDefault("language", "").replace("\u00a0", " "));
try {
result.setPages(Integer.parseInt(pages.group()));
result.setFileSize(CustomUtil.stringToFileSize(fields.getOrDefault("file size", "").replace(" ", "")));
} catch (RuntimeException failure) { throw new IOException("上游画廊页数或文件大小格式不支持", failure); }
if (result.getPages() <= 0) throw new IOException("上游画廊页数必须为正数");
result.setDisplayFileSize(CustomUtil.fileSizeToString(result.getFileSize()));
Element thumbnail = document.selectFirst("#gd1 [style*=url]");
var image = Pattern.compile("url\\(\\s*['\"]?(.*?)['\"]?\\s*\\)").matcher(thumbnail == null ? "" : thumbnail.attr("style"));
String imageUrl = image.find() ? image.group(1) : document.select("#gd1 img").attr("src");
result.setThumb_link(imageUrl.replace("https://s.exhentai.org", ""));
return result;
}
public static String archiveLink(Document document) throws IOException {
for (Element anchor : document.select("#gd5 a")) {
var matcher = Pattern.compile("https?://[^'\"\\s]+").matcher(anchor.attr("onclick"));
if (matcher.find()) return matcher.group();
String href = anchor.absUrl("href");
if (!href.isBlank() && (anchor.text().toLowerCase(Locale.ROOT).contains("download") || href.contains("archive")))
return href;
}
throw new IOException("上游页面缺少归档入口");
}
public static Map<String, String> resolutions(Document document) throws IOException {
var expression = Pattern.compile("^(\\d+x|Original)\\s+([\\d.]+)\\s*([A-Za-z]+)$", Pattern.CASE_INSENSITIVE);
TreeMap<Integer, String> sizes = new TreeMap<>();
String original = null;
for (Element paragraph : document.select("#db td p")) {
var matcher = expression.matcher(paragraph.text().strip());
if (!matcher.matches()) continue;
String size = matcher.group(2) + matcher.group(3);
if (matcher.group(1).equalsIgnoreCase("Original")) original = size;
else sizes.put(Integer.parseInt(matcher.group(1).replace("x", "")), size);
}
Map<String, String> result = new LinkedHashMap<>();
sizes.forEach((width, size) -> result.put(width + "x", size));
if (original != null) result.put("Original", original);
if (result.isEmpty()) throw new IOException("上游页面没有可用分辨率");
return result;
}
}
@@ -0,0 +1,45 @@
package com.lion.lionwebsite.Util;
import com.lion.lionwebsite.Domain.GalleryForQuery;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
/** Search rows are located by gallery links, independent of table nesting and headings. */
public final class GallerySearchParser {
private GallerySearchParser() { }
public static List<GalleryForQuery> parse(Document document) throws IOException {
List<GalleryForQuery> result = new ArrayList<>();
for (Element row : document.select("tr")) {
Element link = row.selectFirst("a[href*=/g/]");
if (link == null) continue;
String href = link.attr("href");
Integer gid = GalleryUtil.parseGid(href);
if (gid == null) continue;
GalleryForQuery gallery = new GalleryForQuery();
gallery.setGid(gid.toString()); gallery.setLink(href);
gallery.setName(link.select(".glink").isEmpty() ? link.text() : link.select(".glink").text());
Element type = row.selectFirst(".cn, .cs");
gallery.setType(type == null ? row.select("td").first().text() : type.text());
Element image = row.selectFirst("img");
String thumbnail = image == null ? "" : image.attr("src");
if (thumbnail.isBlank() || thumbnail.startsWith("data:")) thumbnail = image == null ? "" : image.attr("data-src");
gallery.setThumbnailUrl(thumbnail.replace("https://s.exhentai.org", ""));
var pages = Pattern.compile("(\\d+)\\s+pages?", Pattern.CASE_INSENSITIVE).matcher(row.text());
if (!pages.find()) throw new IOException("上游搜索结果缺少页数 gid=" + gid);
gallery.setPage(Integer.parseInt(pages.group(1)));
Element date = row.selectFirst("[id^=posted_]");
if (date != null) gallery.setUploadTime(date.text());
else {
var dates = Pattern.compile("\\d{4}-\\d{2}-\\d{2}(?:\\s+\\d{2}:\\d{2})?").matcher(row.text());
gallery.setUploadTime(dates.find() ? dates.group() : "");
}
if (gallery.getName().isBlank()) throw new IOException("上游搜索结果缺少名称 gid=" + gid);
result.add(gallery);
}
return result;
}
}
@@ -82,6 +82,12 @@ public class GalleryUtil {
* @throws ResolutionNotMatchException 没有目标分辨率
*/
public static Gallery parse(String url, boolean isDownload, String targetResolution) throws IOException, ResolutionNotMatchException {
PreparedGallery prepared = prepare(url);
if (prepared == null) return null;
return isDownload ? submit(prepared, targetResolution) : prepared.gallery();
}
public static PreparedGallery prepare(String url) throws IOException {
String origin = verifyLink(url);
if(origin == null){
@@ -94,66 +100,26 @@ public class GalleryUtil {
log.warn("链接无法解析出 gid,按无效链接处理");
return null;
}
Gallery gallery = new Gallery();
gallery.setLink(url);
gallery.setCreateTime(System.currentTimeMillis()/1000);
gallery.setGid(gid);
gallery.setProceeding(0);
//访问图片页面
String galleryPage = requests(url, GET, null, null);
Document galleryDoc = Jsoup.parse(galleryPage);
//收集图片基本信息
gallery.setName(galleryDoc.select("#gn").text() + " [" + gallery.getGid() + ']');
gallery.setLanguage(galleryDoc.select("#gdd > table > tbody > tr:nth-child(4) > td.gdt2").text().replace("%nbps", ""));
gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0]));
gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", "")));
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
String thumb_link = galleryDoc.select("#gd1 > div").getFirst().attr("style");
gallery.setThumb_link(thumb_link.substring(thumb_link.indexOf("url(") + 4, thumb_link.lastIndexOf(")")).replace("https://s.exhentai.org", ""));
//查找下载页面链接并初始化参数
String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1];
Document galleryDoc = Jsoup.parse(requests(url, GET, null, null), url);
Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
HashMap<String, String> headers = new HashMap<>();
headers.put("origin", origin);
headers.put("referer", archiveLink);
Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
gallery.setAvailableResolution(GalleryPageParser.resolutions(downloadDoc));
gallery.setStatus("等待确认下载");
return new PreparedGallery(gallery, download_link, origin);
}
public static Gallery submit(PreparedGallery prepared, String targetResolution) throws IOException, ResolutionNotMatchException {
Gallery gallery = prepared.gallery();
String download_link = prepared.formUrl();
HashMap<String, String> headers = new HashMap<>();
headers.put("origin", prepared.origin());
headers.put("referer", download_link);
//访问下载页面,获取分辨率
String downloadPage = requests(download_link, GET, headers, null);
Document downloadDoc = Jsoup.parse(downloadPage);
Elements resolutions = downloadDoc.select("#db > div > table > tbody > tr > td");
download_link = downloadDoc.select("#hathdl_form").attr("action");
Map<String, String> availableResolution = new LinkedHashMap<>();
Map<String, String> tempResolutions = new LinkedHashMap<>();
ArrayList<Integer> tempResolutionKeys = new ArrayList<>();
for(Element element: resolutions){
String[] parameter = element.select("td > p").text().split(" ");
if(parameter[1].contains("N/A"))
continue;
tempResolutions.put(parameter[0], parameter[1] + parameter[2]);
if(parameter[0].contains("x")){
tempResolutionKeys.add(Integer.valueOf(parameter[0].replace("x", "")));
}
}
tempResolutionKeys.sort(Comparator.comparingInt(o -> o));
for(Integer resolution: tempResolutionKeys){
availableResolution.put(resolution.toString()+"x", tempResolutions.get(resolution+"x"));
}
availableResolution.put("Original", tempResolutions.get("Original"));
gallery.setAvailableResolution(availableResolution);
//如果不是下载,则直接返回
if(!isDownload){
gallery.setStatus("等待确认下载");
return gallery;
}
//如果目标分辨率不存在,抛出错误
if(!gallery.getAvailableResolution().containsKey(targetResolution)){
log.warn("目标分辨率不存在,可用分辨率: {}", gallery.getAvailableResolution());
@@ -161,15 +127,15 @@ public class GalleryUtil {
}
gallery.setResolution(targetResolution);
gallery.setFileSize(CustomUtil.stringToFileSize(availableResolution.get(targetResolution).replace(" ", "")));
gallery.setFileSize(CustomUtil.stringToFileSize(gallery.getAvailableResolution().get(targetResolution).replace(" ", "")));
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
//提交下载请求
HashMap<String, String> body = new HashMap<>();
body.put("payload", FORM_DATA);
body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
downloadPage = requests(download_link, POST, headers, body);
downloadDoc = Jsoup.parse(downloadPage);
String downloadPage = requests(download_link, POST, headers, body);
Document downloadDoc = Jsoup.parse(downloadPage);
//判断下载请求是否提交成功
if(downloadDoc.select("#db > p:nth-child(2) > strong").text().startsWith("#"))
@@ -192,14 +158,12 @@ public class GalleryUtil {
if(content.trim().isEmpty())
return null;
Document document = Jsoup.parse(content);
if(document.select("body > script").size() < 2)
return null;
Element script = document.select("body > script").get(1);
String[] scripts = script.html().split("\n");
String script = document.select("script").stream().map(Element::data).collect(java.util.stream.Collectors.joining("\n"));
if (!script.contains("imagelist") || !script.contains("mpvkey")) return null;
ArrayList<ImageKeyCache> imageKeyCaches = new ArrayList<>();
AtomicInteger page = new AtomicInteger(1);
gid2MpvKey.put(gid, scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", ""));
JsonNode nodes = parseImagelist(scripts[2]);
gid2MpvKey.put(gid, JavaScriptAssignments.string(script, "mpvkey"));
JsonNode nodes = objectMapper.readTree(JavaScriptAssignments.array(script, "imagelist"));
nodes.forEach((n) -> {
ImageKeyCache imageKeyCache = new ImageKeyCache();
imageKeyCache.setGid(gid);
@@ -247,12 +211,14 @@ public class GalleryUtil {
gid2MpvKey.remove(parseGid(url) + "");
return;
}
Document document = Jsoup.parse(content);
Element script = document.select("body > script").get(1);
String[] scripts = script.html().split("\n");
String mpvKey = scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", "");
gid2MpvKey.put(parseGid(url) + "", mpvKey);
log.info("刷新key:{}", mpvKey);
try {
String scripts = Jsoup.parse(content).select("script").stream().map(Element::data)
.collect(java.util.stream.Collectors.joining("\n"));
gid2MpvKey.put(parseGid(url) + "", JavaScriptAssignments.string(scripts, "mpvkey"));
} catch (IOException e) {
gid2MpvKey.remove(parseGid(url) + "");
log.warn("上游mpv脚本无法解析,缓存已失效");
}
}
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) {
@@ -0,0 +1,34 @@
package com.lion.lionwebsite.Util;
import java.io.IOException;
import java.util.regex.Pattern;
/** Extracts data assignments without depending on script position or line layout. */
public final class JavaScriptAssignments {
private JavaScriptAssignments() { }
public static String string(String source, String name) throws IOException {
var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*(['\"])(.*?)\\1", Pattern.DOTALL).matcher(source);
if (!matcher.find()) throw new IOException("上游脚本缺少 " + name);
return matcher.group(2);
}
public static String array(String source, String name) throws IOException {
var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*\\[").matcher(source);
if (!matcher.find()) throw new IOException("上游脚本缺少 " + name);
int start = matcher.end() - 1, depth = 0; char quote = 0; boolean escaped = false;
for (int i = start; i < source.length(); i++) {
char c = source.charAt(i);
if (quote != 0) {
if (escaped) escaped = false;
else if (c == '\\') escaped = true;
else if (c == quote) quote = 0;
continue;
}
if (c == '"' || c == '\'') quote = c;
else if (c == '[') depth++;
else if (c == ']' && --depth == 0) return source.substring(start, i + 1);
}
throw new IOException("上游脚本的 " + name + " 不完整");
}
}
@@ -0,0 +1,6 @@
package com.lion.lionwebsite.Util;
import com.lion.lionwebsite.Domain.Gallery;
/** A read-only upstream preparation; creating this value never submits a download. */
public record PreparedGallery(Gallery gallery, String formUrl, String origin) { }
@@ -0,0 +1,328 @@
package com.lion.lionwebsite.Util;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.util.Comparator;
import java.util.HexFormat;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
/**
* 订阅缓存的分代存储:v2ray 与 clash 两份内容必须同时生效。
*
* <p>旧实现把两份文件各自原子替换,第二份失败时用户会拿到「新 v2ray + 旧 clash」的
* 混合版本。这里把两份文件写进同一个不可变分代目录,校验通过后用一次原子替换发布
* 指针(manifest),因此抓取、解析、写入或发布任何一步失败,都只会保留上一个完整分代。
*
* <p>目录结构:
* <pre>
* &lt;cacheRoot&gt;/&lt;accountId&gt;/current 指针(原子替换,最后发布)
* &lt;cacheRoot&gt;/&lt;accountId&gt;/generations/&lt;generation&gt;/v2ray.txt
* &lt;cacheRoot&gt;/&lt;accountId&gt;/generations/&lt;generation&gt;/clash.yaml
* &lt;cacheRoot&gt;/&lt;accountId&gt;/v2ray.txt + clash.yaml 旧结构,两份都在才按 legacy 读
* </pre>
*
* <p>读取始终从指针指向的同一个分代里取两份文件,所以拿到的一定是同版本内容。
*/
@Slf4j
public class SubscriptionCacheStore {
private static final String MANIFEST_FILE = "current";
private static final String GENERATION_DIR = "generations";
private static final String V2_FILE = "v2ray.txt";
private static final String CLASH_FILE = "clash.yaml";
/** 保留目录名,结构上永远不存在,用于表达「当前没有可分发版本」。 */
private static final String UNAVAILABLE_DIR = ".unavailable";
private static final String MANIFEST_GENERATION = "generation";
private static final String MANIFEST_V2_SHA = "v2-sha256";
private static final String MANIFEST_CLASH_SHA = "clash-sha256";
/** 旧结构读出来的版本的 generation 标记;真实分代名是定宽时间戳,不会与之碰撞。 */
private static final String LEGACY_GENERATION = "legacy";
/** 保留分代数:当前与上一个,足够回退且不会无限增长。 */
private static final int RETAINED_GENERATIONS = 2;
private static final AtomicLong SEQUENCE = new AtomicLong();
private final Path root;
public SubscriptionCacheStore(String cacheRoot) {
this(Paths.get(cacheRoot));
}
public SubscriptionCacheStore(Path root) {
this.root = root;
}
/** 同一个分代目录里读出的一对内容;两份文件必然同版本。 */
public record Pair(String generation, byte[] v2, byte[] clash) { }
public Path accountDir(int accountId) {
return root.resolve(String.valueOf(accountId));
}
/** 旧结构的账号根文件路径。仅供兼容测试定位 legacy 文件,不作为回退目标。 */
public Path legacyPath(int accountId, String client) {
return accountDir(accountId).resolve(fileName(client));
}
/**
* 没有完整可分发版本时返回的哨兵路径。
*
* <p>它保证不存在,因此调用方用 {@code Files.isRegularFile} 判定就会走「订阅暂不可用」
* 分支,而不会被半份 legacy 文件骗过。
*/
public Path unavailablePath(int accountId, String client) {
return accountDir(accountId).resolve(UNAVAILABLE_DIR).resolve(fileName(client));
}
private static String fileName(String client) {
return "v2".equals(client) ? V2_FILE : CLASH_FILE;
}
/**
* 写入一份新分代并在最后发布指针。
*
* <p>失败时抛出异常,且不改变指针与上一个分代,调用方据此保留上次的完整版本。
*/
public Pair publish(int accountId, byte[] v2, byte[] clash) throws IOException {
Path account = accountDir(accountId);
Files.createDirectories(account);
String generation = nextGeneration();
Path dir = account.resolve(GENERATION_DIR).resolve(generation);
boolean published = false;
try {
Files.createDirectories(dir);
writeFile(dir.resolve(V2_FILE), v2);
writeFile(dir.resolve(CLASH_FILE), clash);
// 发布前校验:两份文件都必须能按写入内容读回,避免把半截内容挂上指针。
verify(dir.resolve(V2_FILE), v2);
verify(dir.resolve(CLASH_FILE), clash);
publishManifest(account, generation, sha256(v2), sha256(clash));
published = true;
} finally {
if (!published)
deleteRecursively(dir);
}
prune(account, generation);
removeLegacy(account);
return new Pair(generation, v2, clash);
}
/** 读取当前发布内容;只有构成一整个有效版本时才返回。 */
public Optional<Pair> read(int accountId) {
return Optional.ofNullable(load(accountDir(accountId)));
}
/**
* 解析当前应分发的单个文件。
*
* <p>与 {@link #read}、{@link #hasComplete} 使用同一份配对契约:只有两份文件
* 齐全且哈希校验通过时才返回路径,因此分发路径不会发出半份或已损坏的缓存。
*/
public Optional<Path> resolve(int accountId, String client) {
Path account = accountDir(accountId);
Pair pair = load(account);
if (pair == null)
return Optional.empty();
Path path = LEGACY_GENERATION.equals(pair.generation())
? account.resolve(fileName(client))
: account.resolve(GENERATION_DIR).resolve(pair.generation()).resolve(fileName(client));
return Files.isRegularFile(path) ? Optional.of(path) : Optional.empty();
}
/** 是否已有可分发的一对内容(分代两份都在且哈希匹配,或旧结构两份都在)。 */
public boolean hasComplete(int accountId) {
return load(accountDir(accountId)) != null;
}
/** 删除该账号的全部缓存(指针、分代、旧结构文件),使后续请求判定为无缓存。 */
public void invalidate(int accountId) {
Path account = accountDir(accountId);
deleteIfExists(account.resolve(MANIFEST_FILE));
deleteRecursively(account.resolve(GENERATION_DIR));
deleteIfExists(account.resolve(V2_FILE));
deleteIfExists(account.resolve(CLASH_FILE));
}
/** 写入接缝:测试覆写以模拟第二份文件写入失败。 */
protected void writeFile(Path target, byte[] data) throws IOException {
atomicWrite(target, data);
}
/** 指针发布接缝:测试覆写以模拟 manifest 发布失败。 */
protected void publishManifest(Path accountDir, String generation, String v2Sha256, String clashSha256)
throws IOException {
atomicWrite(accountDir.resolve(MANIFEST_FILE),
manifestText(generation, v2Sha256, clashSha256).getBytes(StandardCharsets.UTF_8));
}
/**
* 读取「一整个版本」:一次读入指针快照,再按该快照定位分代并校验双文件哈希。
*
* <p>指针只读一次,因此并发 {@link #publish} 切换指针时不会出现「用新分代名配旧哈希」
* 或哈希字段缺失导致的空指针;要么拿到完整的新版本,要么拿到完整的旧版本。
*/
private Pair load(Path account) {
Properties manifest = readManifest(account);
if (manifest == null) {
// 指针文件不存在才是「旧结构」。指针存在但读不出来说明它已损坏,同样按无缓存处理。
return Files.isRegularFile(account.resolve(MANIFEST_FILE)) ? null : loadLegacy(account);
}
Pair generation = loadGeneration(account, manifest);
if (generation != null)
return generation;
// 指针存在却读不出完整版本(半写、被篡改、内容缺失)时不得回退到 legacy:
// 那会发出一个比指针更旧的版本,掩盖损坏。此时按无可用缓存处理。
return null;
}
private Pair loadGeneration(Path account, Properties manifest) {
String generation = manifest.getProperty(MANIFEST_GENERATION);
if (generation == null || generation.isBlank() || generation.contains("/") || generation.contains("..")) {
log.warn("订阅缓存指针内容非法");
return null;
}
String v2Sha256 = manifest.getProperty(MANIFEST_V2_SHA);
String clashSha256 = manifest.getProperty(MANIFEST_CLASH_SHA);
if (v2Sha256 == null || clashSha256 == null) {
log.warn("订阅缓存指针缺少哈希字段 generation={}", generation);
return null;
}
Path dir = account.resolve(GENERATION_DIR).resolve(generation);
byte[] v2 = readFile(dir.resolve(V2_FILE));
byte[] clash = readFile(dir.resolve(CLASH_FILE));
if (v2 == null || clash == null) {
log.warn("订阅缓存分代内容缺失 generation={}", generation);
return null;
}
if (!matches(v2, v2Sha256) || !matches(clash, clashSha256)) {
log.warn("订阅缓存分代内容校验失败 generation={}", generation);
return null;
}
return new Pair(generation, v2, clash);
}
/** 旧结构:只有账号根下两份文件都在才构成一个有效版本。 */
private Pair loadLegacy(Path account) {
byte[] v2 = readFile(account.resolve(V2_FILE));
byte[] clash = readFile(account.resolve(CLASH_FILE));
if (v2 == null || clash == null)
return null;
return new Pair(LEGACY_GENERATION, v2, clash);
}
/** 指针读取接缝:包内可见,便于测试统计「每次查询只读一次指针」。 */
Properties readManifest(Path account) {
Path manifest = account.resolve(MANIFEST_FILE);
if (!Files.isRegularFile(manifest))
return null;
try {
Properties properties = new Properties();
properties.load(new StringReader(Files.readString(manifest, StandardCharsets.UTF_8)));
return properties;
} catch (IOException e) {
log.warn("读取订阅缓存指针失败: {}", e.getMessage());
return null;
}
}
private static byte[] readFile(Path path) {
try {
return Files.isRegularFile(path) ? Files.readAllBytes(path) : null;
} catch (IOException e) {
log.warn("读取订阅缓存文件失败 path={} error={}", path, e.getMessage());
return null;
}
}
private static void verify(Path path, byte[] expected) throws IOException {
byte[] written = Files.readAllBytes(path);
if (written.length != expected.length || !MessageDigest.isEqual(written, expected))
throw new IOException("订阅缓存写入校验失败 " + path.getFileName());
}
private static boolean matches(byte[] content, String expectedSha256) {
return expectedSha256 != null && expectedSha256.equals(sha256(content));
}
private static String manifestText(String generation, String v2Sha256, String clashSha256) {
return MANIFEST_GENERATION + "=" + generation + "\n"
+ MANIFEST_V2_SHA + "=" + v2Sha256 + "\n"
+ MANIFEST_CLASH_SHA + "=" + clashSha256 + "\n";
}
/** 分代名定宽可排序:时间戳在前保证单调,序号消除同毫秒碰撞。 */
private static String nextGeneration() {
return String.format("%013d-%06d", System.currentTimeMillis(), SEQUENCE.incrementAndGet() % 1_000_000);
}
private void prune(Path account, String current) {
Path dir = account.resolve(GENERATION_DIR);
List<Path> generations;
try (Stream<Path> entries = Files.list(dir)) {
generations = entries.filter(Files::isDirectory)
.sorted(Comparator.comparing(path -> path.getFileName().toString(), Comparator.reverseOrder()))
.toList();
} catch (IOException e) {
log.warn("清理订阅缓存分代失败 path={} error={}", dir, e.getMessage());
return;
}
for (int index = RETAINED_GENERATIONS; index < generations.size(); index++) {
Path stale = generations.get(index);
if (!stale.getFileName().toString().equals(current))
deleteRecursively(stale);
}
}
/** 分代发布成功后旧结构文件被取代,避免两份来源同时存在。 */
private void removeLegacy(Path account) {
deleteIfExists(account.resolve(V2_FILE));
deleteIfExists(account.resolve(CLASH_FILE));
}
private static void atomicWrite(Path target, byte[] data) throws IOException {
Path temp = target.resolveSibling(target.getFileName() + ".tmp");
Files.write(temp, data);
try {
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
}
}
private static void deleteIfExists(Path path) {
try {
Files.deleteIfExists(path);
} catch (IOException e) {
log.warn("删除订阅缓存文件失败 path={} error={}", path, e.getMessage());
}
}
private static void deleteRecursively(Path path) {
if (!Files.exists(path))
return;
try (Stream<Path> entries = Files.walk(path)) {
entries.sorted(Comparator.reverseOrder()).forEach(SubscriptionCacheStore::deleteIfExists);
} catch (IOException e) {
log.warn("删除订阅缓存目录失败 path={} error={}", path, e.getMessage());
}
}
private static String sha256(byte[] input) {
try {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(input));
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
}