Rewrite durable task lifecycle, gallery indexing and subscription publication
This commit is contained in:
@@ -27,6 +27,9 @@ import java.util.Random;
|
||||
Chat.class, MessageEntity.class,
|
||||
AbstractMethodError.class, DeleteGalleryMessage.class, DownloadPostMessage.class, DownloadStatusMessage.class,
|
||||
IdentityMessage.class, MaintainMessage.class, ResponseMessage.class, AvailableCheckMessage.class,
|
||||
// 协议 v2 的消息与任务操作实体:原生镜像下 Jackson/MyBatis 都要反射构造它们。
|
||||
ProtocolHelloMessage.class, TaskCommandMessage.class, TaskReportMessage.class, TaskReportAckMessage.class,
|
||||
TaskOperation.class,
|
||||
SubscriptionSnapshotMessage.class, SubscriptionSnapshotPayload.class, SubscriptionAccountSnapshot.class,
|
||||
SubscriptionBindingSnapshot.class, LinkPreviewOptions.class})
|
||||
public class CustomBean {
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package com.lion.lionwebsite.Configuration;
|
||||
|
||||
import com.lion.lionwebsite.Dao.cache.ImageCacheMapper;
|
||||
import com.lion.lionwebsite.Domain.ImageKeyCache;
|
||||
import com.lion.lionwebsite.Service.GalleryIndexService;
|
||||
import com.lion.lionwebsite.Util.GalleryUtil;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 图片索引服务的事务装配。
|
||||
*
|
||||
* <p>单独成文件是为了不触碰既有的 SqlConfiguration:那里的两个 SqlSessionFactory
|
||||
* 保持原样,这里只额外声明缓存库的事务管理器与模板。
|
||||
*
|
||||
* <p>事务管理器必须绑定 datasource-cache —— 索引只写缓存库,
|
||||
* 若误用主库事务管理器,整组替换就失去原子性(删除与写入落在不同连接上)。
|
||||
*/
|
||||
@Configuration
|
||||
public class GalleryIndexConfiguration {
|
||||
|
||||
@Bean
|
||||
public PlatformTransactionManager cacheTransactionManager(
|
||||
@Qualifier("datasource-cache") DataSource cacheDataSource) {
|
||||
return new DataSourceTransactionManager(cacheDataSource);
|
||||
}
|
||||
|
||||
/** 只用于图片索引的短事务发布,超时留给上层请求超时控制,这里显式设为 30 秒。 */
|
||||
@Bean
|
||||
public TransactionTemplate cacheTransactionTemplate(
|
||||
@Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) {
|
||||
TransactionTemplate template = new TransactionTemplate(cacheTransactionManager);
|
||||
template.setTimeout(30);
|
||||
return template;
|
||||
}
|
||||
|
||||
/**
|
||||
* 只读快照事务,用于一次性读取头部与页码。
|
||||
*
|
||||
* <p>读事务不写库,因此只读连接可以与写入并发;它的作用是让「读头部 + 读页码」
|
||||
* 落在同一个快照上,避免恰好读到整组替换的中间状态。
|
||||
*/
|
||||
@Bean
|
||||
public TransactionTemplate cacheReadTransactionTemplate(
|
||||
@Qualifier("cacheTransactionManager") PlatformTransactionManager cacheTransactionManager) {
|
||||
TransactionTemplate template = new TransactionTemplate(cacheTransactionManager);
|
||||
template.setTimeout(10);
|
||||
return template;
|
||||
}
|
||||
|
||||
/**
|
||||
* 这里显式不设置 {@code readOnly}:sqlite-jdbc 不允许在已建立的连接上改只读标志
|
||||
* (实测抛 SQLException: Cannot change read-only flag after establishing a connection),
|
||||
* 而事务模板拿到的是连接池里的既有连接,设置只读会让每次读取都直接失败。
|
||||
* 快照一致性由「同一个事务」保证,不需要额外的只读标志。
|
||||
*/
|
||||
|
||||
@Bean
|
||||
public GalleryIndexService galleryIndexService(
|
||||
ImageCacheMapper imageCacheMapper,
|
||||
@Qualifier("cacheTransactionTemplate") TransactionTemplate cacheTransactionTemplate,
|
||||
@Qualifier("cacheReadTransactionTemplate") TransactionTemplate cacheReadTransactionTemplate) {
|
||||
return new GalleryIndexService(imageCacheMapper, cacheTransactionTemplate,
|
||||
cacheReadTransactionTemplate,
|
||||
GalleryIndexConfiguration::fetchPageKeys);
|
||||
}
|
||||
|
||||
/**
|
||||
* 生产环境的上游实现:沿用 GalleryUtil.parseImageKeys(HTTP + JS 解析)。
|
||||
*
|
||||
* <p>放在这里而不是服务内部,是为了让服务在测试中可以脱离静态工具类。
|
||||
*/
|
||||
static List<ImageKeyCache> fetchPageKeys(String url) throws Exception {
|
||||
return GalleryUtil.parseImageKeys(url);
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,11 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.jdbc.DataSourceBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
@@ -16,7 +21,9 @@ public class SqlConfiguration {
|
||||
@Bean(name = "datasource-main")
|
||||
@ConfigurationProperties(prefix = "spring.datasource-main")
|
||||
public DataSource dataSourceMain() {
|
||||
return DataSourceBuilder.create().build();
|
||||
DataSource dataSource = DataSourceBuilder.create().build();
|
||||
applyBusyTimeout(dataSource);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
@Bean(name = "datasource-cache")
|
||||
@@ -25,6 +32,18 @@ public class SqlConfiguration {
|
||||
return DataSourceBuilder.create().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 给主库连接打开 busy 等待。
|
||||
*
|
||||
* <p>主库是单文件 SQLite,任务生命周期会并发开启短事务(创建、状态回执、删除)。
|
||||
* 默认没有 busy 超时,第二个写事务会立刻拿到 SQLITE_BUSY 并被当作 500;
|
||||
* 打开等待后,写冲突会按序排队重试而不是失败。
|
||||
*/
|
||||
private void applyBusyTimeout(DataSource dataSource) {
|
||||
if (dataSource instanceof HikariDataSource hikari)
|
||||
hikari.setConnectionInitSql("PRAGMA busy_timeout=5000");
|
||||
}
|
||||
|
||||
@Bean(name = "sqlSessionFactory-main")
|
||||
public SqlSessionFactory sqlSessionFactoryMain(@Qualifier("datasource-main") DataSource dataSource) throws Exception {
|
||||
SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
|
||||
@@ -48,4 +67,30 @@ public class SqlConfiguration {
|
||||
public SqlSessionTemplate sqlSessionTemplateCache(@Qualifier("sqlSessionFactory-cache") SqlSessionFactory sqlSessionFactory) {
|
||||
return new SqlSessionTemplate(sqlSessionFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* 主库事务管理器。
|
||||
*
|
||||
* <p>必须显式绑定到 datasource-main:应用里有两个 DataSource,Spring Boot 的自动配置
|
||||
* 会退避,若没有这个 bean 就没有任何事务管理器,@Transactional 会静默失效。
|
||||
* @Primary 同时让「不指定名字就注入 TransactionManager」的旧代码继续可用。
|
||||
*/
|
||||
@Bean(name = "mainTransactionManager")
|
||||
@Primary
|
||||
public DataSourceTransactionManager mainTransactionManager(@Qualifier("datasource-main") DataSource dataSource) {
|
||||
return new DataSourceTransactionManager(dataSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 主库编程式事务模板。
|
||||
*
|
||||
* <p>任务生命周期需要把「短事务」放在网络调用之外,并且要求「回调里捕获异常后不再抛出」
|
||||
* 不能错误提交,因此调用方必须用 {@code execute} 的返回状态或再次抛出决定回滚,
|
||||
* 而不是在回调里吞掉异常(见 GalleryTaskLifecycleService)。
|
||||
*/
|
||||
@Bean(name = "mainTransactionTemplate")
|
||||
@Primary
|
||||
public TransactionTemplate mainTransactionTemplate(@Qualifier("mainTransactionManager") DataSourceTransactionManager transactionManager) {
|
||||
return new TransactionTemplate(transactionManager);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.lion.lionwebsite.Configuration;
|
||||
|
||||
import com.lion.lionwebsite.Domain.TaskOperation;
|
||||
import com.lion.lionwebsite.Service.GalleryTaskLifecycleService;
|
||||
import com.lion.lionwebsite.Service.RemoteService;
|
||||
import com.lion.lionwebsite.Service.TaskReportHandler;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 装配协议 v2 的双向依赖,并做启动恢复。
|
||||
*
|
||||
* <p>RemoteService 消费上报时需要生命周期服务,而生命周期服务下发命令时需要 RemoteService。
|
||||
* 直接互相注入构造参数会形成环,所以这里在两端都构造完成之后用 setter 接上;
|
||||
* 同时也把「启动时把中断的 SUBMITTING 判为 UNKNOWN、释放 RESERVED 占位」这一步接到应用就绪事件上。
|
||||
*
|
||||
* <p>顺序是不可交换的:
|
||||
* <ol>
|
||||
* <li>先绑定上报处理器、下发判定解析器与 operationId 解析器;</li>
|
||||
* <li>再执行启动恢复(SUBMITTING → UNKNOWN、释放 RESERVED);</li>
|
||||
* <li>最后才允许建立连接与自动重投。连接由本类显式触发,不再由 RemoteService 的
|
||||
* PostConstruct 在装配前发起,否则会把状态不明的任务盲目重投给节点。</li>
|
||||
* </ol>
|
||||
* 恢复失败时仍然允许连接(保持节点在线/心跳),但明确禁止自动派发。
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class TaskLifecycleWiring {
|
||||
|
||||
final RemoteService remoteService;
|
||||
|
||||
final GalleryTaskLifecycleService lifecycleService;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void wire() {
|
||||
TaskReportHandler handler = lifecycleService;
|
||||
remoteService.setTaskReportHandler(handler);
|
||||
remoteService.setDownloadOperationIdResolver(lifecycleService::currentOperationId);
|
||||
remoteService.setDispatchResolver(lifecycleService::resolveDispatch);
|
||||
boolean recoverySafe = false;
|
||||
try {
|
||||
int recovered = lifecycleService.recoverInterruptedSubmissions();
|
||||
recoverySafe = true;
|
||||
if (recovered > 0)
|
||||
log.warn("启动恢复:{} 个中断的创建/预占任务已处理(提交中的标为提交未知)", recovered);
|
||||
} catch (RuntimeException e) {
|
||||
// 恢复失败说明库中任务真实状态未知:可以保持连接,但绝不能自动重投/派发。
|
||||
log.error("启动恢复失败,本次启动将拒绝自动重投未完成任务", e);
|
||||
}
|
||||
// 绑定与恢复完成之后才连接:连接成功后的自动重投受 lifecycleReady/recoverySafe 双重约束。
|
||||
remoteService.startAfterWiring(recoverySafe);
|
||||
}
|
||||
}
|
||||
@@ -62,16 +62,14 @@ public class PublicController {
|
||||
public String validate(String AuthCode){
|
||||
Response response = Response.generateResponse();
|
||||
User user = publicService.getUserId(AuthCode);
|
||||
// 用 ObjectMapper 组装内层 JSON:手工 String.format 拼用户名时,
|
||||
// 名字里带引号或反斜杠会直接产出非法 JSON,前端 JSON.parse 随即失败。
|
||||
// 这里仍以「JSON 文本」形式放进 data(历史契约,前端按字符串再解析一次)。
|
||||
// 身份字段直接作为结构化 data 返回;共享前端解码器兼容旧版本字符串。
|
||||
ObjectNode identity = CustomUtil.objectMapper.createObjectNode();
|
||||
identity.put("userId", user.getId());
|
||||
identity.put("username", user.getUsername());
|
||||
identity.put("isAvailable", !remoteService.isDead());
|
||||
// 管理员标记随登录一起下发,前端据此决定是否显示下载人信息与筛选。
|
||||
identity.put("isAdmin", user.getId() == UserService.ADMIN_USER_ID);
|
||||
response.success(identity.toString());
|
||||
response.success(identity);
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ImageCacheMapper {
|
||||
@Insert("insert into ImageKeyCache values (#{gid}, #{imgkey}, #{page})")
|
||||
@@ -31,4 +33,32 @@ public interface ImageCacheMapper {
|
||||
|
||||
@Delete("delete from gidToKey where gid=#{gid}")
|
||||
void deleteGidToKey(String gid);
|
||||
|
||||
/**
|
||||
* 某个 gid 已落库的页码列表(升序)。
|
||||
*
|
||||
* <p>只按行数判断完整性会把「页码 1 和 3、共 2 行」当成完整索引,
|
||||
* 因此完整性判定改为按页码集合比对——先取回页码,再校验它是否恰好是 1..pages。
|
||||
*/
|
||||
@Select("select page from ImageKeyCache where gid=#{gid} order by page")
|
||||
List<Integer> selectPageNumbersByGid(String gid);
|
||||
|
||||
/** 该 gid 下 imgkey 为空的脏行数;任何一行脏行都说明索引不可用。 */
|
||||
@Select("select count(*) from ImageKeyCache where gid=#{gid} and (imgkey is null or trim(imgkey)='')")
|
||||
int countInvalidImageKeysByGid(String gid);
|
||||
|
||||
/**
|
||||
* 批量写入页 key(单条多值 INSERT)。
|
||||
*
|
||||
* <p>整本画廊的索引可能上千页,逐行 INSERT 会拉长写事务并放大锁竞争;
|
||||
* 调用方按固定大小分片后,一个写事务里只需少量语句即可写完。
|
||||
*/
|
||||
// MyBatis 只在注解值「以 <script> 开头」时才按 XML 动态 SQL 解析,因此这里不能有前导空白。
|
||||
@Insert("<script>"
|
||||
+ "insert into ImageKeyCache (gid, imgkey, page) values "
|
||||
+ "<foreach collection='pages' item='p' separator=','>"
|
||||
+ "(#{p.gid}, #{p.imgkey}, #{p.page})"
|
||||
+ "</foreach>"
|
||||
+ "</script>")
|
||||
int insertImageKeyCacheBatch(@Param("pages") List<ImageKeyCache> pages);
|
||||
}
|
||||
|
||||
@@ -20,4 +20,8 @@ public interface CollectMapper {
|
||||
|
||||
@Select("select collector from collect where gid=#{gid}")
|
||||
ArrayList<Integer> selectCollectorByGid(int gid);
|
||||
|
||||
/** 任务删除完成后清理该 gid 的全部收藏记录(与删任务行同一事务)。 */
|
||||
@Delete("delete from collect where gid=#{gid}")
|
||||
void deleteByGid(@Param("gid") int gid);
|
||||
}
|
||||
|
||||
@@ -36,9 +36,10 @@ public interface GalleryMapper {
|
||||
|
||||
|
||||
@Update("""
|
||||
update gallery set name=#{name}, link=#{link}, language=#{language},
|
||||
update gallery set name=#{name}, link=#{link}, language=#{language}, downloader=#{downloader},
|
||||
pages=#{pages}, status=#{status}, fileSize=#{fileSize}, createTime=#{createTime},
|
||||
proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize}, is_download=#{is_download}
|
||||
proceeding=#{proceeding}, resolution=#{resolution}, displayFileSize=#{displayFileSize},
|
||||
thumb_link=#{thumb_link}, is_download=#{is_download}
|
||||
where gid=#{gid}""")
|
||||
void updateGallery(Gallery gallery);
|
||||
|
||||
@@ -47,4 +48,8 @@ public interface GalleryMapper {
|
||||
|
||||
@Delete("delete from gallery where gid=#{gid}")
|
||||
void deleteGalleryByGid(Integer gid);
|
||||
|
||||
/** 只改状态列,用于「待删除」这类不牵动其他字段的意图标记。 */
|
||||
@Update("update gallery set status=#{status} where gid=#{gid}")
|
||||
int updateGalleryStatus(@Param("gid") int gid, @Param("status") String status);
|
||||
}
|
||||
|
||||
@@ -43,49 +43,57 @@ public interface SubMapper {
|
||||
@Delete("delete from subscription_account where id=#{id}")
|
||||
void deleteSubscriptionAccount(Integer id);
|
||||
|
||||
@Insert("insert into sub_bind (key, user, subscription_account_id) values (#{key}, #{user}, #{subscriptionAccountId})")
|
||||
@Insert("insert into sub_bind (key, user, subscription_account_id, user_id) values (#{key}, #{user}, #{subscriptionAccountId}, (select id from User where username=#{user}))")
|
||||
void insertSubBind(SubBind subBind);
|
||||
|
||||
@Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user")
|
||||
@Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id order by sb.user")
|
||||
ArrayList<SubBind> selectAllSubBind();
|
||||
|
||||
@Select("select sb.key, sb.user, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}")
|
||||
@Select("select sb.key, coalesce(u.username, sb.user) as user, sb.user_id as userId, sb.subscription_account_id as subscriptionAccountId, sa.name as subscriptionAccountName, sa.enabled as subscriptionAccountEnabled, sa.filter_high_multiplier as filterHighMultiplier from sub_bind sb left join User u on u.id=sb.user_id left join subscription_account sa on sa.id=sb.subscription_account_id where sb.key=#{key}")
|
||||
SubBind selectSubBind(String key);
|
||||
|
||||
@Select("select count(key) from sub_bind where key=#{key}")
|
||||
boolean selectSubBindExist(String key);
|
||||
|
||||
@Select("select count(*) from sub_bind where user=#{user}")
|
||||
@Select("select count(*) from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
int countSubBindByUser(String user);
|
||||
|
||||
@Update("update sub_bind set subscription_account_id=#{accountId} where user=#{user}")
|
||||
@Update("update sub_bind set subscription_account_id=#{accountId} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
int updateSubBindAccount(@Param("user") String user, @Param("accountId") Integer accountId);
|
||||
|
||||
@Update("update sub_bind set key=#{key} where user=#{user}")
|
||||
@Update("update sub_bind set key=#{key} where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
int updateSubBindKey(@Param("user") String user, @Param("key") String key);
|
||||
|
||||
@Delete("delete from sub_bind where user=#{user}")
|
||||
@Delete("delete from sub_bind where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
void deleteSubBind(String user);
|
||||
|
||||
@Insert("insert into sub_update_record (user, ip, UA, time, location) values (#{user}, #{ip}, #{UA}, #{time}, #{location})")
|
||||
@Insert("insert into sub_update_record (user, ip, UA, time, location, user_id) values (#{user}, #{ip}, #{UA}, #{time}, #{location}, coalesce(#{userId}, (select id from User where username=#{user})))")
|
||||
@Options(useGeneratedKeys = true, keyProperty = "id")
|
||||
void insertSubUpdateRecord(SubUpdateRecord subUpdateRecord);
|
||||
|
||||
@Select("select * from sub_update_record order by time desc")
|
||||
@Select("select id,user,ip,UA,time,location,user_id as userId from sub_update_record order by time desc")
|
||||
ArrayList<SubUpdateRecord> selectAllSubUpdateRecord();
|
||||
|
||||
@Delete("delete from sub_update_record where user=#{user}")
|
||||
@Delete("delete from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
void deleteSubUpdateRecord(String user);
|
||||
|
||||
@Select("select count(user) from sub_update_record where user=#{user}")
|
||||
@Select("select count(user) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
Integer selectUpdateRecordCount(String user);
|
||||
|
||||
@Select("select min(id) from sub_update_record where user=#{user}")
|
||||
@Select("select min(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
|
||||
Integer selectMinUpdateRecordId(String user);
|
||||
|
||||
/** 后台补齐归属地:只更新该用户最近一条记录,避免覆盖更早的历史记录。 */
|
||||
@Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user=#{user})")
|
||||
@Update("update sub_update_record set location=#{location} where id=(select max(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user}))")
|
||||
void updateLatestSubUpdateRecordLocation(@Param("user") String user, @Param("location") String location);
|
||||
|
||||
@Update("update sub_update_record set location=#{location} where id=#{id}")
|
||||
void updateSubUpdateRecordLocation(@Param("id") int id, @Param("location") String location);
|
||||
|
||||
@Update("update sub_bind set user=#{newName} where user_id=#{userId} or (user_id is null and user=#{oldName})")
|
||||
void renameBoundUser(@Param("userId") int userId, @Param("oldName") String oldName,
|
||||
@Param("newName") String newName);
|
||||
|
||||
@Delete("delete from sub_update_record where id=#{id}")
|
||||
void deleteSubUpdateRecordById(int id);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package com.lion.lionwebsite.Dao.normal;
|
||||
|
||||
import com.lion.lionwebsite.Domain.TaskOperation;
|
||||
import org.apache.ibatis.annotations.*;
|
||||
|
||||
/**
|
||||
* 任务操作记录的持久化入口(主库)。
|
||||
*
|
||||
* <p>所有写操作都在调用方的事务里执行;这里的注解只描述 SQL,不包含业务判断。
|
||||
* 主键为 gid,保证「同一 gid 只能有一条有效操作意图」。
|
||||
*/
|
||||
@Mapper
|
||||
public interface TaskOperationRepository {
|
||||
|
||||
@Select("select * from task_operation where gid=#{gid}")
|
||||
TaskOperation selectByGid(int gid);
|
||||
|
||||
/** 生成该 gid 的稳定下载 operationId:只有还没有记录时才写入。 */
|
||||
@Insert("insert or ignore into task_operation (gid, operationId, deleteOperationId, state, lastRevision," +
|
||||
" lastReportId, updateTime) values (#{gid}, #{operationId}, null, #{state}, 0, null, #{updateTime})")
|
||||
int insertIfAbsent(@Param("gid") int gid, @Param("operationId") String operationId,
|
||||
@Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
@Insert("insert into task_operation (gid, operationId, deleteOperationId, state, lastRevision, lastReportId, updateTime)" +
|
||||
" values (#{gid}, #{operationId}, #{deleteOperationId}, #{state}, #{lastRevision}, #{lastReportId}, #{updateTime})")
|
||||
void insert(TaskOperation operation);
|
||||
|
||||
/** 推进创建意图的状态(RESERVED → SUBMITTING → SUBMITTED/UNKNOWN)。 */
|
||||
@Update("update task_operation set state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
|
||||
int updateState(@Param("gid") int gid, @Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 带前置条件的创建状态推进:只有当前状态与 operationId 都匹配时才写入。
|
||||
*
|
||||
* <p>这是「提交前先确认这一代意图仍然有效」的 CAS:若期间被删除/重建,
|
||||
* 返回值 0,调用方必须停下,绝不能再产生上游副作用。
|
||||
*/
|
||||
@Update("update task_operation set state=#{state}, updateTime=#{updateTime}" +
|
||||
" where gid=#{gid} and operationId=#{operationId} and state=#{expectedState}")
|
||||
int compareAndSetCreateState(@Param("gid") int gid, @Param("operationId") String operationId,
|
||||
@Param("expectedState") String expectedState, @Param("state") String state,
|
||||
@Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 只按当前删除意图 ID 匹配的删除完成:期间若已被删除/重建,则不改动新世代记录。
|
||||
*/
|
||||
@Update("update task_operation set deleteOperationId=null, state=#{state}, updateTime=#{updateTime}" +
|
||||
" where gid=#{gid} and deleteOperationId=#{deleteOperationId}")
|
||||
int completeDeleteIfSameGeneration(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId,
|
||||
@Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 删除「本次创建意图」的预占记录:必须同时匹配 gid 与 operationId,且状态仍是
|
||||
* RESERVED/SUBMITTING。
|
||||
*
|
||||
* <p>三重匹配保证释放流程只能清掉自己那一代:期间被删除/重建(新 operationId 或墓碑)
|
||||
* 的记录一律不动,避免把新世代抹掉。
|
||||
*/
|
||||
@Delete("delete from task_operation where gid=#{gid} and operationId=#{operationId}" +
|
||||
" and state in ('RESERVED','SUBMITTING')")
|
||||
int deleteReservationIfSameGeneration(@Param("gid") int gid, @Param("operationId") String operationId);
|
||||
|
||||
/** 记录/替换当前下载 operationId 并推进状态。 */
|
||||
@Update("update task_operation set operationId=#{operationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
|
||||
int updateOperationId(@Param("gid") int gid, @Param("operationId") String operationId,
|
||||
@Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 给「有记录但缺 operationId」的历史任务补一个稳定 operationId。
|
||||
* 只在确实缺失时写入,避免覆盖并发创建出来的新世代。
|
||||
*/
|
||||
@Update("update task_operation set operationId=#{operationId}, updateTime=#{updateTime}" +
|
||||
" where gid=#{gid} and operationId is null")
|
||||
int assignOperationIdIfMissing(@Param("gid") int gid, @Param("operationId") String operationId,
|
||||
@Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 墓碑复活:删除完成后的 gid 允许用全新 operationId 重新创建。
|
||||
* revision 与回执一并清零,旧操作的迟到上报会因 operationId 不匹配被丢弃。
|
||||
*/
|
||||
@Update("update task_operation set operationId=#{operationId}, deleteOperationId=null, state=#{state}," +
|
||||
" lastRevision=0, lastReportId=null, updateTime=#{updateTime} where gid=#{gid}")
|
||||
int reactivate(@Param("gid") int gid, @Param("operationId") String operationId,
|
||||
@Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
/** 记录删除意图与其稳定的删除操作 ID(重试必须复用同一个 ID)。 */
|
||||
@Update("update task_operation set deleteOperationId=#{deleteOperationId}, state=#{state}, updateTime=#{updateTime} where gid=#{gid}")
|
||||
int updateDeleteIntent(@Param("gid") int gid, @Param("deleteOperationId") String deleteOperationId,
|
||||
@Param("state") String state, @Param("updateTime") long updateTime);
|
||||
|
||||
/**
|
||||
* 写入状态上报回执(revision + reportId)。
|
||||
*
|
||||
* <p>回执与任务行更新在同一个事务里提交,提交成功之后才回 ACK;
|
||||
* 因此节点重复投递同一份上报时,只能看到已经提交过的回执,不会重复落库或重复通知。
|
||||
*/
|
||||
@Update("update task_operation set lastRevision=#{revision}, lastReportId=#{reportId}, updateTime=#{updateTime}" +
|
||||
" where gid=#{gid}")
|
||||
int updateReportReceipt(@Param("gid") int gid, @Param("revision") long revision,
|
||||
@Param("reportId") String reportId, @Param("updateTime") long updateTime);
|
||||
|
||||
@Select("select * from task_operation where state=#{state}")
|
||||
java.util.List<TaskOperation> selectByState(String state);
|
||||
|
||||
@Delete("delete from task_operation where gid=#{gid}")
|
||||
int deleteByGid(int gid);
|
||||
}
|
||||
@@ -14,4 +14,10 @@ public class SubBind {
|
||||
String subscriptionAccountName;
|
||||
boolean subscriptionAccountEnabled;
|
||||
boolean filterHighMultiplier;
|
||||
Integer userId;
|
||||
|
||||
public SubBind(String key, String user, Integer accountId, String accountName,
|
||||
boolean accountEnabled, boolean filterHighMultiplier) {
|
||||
this(key, user, accountId, accountName, accountEnabled, filterHighMultiplier, null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,4 +16,9 @@ public class SubUpdateRecord {
|
||||
String UA;
|
||||
Date time;
|
||||
String location;
|
||||
Integer userId;
|
||||
|
||||
public SubUpdateRecord(int id, String user, String ip, String UA, Date time, String location) {
|
||||
this(id, user, ip, UA, time, location, null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.lion.lionwebsite.Domain;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 单个 gid 的下载生命周期操作记录(协议 v2 的持久化意图)。
|
||||
*
|
||||
* <p>这张表是「先持久化意图、再产生外部副作用」的落点:
|
||||
* <ul>
|
||||
* <li>CREATE 意图先于上游 POST 落库,用来保证同一 gid 只有一个未完成意图;</li>
|
||||
* <li>提交结果不确定时只能停在 {@link #STATE_UNKNOWN},绝不自动重复上游 POST;</li>
|
||||
* <li>DELETE 意图先于节点删除落库,节点确认后(或文件已不存在)才在同一事务里删任务行并留墓碑。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>旧任务行没有对应操作记录,查询时按 gid 惰性补建(见 GalleryTaskLifecycleService)。
|
||||
*/
|
||||
@Data
|
||||
public class TaskOperation {
|
||||
|
||||
/** 已登记创建意图,尚未向上游提交。 */
|
||||
public static final String STATE_RESERVED = "RESERVED";
|
||||
/** 正在向上游提交;此时不可再发第二次 POST。 */
|
||||
public static final String STATE_SUBMITTING = "SUBMITTING";
|
||||
/** 上游确认已接受下载请求。 */
|
||||
public static final String STATE_SUBMITTED = "SUBMITTED";
|
||||
/**
|
||||
* 提交结果未知:超时、断连或在 SUBMITTING 期间重启。
|
||||
* 上游不支持幂等,因此这里只能等对账(reconcile),不能自动重发。
|
||||
*/
|
||||
public static final String STATE_UNKNOWN = "UNKNOWN";
|
||||
/** 已登记删除意图,等待节点确认。 */
|
||||
public static final String STATE_DELETE_PENDING = "DELETE_PENDING";
|
||||
/** 节点已确认删除,任务行也随之删除;本行作为墓碑保留,用于停止迟到状态的重放。 */
|
||||
public static final String STATE_DELETED = "DELETED";
|
||||
|
||||
private int gid;
|
||||
|
||||
/** 当前有效的下载操作 ID;同一 gid 的 DOWNLOAD 重试必须复用它。 */
|
||||
private String operationId;
|
||||
|
||||
/** 独立于下载操作 ID 的删除操作 ID,删除重试复用它。 */
|
||||
private String deleteOperationId;
|
||||
|
||||
private String state;
|
||||
|
||||
/** 最近一次应用的状态上报 revision;用于丢弃乱序/重放的旧上报。 */
|
||||
private long lastRevision;
|
||||
|
||||
/** 节点已确认收到的最新上报 ID;用于重复 ACK 的幂等判断。 */
|
||||
private String lastReportId;
|
||||
|
||||
private long updateTime;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.lion.lionwebsite.Exception;
|
||||
|
||||
/**
|
||||
* 图片索引构建失败。
|
||||
*
|
||||
* <p>调用方需要区分三类结果,才能给出正确的用户可见响应:
|
||||
* 链接本身有问题({@link Reason#INVALID_LINK},属于请求错误)、
|
||||
* 画廊已不存在({@link Reason#UNAVAILABLE},重试无意义)、
|
||||
* 上游或数据库暂时不可用({@link Reason#TRANSIENT},可以重试)。
|
||||
*/
|
||||
public class GalleryIndexException extends Exception {
|
||||
|
||||
public enum Reason {
|
||||
/** 链接无法解析出 gid 或 key。 */
|
||||
INVALID_LINK,
|
||||
/** 上游明确表示画廊不存在(下架/删除/解析不到页 key)。 */
|
||||
UNAVAILABLE,
|
||||
/** 上游返回的页 key 无法自洽(gid 不匹配、页码重复/跳号、imgkey 为空)。 */
|
||||
MALFORMED,
|
||||
/** 上游波动或索引写入失败,重试可能成功。 */
|
||||
TRANSIENT
|
||||
}
|
||||
|
||||
private final Reason reason;
|
||||
|
||||
public GalleryIndexException(Reason reason, String message) {
|
||||
super(message);
|
||||
this.reason = reason;
|
||||
}
|
||||
|
||||
public GalleryIndexException(Reason reason, String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.reason = reason;
|
||||
}
|
||||
|
||||
public Reason getReason() {
|
||||
return reason;
|
||||
}
|
||||
|
||||
/** 重试没有意义(链接错误或画廊不可用);上游数据畸形与会话级故障仍可重试。 */
|
||||
public boolean isPermanent() {
|
||||
return reason == Reason.INVALID_LINK || reason == Reason.UNAVAILABLE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
package com.lion.lionwebsite.Maintenance;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 缓存库图片索引的离线体检与修复工具。
|
||||
*
|
||||
* <p>它是独立可执行类(只用 JDBC,不依赖 Spring / MyBatis),因此可以在服务停止后
|
||||
* 直接对数据库文件运行,也可以先对生产库的副本演练。</p>
|
||||
*
|
||||
* <pre>
|
||||
* java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db
|
||||
* java -cp sqlite-jdbc.jar GalleryIndexMaintenance.java --db /path/cache.db --apply
|
||||
* </pre>
|
||||
*
|
||||
* <p>它只做两类写入:删除不完整或有冲突的 gid 整组索引(gidToKey 头部加该 gid 的全部
|
||||
* 页 key),以及删除没有头部记录的孤儿页 key。被删掉的 gid 由下一次在线阅读重新抓取,
|
||||
* 因此不需要猜测保留哪一组 key。工具不建表、不改列、不碰其它业务表。</p>
|
||||
*/
|
||||
public final class GalleryIndexMaintenance {
|
||||
|
||||
/** 生产运行时数据库所在目录;默认拒绝直接写入,必须显式加 --allow-production。 */
|
||||
private static final String PRODUCTION_DIR = "/home/lionwebsite";
|
||||
|
||||
private static final String PLAIN_REPAIR_NOTE =
|
||||
"这些画廊将在下次被访问时重新抓取。";
|
||||
|
||||
public static void main(String[] args) {
|
||||
Map<String, String> options = parseOptions(args);
|
||||
if (options.containsKey("help") || !options.containsKey("db")) {
|
||||
printUsage();
|
||||
System.exit(options.containsKey("help") ? 0 : 2);
|
||||
return;
|
||||
}
|
||||
|
||||
Path database = Path.of(options.get("db")).toAbsolutePath().normalize();
|
||||
boolean apply = options.containsKey("apply");
|
||||
if (!Files.isRegularFile(database)) {
|
||||
System.err.println("数据库不存在: " + database);
|
||||
System.exit(2);
|
||||
return;
|
||||
}
|
||||
if (apply && isProductionPath(database) && !options.containsKey("allow-production")) {
|
||||
System.err.println("拒绝写入生产库路径 " + database
|
||||
+ "。请先在副本上演练;确需在本机生产库执行时显式加 --allow-production,"
|
||||
+ "并确保服务已停止。");
|
||||
System.exit(3);
|
||||
return;
|
||||
}
|
||||
|
||||
try (Connection connection = open(database)) {
|
||||
Report before = inspect(connection);
|
||||
System.out.println(before.render(apply ? "修复前" : "体检结果"));
|
||||
|
||||
if (!apply) {
|
||||
System.out.println();
|
||||
System.out.println("dry-run:未做任何写入。确认无误后加 --apply 执行修复。");
|
||||
System.exit(before.isHealthy() ? 0 : 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (before.isHealthy()) {
|
||||
System.out.println();
|
||||
System.out.println("索引已健康,无需修复。");
|
||||
return;
|
||||
}
|
||||
|
||||
connection.setAutoCommit(false);
|
||||
int removed = repair(connection, before.badGids, before.orphanGids);
|
||||
connection.commit();
|
||||
System.out.println();
|
||||
System.out.println("已删除 " + removed + " 个 gid 的索引整组," + PLAIN_REPAIR_NOTE);
|
||||
|
||||
Report after = inspect(connection);
|
||||
System.out.println();
|
||||
System.out.println(after.render("修复后"));
|
||||
if (!after.isHealthy()) {
|
||||
System.err.println("修复后仍未达到健康状态,请人工检查。");
|
||||
System.exit(4);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
System.err.println("执行失败: " + e.getMessage());
|
||||
e.printStackTrace(System.err);
|
||||
System.exit(5);
|
||||
}
|
||||
}
|
||||
|
||||
private static Connection open(Path database) throws SQLException {
|
||||
Connection connection = DriverManager.getConnection("jdbc:sqlite:" + database);
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute("pragma busy_timeout=5000");
|
||||
statement.execute("pragma foreign_keys=off");
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
/** 是否位于生产运行时目录;工具据此拒绝在未显式确认时写入生产库。 */
|
||||
static boolean isProductionPath(Path database) {
|
||||
return database.startsWith(Path.of(PRODUCTION_DIR).toAbsolutePath().normalize());
|
||||
}
|
||||
|
||||
// ---------- 体检 ----------
|
||||
|
||||
static Report inspect(Connection connection) throws SQLException {
|
||||
Report report = new Report();
|
||||
report.hasHeaderTable = tableExists(connection, "gidToKey");
|
||||
report.hasPageTable = tableExists(connection, "ImageKeyCache");
|
||||
if (!report.hasHeaderTable || !report.hasPageTable)
|
||||
return report;
|
||||
|
||||
report.headerRows = count(connection, "select count(*) from gidToKey");
|
||||
report.pageRows = count(connection, "select count(*) from ImageKeyCache");
|
||||
report.duplicateHeaderGids = count(connection,
|
||||
"select count(*) from (select gid from gidToKey group by gid having count(*) > 1)");
|
||||
report.duplicatePageGroups = count(connection,
|
||||
"select count(*) from (select gid, page from ImageKeyCache group by gid, page having count(*) > 1)");
|
||||
|
||||
try (PreparedStatement statement = connection.prepareStatement(BAD_GID_QUERY);
|
||||
ResultSet rows = statement.executeQuery()) {
|
||||
while (rows.next()) {
|
||||
String gid = rows.getString(1);
|
||||
report.badGids.add(gid);
|
||||
report.badReasons.put(gid, describe(rows));
|
||||
}
|
||||
}
|
||||
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"select distinct gid from ImageKeyCache where gid not in (select gid from gidToKey)");
|
||||
ResultSet rows = statement.executeQuery()) {
|
||||
while (rows.next())
|
||||
report.orphanGids.add(rows.getString(1));
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一个 gid 触发的全部不健康条件集中在一句里检查,避免多次全表扫描。
|
||||
*
|
||||
* <p>必须按 gid 分组后再判断:重复 gid(同一 gid 有多条头部记录)本身就是要清除的对象,
|
||||
* 而重复行里的某一条可能恰好与页 key 数一致,逐行判断会把它当成健康记录漏掉。
|
||||
*/
|
||||
private static final String BAD_GID_QUERY = """
|
||||
select g.gid as gid,
|
||||
count(*) as header_rows,
|
||||
max(g.pages) as pages,
|
||||
min(g.key) as key,
|
||||
(select count(*) from ImageKeyCache i where i.gid = g.gid) as page_rows,
|
||||
(select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) as distinct_pages,
|
||||
(select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as min_page,
|
||||
(select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) as max_page,
|
||||
(select count(*) from ImageKeyCache i where i.gid = g.gid
|
||||
and (i.imgkey is null or trim(i.imgkey) = '')) as dirty_keys
|
||||
from gidToKey g
|
||||
group by g.gid
|
||||
having count(*) > 1
|
||||
or max(g.pages) is null or max(g.pages) <= 0
|
||||
or min(g.key) is null or trim(min(g.key)) = ''
|
||||
or (select count(*) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
|
||||
or (select count(distinct i.page) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
|
||||
or (select coalesce(min(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> 1
|
||||
or (select coalesce(max(i.page), 0) from ImageKeyCache i where i.gid = g.gid) <> max(g.pages)
|
||||
or (select count(*) from ImageKeyCache i where i.gid = g.gid
|
||||
and (i.imgkey is null or trim(i.imgkey) = '')) > 0
|
||||
order by g.gid
|
||||
""";
|
||||
|
||||
private static String describe(ResultSet row) throws SQLException {
|
||||
List<String> reasons = new ArrayList<>();
|
||||
int headerRows = row.getInt("header_rows");
|
||||
int pages = row.getInt("pages");
|
||||
String key = row.getString("key");
|
||||
int pageRows = row.getInt("page_rows");
|
||||
int distinctPages = row.getInt("distinct_pages");
|
||||
int minPage = row.getInt("min_page");
|
||||
int maxPage = row.getInt("max_page");
|
||||
int dirtyKeys = row.getInt("dirty_keys");
|
||||
if (headerRows > 1) reasons.add("重复头部记录 " + headerRows + " 条");
|
||||
if (key == null || key.isBlank()) reasons.add("key 为空");
|
||||
if (pages <= 0) reasons.add("pages 非法(" + pages + ")");
|
||||
if (pageRows != pages) reasons.add("页 key 行数 " + pageRows + " 与 pages " + pages + " 不符");
|
||||
if (distinctPages != pages) reasons.add("页码去重后 " + distinctPages + " 个");
|
||||
if (minPage != 1 || maxPage != pages) reasons.add("页码区间 " + minPage + "-" + maxPage);
|
||||
if (dirtyKeys > 0) reasons.add("imgkey 为空 " + dirtyKeys + " 行");
|
||||
return String.join("、", reasons);
|
||||
}
|
||||
|
||||
// ---------- 修复 ----------
|
||||
|
||||
/** 删除不健康 gid 的整组索引与孤儿页 key,返回被移除索引的 gid 个数。 */
|
||||
static int repair(Connection connection, List<String> badGids, List<String> orphanGids)
|
||||
throws SQLException {
|
||||
int touched = 0;
|
||||
try (PreparedStatement pages = connection.prepareStatement("delete from ImageKeyCache where gid = ?");
|
||||
PreparedStatement header = connection.prepareStatement("delete from gidToKey where gid = ?")) {
|
||||
for (String gid : badGids) {
|
||||
pages.setString(1, gid);
|
||||
pages.executeUpdate();
|
||||
header.setString(1, gid);
|
||||
header.executeUpdate();
|
||||
touched++;
|
||||
}
|
||||
for (String gid : orphanGids) {
|
||||
pages.setString(1, gid);
|
||||
if (pages.executeUpdate() > 0)
|
||||
touched++;
|
||||
}
|
||||
}
|
||||
return touched;
|
||||
}
|
||||
|
||||
// ---------- 基础设施 ----------
|
||||
|
||||
private static boolean tableExists(Connection connection, String table) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"select count(*) from sqlite_master where type='table' and name=?")) {
|
||||
statement.setString(1, table);
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
return rows.next() && rows.getInt(1) > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static int count(Connection connection, String sql) throws SQLException {
|
||||
try (Statement statement = connection.createStatement();
|
||||
ResultSet rows = statement.executeQuery(sql)) {
|
||||
return rows.next() ? rows.getInt(1) : 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, String> parseOptions(String[] args) {
|
||||
Map<String, String> options = new LinkedHashMap<>();
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
String arg = args[i];
|
||||
if (!arg.startsWith("--"))
|
||||
continue;
|
||||
String name = arg.substring(2);
|
||||
if (name.equals("apply") || name.equals("help") || name.equals("allow-production")) {
|
||||
options.put(name, "true");
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= args.length) {
|
||||
System.err.println("缺少参数值: " + arg);
|
||||
System.exit(2);
|
||||
}
|
||||
options.put(name, args[++i]);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private static void printUsage() {
|
||||
System.out.println("""
|
||||
用法: java -cp <sqlite-jdbc.jar> GalleryIndexMaintenance.java --db <cache.db> [--apply] [--allow-production]
|
||||
|
||||
--db <path> 必填,缓存库文件
|
||||
--apply 执行修复;不加则只做 dry-run 体检
|
||||
--allow-production 允许写入 /home/lionwebsite 下的数据库(默认拒绝)
|
||||
|
||||
退出码: 0 健康或修复成功 / 1 存在待修复项(dry-run) / 2 参数错误 /
|
||||
3 拒绝写生产路径 / 4 修复后仍不健康 / 5 执行异常
|
||||
""");
|
||||
}
|
||||
|
||||
private GalleryIndexMaintenance() { }
|
||||
|
||||
/** 一次体检结果。 */
|
||||
static final class Report {
|
||||
boolean hasHeaderTable;
|
||||
boolean hasPageTable;
|
||||
int headerRows;
|
||||
int pageRows;
|
||||
int duplicateHeaderGids;
|
||||
int duplicatePageGroups;
|
||||
final List<String> badGids = new ArrayList<>();
|
||||
final List<String> orphanGids = new ArrayList<>();
|
||||
final Map<String, String> badReasons = new LinkedHashMap<>();
|
||||
|
||||
boolean isHealthy() {
|
||||
return hasHeaderTable && hasPageTable && badGids.isEmpty() && orphanGids.isEmpty();
|
||||
}
|
||||
|
||||
String render(String title) {
|
||||
StringBuilder out = new StringBuilder();
|
||||
out.append("=== 图片索引").append(title).append("(").append(Instant.now()).append(")===\n");
|
||||
if (!hasHeaderTable || !hasPageTable) {
|
||||
out.append("缺少缓存表: gidToKey=").append(hasHeaderTable)
|
||||
.append(" ImageKeyCache=").append(hasPageTable).append('\n');
|
||||
return out.toString();
|
||||
}
|
||||
out.append("gidToKey 行数: ").append(headerRows).append('\n');
|
||||
out.append("ImageKeyCache 行数: ").append(pageRows).append('\n');
|
||||
out.append("重复 gid 组: ").append(duplicateHeaderGids).append('\n');
|
||||
out.append("重复 (gid,page) 组: ").append(duplicatePageGroups).append('\n');
|
||||
out.append("不完整或冲突 gid: ").append(badGids.size()).append('\n');
|
||||
int limit = 20;
|
||||
for (int i = 0; i < badGids.size() && i < limit; i++) {
|
||||
String gid = badGids.get(i);
|
||||
out.append(" - gid=").append(gid).append(' ')
|
||||
.append(badReasons.getOrDefault(gid, "")).append('\n');
|
||||
}
|
||||
if (badGids.size() > limit)
|
||||
out.append(" ... 其余 ").append(badGids.size() - limit).append(" 个略\n");
|
||||
out.append("孤儿页 key 的 gid: ").append(orphanGids.size()).append('\n');
|
||||
out.append(isHealthy() ? "结论: 健康\n" : "结论: 需要修复\n");
|
||||
return out.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,20 @@ public class AbstractMessage {
|
||||
|
||||
public static final byte SUBSCRIPTION_SNAPSHOT_MESSAGE = 9;
|
||||
|
||||
// ---- 协议 v2(可靠任务)。旧类型 0..9 的字段保持不变,不能新增字段。 ----
|
||||
|
||||
/** 能力协商:{@code {protocolVersion: 2, features: ["reliable-task-v2"]}}。 */
|
||||
public static final byte PROTOCOL_HELLO_MESSAGE = 10;
|
||||
|
||||
/** 带稳定 operationId 的任务命令(DOWNLOAD/DELETE)。 */
|
||||
public static final byte TASK_COMMAND_MESSAGE = 11;
|
||||
|
||||
/** 带 reportId + revision 的状态上报,替代旧的无确认上报。 */
|
||||
public static final byte TASK_REPORT_MESSAGE = 12;
|
||||
|
||||
/** 主站提交事务成功后的上报回执,节点据此清理快照。 */
|
||||
public static final byte TASK_REPORT_ACK_MESSAGE = 13;
|
||||
|
||||
public byte messageType;
|
||||
|
||||
public int messageId;
|
||||
|
||||
@@ -45,6 +45,10 @@ public class MessageCodec extends ByteToMessageCodec<AbstractMessage> {
|
||||
case AbstractMessage.MAINTAIN_MESSAGE -> objectMapper.readValue(metadata, MaintainMessage.class);
|
||||
case AbstractMessage.AVAILABLE_CHECK_MESSAGE -> objectMapper.readValue(metadata, AvailableCheckMessage.class);
|
||||
case AbstractMessage.SUBSCRIPTION_SNAPSHOT_MESSAGE -> objectMapper.readValue(metadata, SubscriptionSnapshotMessage.class);
|
||||
case AbstractMessage.PROTOCOL_HELLO_MESSAGE -> objectMapper.readValue(metadata, ProtocolHelloMessage.class);
|
||||
case AbstractMessage.TASK_COMMAND_MESSAGE -> objectMapper.readValue(metadata, TaskCommandMessage.class);
|
||||
case AbstractMessage.TASK_REPORT_MESSAGE -> objectMapper.readValue(metadata, TaskReportMessage.class);
|
||||
case AbstractMessage.TASK_REPORT_ACK_MESSAGE -> objectMapper.readValue(metadata, TaskReportAckMessage.class);
|
||||
default -> null;
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.lion.lionwebsite.Message;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** 协议 v2 能力协商(类型 10)。双方都以对方声明了 version>=2 为前提才启用新消息。 */
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class ProtocolHelloMessage extends AbstractMessage {
|
||||
{
|
||||
messageType = PROTOCOL_HELLO_MESSAGE;
|
||||
}
|
||||
|
||||
public static final int PROTOCOL_V2 = 2;
|
||||
|
||||
public static final String FEATURE_RELIABLE_TASK_V2 = "reliable-task-v2";
|
||||
|
||||
private int protocolVersion = PROTOCOL_V2;
|
||||
|
||||
private List<String> features = new ArrayList<>(List.of(FEATURE_RELIABLE_TASK_V2));
|
||||
|
||||
/** 对端是否声明了 v2 可靠任务能力。 */
|
||||
public boolean supportsReliableTasks() {
|
||||
return protocolVersion >= PROTOCOL_V2
|
||||
&& features != null && features.contains(FEATURE_RELIABLE_TASK_V2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lion.lionwebsite.Message;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 任务命令(类型 11)。
|
||||
*
|
||||
* <p>只有确认对端支持 v2 之后才发送。同一逻辑操作在重试/重连后必须复用同一个
|
||||
* {@link #operationId},节点据此返回原结果而不重复执行副作用。
|
||||
*/
|
||||
@Data
|
||||
public class TaskCommandMessage extends AbstractMessage {
|
||||
{
|
||||
messageType = TASK_COMMAND_MESSAGE;
|
||||
}
|
||||
|
||||
public static final String ACTION_DOWNLOAD = "DOWNLOAD";
|
||||
|
||||
public static final String ACTION_DELETE = "DELETE";
|
||||
|
||||
/** 稳定操作 ID:DOWNLOAD 与 DELETE 各自独立。 */
|
||||
private String operationId;
|
||||
|
||||
private String action;
|
||||
|
||||
private int gid;
|
||||
|
||||
private String name;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.lion.lionwebsite.Message;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* 上报回执(类型 13)。
|
||||
*
|
||||
* <p>只在任务行与回执已经提交之后才发送;重复上报(同一 reportId)也会得到回执,
|
||||
* 让节点停止重放。{@link #result} 0 表示已落库。
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class TaskReportAckMessage extends AbstractMessage {
|
||||
{
|
||||
messageType = TASK_REPORT_ACK_MESSAGE;
|
||||
}
|
||||
|
||||
public static final byte RESULT_SUCCESS = 0;
|
||||
|
||||
private String reportId;
|
||||
|
||||
private byte result;
|
||||
|
||||
public TaskReportAckMessage(String reportId, byte result) {
|
||||
this.reportId = reportId;
|
||||
this.result = result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.lion.lionwebsite.Message;
|
||||
|
||||
import com.lion.lionwebsite.Domain.GalleryTask;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 节点状态上报(类型 12)。
|
||||
*
|
||||
* <p>相比旧的无确认上报,这里带上单调递增的 {@link #revision} 与唯一 {@link #reportId}:
|
||||
* 主站据此丢弃乱序/重放的旧快照,并在事务提交后回 {@link TaskReportAckMessage},
|
||||
* 节点只有在收到匹配回执后才清理自己的快照。
|
||||
*/
|
||||
@Data
|
||||
public class TaskReportMessage extends AbstractMessage {
|
||||
{
|
||||
messageType = TASK_REPORT_MESSAGE;
|
||||
}
|
||||
|
||||
/** 本次上报的唯一 ID,重复投递同一份快照时保持不变。 */
|
||||
private String reportId;
|
||||
|
||||
/** 节点侧单调递增版本号;主站只接受比已记录更大的 revision。 */
|
||||
private long revision;
|
||||
|
||||
private GalleryTask[] galleryTasks;
|
||||
|
||||
/** gid → 产生该状态的下载 operationId,用于丢弃属于旧操作的迟到上报。 */
|
||||
private Map<Integer, String> operationIds = new LinkedHashMap<>();
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Dao.cache.ImageCacheMapper;
|
||||
import com.lion.lionwebsite.Domain.GidToKey;
|
||||
import com.lion.lionwebsite.Domain.ImageKeyCache;
|
||||
import com.lion.lionwebsite.Exception.GalleryIndexException;
|
||||
import com.lion.lionwebsite.Util.GalleryUtil;
|
||||
import com.lion.lionwebsite.Util.SingleFlight;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.transaction.support.TransactionOperations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 图片索引(gid 到每页 imgkey)的唯一读写入口。
|
||||
*
|
||||
* <p>这个类替换掉原先「读缓存 / 写缓存 / 回滚」分散在业务服务里、并且逐页 INSERT
|
||||
* 的写法。三件事在这里被固定下来:
|
||||
*
|
||||
* <ol>
|
||||
* <li><b>同 gid 合并</b>:并发请求同一本画廊时只有一个线程真的去抓上游,
|
||||
* 其余线程等待同一个结果(含失败),因此不会出现两个构建者互相删除对方写入的索引。</li>
|
||||
* <li><b>抓取在事务外</b>:上游 HTTP 可能耗时十几秒,绝不能占着 SQLite 写事务,
|
||||
* 否则整个缓存库的写入都会排队。</li>
|
||||
* <li><b>整组短事务发布</b>:一个事务里先删该 gid 的旧索引,再批量写入新索引。
|
||||
* 失败整体回滚,磁盘上仍保留旧的完整索引;只有写入成功这一个状态是对外可见的。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>依赖全部由构造函数注入:mapper 负责落库,operations 负责事务边界,
|
||||
* upstream 负责把 mpv 页解析成页 key 列表。测试可以直接给假的实现,
|
||||
* 不需要起 Spring 上下文,也不需要网络。
|
||||
*/
|
||||
public class GalleryIndexService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GalleryIndexService.class);
|
||||
|
||||
/** 单条 INSERT 携带的最大行数,避免 SQL 过长。 */
|
||||
static final int BATCH_SIZE = 200;
|
||||
|
||||
private final ImageCacheMapper mapper;
|
||||
private final TransactionOperations operations;
|
||||
private final TransactionOperations readOperations;
|
||||
private final GalleryKeyUpstream upstream;
|
||||
private final SingleFlight<String, GidToKey> builds = new SingleFlight<>();
|
||||
|
||||
public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations,
|
||||
GalleryKeyUpstream upstream) {
|
||||
this(mapper, operations, operations, upstream);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param operations 发布索引的写事务(删除加批量写入必须整体生效)
|
||||
* @param readOperations 读取索引的一致性快照事务;传 null 表示不额外开启读事务
|
||||
*/
|
||||
public GalleryIndexService(ImageCacheMapper mapper, TransactionOperations operations,
|
||||
TransactionOperations readOperations, GalleryKeyUpstream upstream) {
|
||||
if (mapper == null || operations == null || upstream == null)
|
||||
throw new IllegalArgumentException("mapper/operations/upstream 不能为空");
|
||||
this.mapper = mapper;
|
||||
this.operations = operations;
|
||||
this.readOperations = readOperations;
|
||||
this.upstream = upstream;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保证该画廊的索引可用,返回 gid 到 key 的映射。
|
||||
*
|
||||
* @param url 画廊链接,形如 https://exhentai.org/g/1596929/f08534d87d/
|
||||
* @throws GalleryIndexException 链接错误 / 画廊不可用 / 暂时失败,见 GalleryIndexException.Reason
|
||||
*/
|
||||
public GidToKey ensureIndex(String url) throws GalleryIndexException {
|
||||
return ensureIndex(url, this::buildAndPublish);
|
||||
}
|
||||
|
||||
/**
|
||||
* 同 ensureIndex,但允许调用方提供构建动作。
|
||||
*
|
||||
* <p>离线重建需要「先检查再决定是否重建」的不同策略,因此把构建步骤作为参数暴露出来;
|
||||
* 生产调用使用默认实现即可。
|
||||
*/
|
||||
GidToKey ensureIndex(String url, IndexBuilder builder) throws GalleryIndexException {
|
||||
GalleryLink link = GalleryLink.parse(url);
|
||||
GidToKey cached = lookup(link);
|
||||
if (cached != null)
|
||||
return cached;
|
||||
try {
|
||||
return builds.run(link.gid(), () -> {
|
||||
// 进入临界区后必须再看一眼:等待期间别的线程可能已经把索引建好了。
|
||||
GidToKey existing = lookup(link);
|
||||
return existing != null ? existing : builder.build(url);
|
||||
});
|
||||
} catch (GalleryIndexException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT,
|
||||
"构建图片索引失败 gid=" + link.gid(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 已缓存且完整则返回索引,否则返回 null。
|
||||
*
|
||||
* <p>完整性由页码集合判定,而不是行数:历史重复行与半截写入都会让行数与页数不符。
|
||||
*/
|
||||
GidToKey lookup(GalleryLink link) {
|
||||
if (readOperations != null)
|
||||
return readOperations.execute(status -> lookupInTransaction(link));
|
||||
return lookupInTransaction(link);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在同一个读事务里读头部与页码,得到一个一致的快照。
|
||||
*
|
||||
* <p>两个独立的 SELECT 之间可能夹着另一个线程的整组替换,于是头部是旧的、页码是新的
|
||||
* (或相反),完整性判断会得出错误结论。SQLite 的读事务能看到固定快照,避免这种交错。
|
||||
*/
|
||||
private GidToKey lookupInTransaction(GalleryLink link) {
|
||||
GidToKey header = mapper.selectKeyByGid(link.gid());
|
||||
if (header == null)
|
||||
return null;
|
||||
if (header.getPages() <= 0 || header.getKey() == null || header.getKey().isBlank())
|
||||
return null;
|
||||
if (!isComplete(header))
|
||||
return null;
|
||||
return header;
|
||||
}
|
||||
|
||||
private boolean isComplete(GidToKey header) {
|
||||
List<Integer> pages = mapper.selectPageNumbersByGid(header.getGid());
|
||||
if (pages == null || pages.size() != header.getPages())
|
||||
return false;
|
||||
if (mapper.countInvalidImageKeysByGid(header.getGid()) != 0)
|
||||
return false;
|
||||
// 页码必须是严格递增的 1..pages,重复或跳号都视为不完整。
|
||||
for (int i = 0; i < pages.size(); i++) {
|
||||
Integer page = pages.get(i);
|
||||
if (page == null || page != i + 1)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 抓上游(事务外)后短事务整组替换。 */
|
||||
private GidToKey buildAndPublish(String url) throws GalleryIndexException {
|
||||
GalleryLink link = GalleryLink.parse(url);
|
||||
List<ImageKeyCache> pageKeys;
|
||||
try {
|
||||
// 抓取必须发生在事务之外:上游可能耗时十几秒,占着 SQLite 写事务会拖垮整个缓存库。
|
||||
pageKeys = upstream.fetchPageKeys(url);
|
||||
} catch (Exception e) {
|
||||
throw new GalleryIndexException(GalleryIndexException.Reason.TRANSIENT,
|
||||
"抓取图片索引失败 gid=" + link.gid(), e);
|
||||
}
|
||||
if (pageKeys == null || pageKeys.isEmpty())
|
||||
throw new GalleryIndexException(GalleryIndexException.Reason.UNAVAILABLE,
|
||||
"该图片已下架或已被删除 gid=" + link.gid());
|
||||
|
||||
// 落库前先校验上游数据自洽:整组替换一旦开始就没有回头路,
|
||||
// 与其把畸形页码写进唯一约束再回滚,不如在此之前就判为失败。
|
||||
validatePageKeys(link, pageKeys);
|
||||
|
||||
GidToKey header = new GidToKey();
|
||||
header.setGid(link.gid());
|
||||
header.setKey(link.key());
|
||||
header.setPages(pageKeys.size());
|
||||
|
||||
publish(header, pageKeys);
|
||||
return header;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验上游返回的页 key 是否自洽。
|
||||
*
|
||||
* <p>四条要求缺一不可,否则索引会以「看起来完整」的样子写进库:
|
||||
* gid 属于请求的那本画廊、页码恰好是 1..N、页码不重复、imgkey 非空。
|
||||
* 任一不满足都抛出 {@link GalleryIndexException.Reason#MALFORMED},此时尚未开启写事务,
|
||||
* 库中旧索引保持原样。
|
||||
*/
|
||||
void validatePageKeys(GalleryLink link, List<ImageKeyCache> pageKeys) throws GalleryIndexException {
|
||||
int expected = pageKeys.size();
|
||||
Set<Integer> seen = new HashSet<>(expected * 2);
|
||||
for (ImageKeyCache row : pageKeys) {
|
||||
if (row == null || row.getGid() == null || !row.getGid().equals(link.gid()))
|
||||
throw malformed(link, "上游页 key 的 gid 与请求不一致");
|
||||
if (row.getImgkey() == null || row.getImgkey().isBlank())
|
||||
throw malformed(link, "上游页 key 的 imgkey 为空");
|
||||
int page = row.getPage();
|
||||
if (page < 1 || page > expected)
|
||||
throw malformed(link, "上游页码越界: " + page + "(共 " + expected + " 页)");
|
||||
if (!seen.add(page))
|
||||
throw malformed(link, "上游页码重复: " + page);
|
||||
}
|
||||
if (seen.size() != expected)
|
||||
throw malformed(link, "上游页码不连续,去重后 " + seen.size() + " 个,期望 " + expected + " 个");
|
||||
}
|
||||
|
||||
private GalleryIndexException malformed(GalleryLink link, String detail) {
|
||||
log.warn("上游图片索引非法 gid={} detail={}", link.gid(), detail);
|
||||
return new GalleryIndexException(GalleryIndexException.Reason.MALFORMED,
|
||||
detail + " gid=" + link.gid());
|
||||
}
|
||||
|
||||
/**
|
||||
* 在单个事务里整组替换索引。
|
||||
*
|
||||
* <p>失败时事务整体回滚:要么看到旧的完整索引,要么看到新的完整索引,
|
||||
* 不存在「头部已更新、页 key 缺失」的中间态。
|
||||
*/
|
||||
void publish(GidToKey header, List<ImageKeyCache> pageKeys) {
|
||||
operations.executeWithoutResult(status -> {
|
||||
mapper.deleteImageKeyCacheByGid(header.getGid());
|
||||
mapper.deleteGidToKey(header.getGid());
|
||||
mapper.insertGidToKey(header);
|
||||
writeBatches(pageKeys);
|
||||
});
|
||||
log.info("图片索引已整体替换 gid={} pages={}", header.getGid(), header.getPages());
|
||||
}
|
||||
|
||||
private void writeBatches(List<ImageKeyCache> pageKeys) {
|
||||
for (int start = 0; start < pageKeys.size(); start += BATCH_SIZE) {
|
||||
int end = Math.min(start + BATCH_SIZE, pageKeys.size());
|
||||
mapper.insertImageKeyCacheBatch(new ArrayList<>(pageKeys.subList(start, end)));
|
||||
}
|
||||
}
|
||||
|
||||
/** 便于日志与错误信息使用的解析结果。 */
|
||||
record GalleryLink(String gid, String key) {
|
||||
|
||||
static GalleryLink parse(String url) throws GalleryIndexException {
|
||||
String[] segments = url == null ? null : url.split("/");
|
||||
Integer gid = GalleryUtil.parseGid(url);
|
||||
if (gid == null || segments == null || segments.length <= 5 || segments[5].isBlank())
|
||||
throw new GalleryIndexException(GalleryIndexException.Reason.INVALID_LINK,
|
||||
"链接错误");
|
||||
return new GalleryLink(String.valueOf(gid), segments[5].strip());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上游取页 key 的端口。
|
||||
*
|
||||
* <p>生产实现只是把 URL 转交给 GalleryUtil.parseImageKeys;
|
||||
* 单独抽出来是为了让本服务在测试里完全脱离网络与静态工具类。
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface GalleryKeyUpstream {
|
||||
List<ImageKeyCache> fetchPageKeys(String url) throws Exception;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建索引的动作。允许抛出业务异常,因此不用 java.util.function.Function
|
||||
* (后者不能抛受检异常,会让调用方被迫包装成运行时异常)。
|
||||
*/
|
||||
@FunctionalInterface
|
||||
interface IndexBuilder {
|
||||
GidToKey build(String url) throws GalleryIndexException;
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,16 @@ public class GalleryManageService {
|
||||
|
||||
final PushService pushService;
|
||||
|
||||
/**
|
||||
* 任务生命周期(协议 v2)。创建/删除都先走它做持久化意图,再产生外部副作用。
|
||||
* 用字段注入是为了兼容现有构造式 mock 测试与旧调用点。
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
GalleryTaskLifecycleService lifecycleService;
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
GalleryIndexService galleryIndexService;
|
||||
|
||||
/**
|
||||
* 创建任务
|
||||
* @param link 任务链接
|
||||
@@ -79,6 +89,20 @@ public class GalleryManageService {
|
||||
pushService.taskCreateReport(user.getUsername(), taskName, response);
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
// 协议 v2 生命周期:gid 唯一意图先落库(占位行「提交中」),提交前持久化 SUBMITTING,
|
||||
// 上游结果不确定时报 UNKNOWN 而不是自动重发;差额额度与任务行在同一短事务里写入。
|
||||
if (lifecycleService != null) {
|
||||
response = lifecycleService.createTask(gid, taskName, link, targetResolution, user.getId());
|
||||
Gallery current = galleryMapper.selectGalleryByGid(gid);
|
||||
taskName = current == null || current.getName() == null ? taskName : current.getName();
|
||||
// 业务结果由生命周期服务给出,data 用结构化对象下发(前端解码器兼容旧字符串)。
|
||||
if (response.isSuccess() && current != null)
|
||||
response.success(objectMapper.valueToTree(current));
|
||||
pushService.taskCreateReport(user.getUsername(), taskName, response);
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
Gallery gallery;
|
||||
|
||||
//判断数据库中是否有这个任务,有则返回状态
|
||||
@@ -124,7 +148,7 @@ public class GalleryManageService {
|
||||
|
||||
// Do not overwrite an immediate node status with the original submitted state.
|
||||
Gallery current = galleryMapper.selectGalleryByGid(gallery.getGid());
|
||||
response.success((current == null ? gallery : current).toString());
|
||||
response.success(objectMapper.valueToTree(current == null ? gallery : current));
|
||||
pushService.taskCreateReport(user.getUsername(), taskName, response);
|
||||
|
||||
return response.toJSONString();
|
||||
@@ -184,7 +208,7 @@ public class GalleryManageService {
|
||||
try {
|
||||
gallery = GalleryUtil.parse(link, false, null);
|
||||
if (gallery != null)
|
||||
response.success(objectMapper.valueToTree(gallery).toString());
|
||||
response.success(objectMapper.valueToTree(gallery));
|
||||
else
|
||||
response.failure("查询失败");
|
||||
} catch (Exception e) {
|
||||
@@ -193,7 +217,7 @@ public class GalleryManageService {
|
||||
}
|
||||
|
||||
else
|
||||
response.success(objectMapper.valueToTree(gallery).toString());
|
||||
response.success(objectMapper.valueToTree(gallery));
|
||||
|
||||
return response.toJSONString();
|
||||
}
|
||||
@@ -212,7 +236,7 @@ public class GalleryManageService {
|
||||
response.failure("未找到该图片,请使用链接");
|
||||
|
||||
else
|
||||
response.success(gallery.toString());
|
||||
response.success(objectMapper.valueToTree(gallery));
|
||||
|
||||
return response.toJSONString();
|
||||
}
|
||||
@@ -245,7 +269,7 @@ public class GalleryManageService {
|
||||
if (userId == UserService.ADMIN_USER_ID)
|
||||
fillDownloaderNames(galleries);
|
||||
|
||||
response.success(objectMapper.valueToTree(galleries).toString());
|
||||
response.success(objectMapper.valueToTree(galleries));
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
@@ -302,7 +326,7 @@ public class GalleryManageService {
|
||||
Gallery gallery = galleryMapper.selectGalleryByName("%" + name + "%");
|
||||
|
||||
if (gallery != null)
|
||||
response.success(objectMapper.valueToTree(gallery).toString());
|
||||
response.success(objectMapper.valueToTree(gallery));
|
||||
else
|
||||
response.failure("没有找到该名字的图片");
|
||||
|
||||
@@ -320,7 +344,7 @@ public class GalleryManageService {
|
||||
|
||||
Gallery[] galleries = galleryMapper.selectGalleryByDownloader(userMapper.selectUserByAuthCode(AuthCode).getId());
|
||||
if (galleries.length > 0)
|
||||
response.success(objectMapper.valueToTree(galleries).toString());
|
||||
response.success(objectMapper.valueToTree(galleries));
|
||||
else
|
||||
response.failure("您未下载图片");
|
||||
|
||||
@@ -358,8 +382,46 @@ public class GalleryManageService {
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
// 通过授权后才落库并通知节点,避免被拒请求仍删除节点文件。
|
||||
// 通过授权后才产生外部副作用。协议 v2 的删除顺序:
|
||||
// 先落「待删除」意图 → 节点确认(或文件已不存在)→ 才在同一事务删任务行并留墓碑。
|
||||
// 失败时任务行与意图都保留,可沿原记录重试并复用同一个删除 operationId。
|
||||
log.info("删除图片{}", gallery.getName());
|
||||
if (lifecycleService != null) {
|
||||
// 被拒绝(例如任务仍在提交中)时必须回业务失败:
|
||||
// 绝不能退避到裸删除,否则会与并发的创建 POST 同时执行并写回已删/重建的 gid。
|
||||
GalleryTaskLifecycleService.DeleteIntent intent = lifecycleService.beginDelete(gallery.getGid());
|
||||
if (!intent.accepted()) {
|
||||
response.failure(intent.message());
|
||||
log.info("拒绝删除 gid={}:{}", gid, intent.message());
|
||||
return response.toJSONString();
|
||||
}
|
||||
byte result = remoteService.sendDeleteCommand(intent.deleteOperationId(),
|
||||
gallery.getGid(), gallery.getName());
|
||||
switch (result) {
|
||||
case 0 -> {
|
||||
lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId());
|
||||
response.success();
|
||||
}
|
||||
// 文件已不存在等于目标达成:同样完成本地清理,避免留下永远删不掉的任务行。
|
||||
case ErrorCode.FILE_NOT_FOUND -> {
|
||||
lifecycleService.completeDelete(gallery.getGid(), intent.deleteOperationId());
|
||||
response.success();
|
||||
}
|
||||
case ErrorCode.IO_ERROR -> {
|
||||
lifecycleService.failDelete(gallery.getGid());
|
||||
response.failure("图片:" + gallery.getName() + "删除失败,IO错误");
|
||||
}
|
||||
// 节点无响应/超时:保留记录与待删除意图,可重试。
|
||||
default -> {
|
||||
lifecycleService.failDelete(gallery.getGid());
|
||||
response.failure("图片:" + gallery.getName() + "删除失败,节点无响应");
|
||||
}
|
||||
}
|
||||
if (!response.isSuccess())
|
||||
log.info(response.getData());
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
galleryMapper.deleteGalleryByGid(gallery.getGid());
|
||||
switch (remoteService.deleteGallery(gallery)) {
|
||||
case ErrorCode.IO_ERROR -> response.failure("图片:" + gallery.getName() + "删除失败,IO错误");
|
||||
@@ -399,76 +461,25 @@ public class GalleryManageService {
|
||||
lastResetAmountTime == null || lastResetAmountTime.getValue() == null
|
||||
? "" : lastResetAmountTime.getValue());
|
||||
|
||||
response.success(objectMapper.valueToTree(data).toString());
|
||||
response.success(objectMapper.valueToTree(data));
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
public String cacheImagesKey(String url) {
|
||||
Response response = Response.generateResponse();
|
||||
|
||||
// 畸形链接在这里就拦掉:段数不足会让 url.split("/")[5] 抛 ArrayIndexOutOfBoundsException,
|
||||
// 而 parseGid 也会对无 /g/ 的链接返回 null,两者都不该变成 500。
|
||||
String[] segments = url == null ? null : url.split("/");
|
||||
Integer parsedGid = parseGidFromLink(url);
|
||||
if (parsedGid == null || segments == null || segments.length <= 5) {
|
||||
response.failure("链接错误");
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
String gid = String.valueOf(parsedGid);
|
||||
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
|
||||
//已缓存过且页 key 完整,直接返回
|
||||
if(gidToKey != null && isIndexComplete(gidToKey)) {
|
||||
return response.success(objectMapper.valueToTree(gidToKey)).toJSONString();
|
||||
}
|
||||
// 半截缓存(历史崩溃或旧版本写入中断)先清干净再整体重建,否则残留的页 key 会重复累积。
|
||||
if (gidToKey != null) {
|
||||
log.warn("图片索引缓存不完整,重建 gid={} 记录页数={}", gid, gidToKey.getPages());
|
||||
rollbackImageIndex(gid);
|
||||
}
|
||||
|
||||
boolean inserted = false;
|
||||
try {
|
||||
gidToKey = new GidToKey();
|
||||
gidToKey.setGid(gid);
|
||||
gidToKey.setKey(segments[5].strip());
|
||||
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
|
||||
if(imageKeyCaches == null)
|
||||
return response.failure("该图片已下架或已被删除").toJSONString();
|
||||
gidToKey.setPages(imageKeyCaches.size());
|
||||
imageCacheMapper.insertGidToKey(gidToKey);
|
||||
inserted = true;
|
||||
for (ImageKeyCache imageKeyCache : imageKeyCaches)
|
||||
imageCacheMapper.insertImageKeyCache(imageKeyCache);
|
||||
response.success(objectMapper.valueToTree(gidToKey));
|
||||
}catch (Exception e){
|
||||
// 索引必须整体生效:先写 gidToKey 再逐页写 key,中途失败会留下「gidToKey 命中、
|
||||
// 页 key 缺失」的半截缓存,后续请求直接返回已缓存而永远取不到图。
|
||||
// 因此任何异常都回滚已写入的部分再回业务失败。Jackson 3 的解析异常继承
|
||||
// RuntimeException 而非 IOException,只用 IOException 会漏掉,这里统一兜住。
|
||||
if (inserted)
|
||||
rollbackImageIndex(gid);
|
||||
log.warn("缓存图片索引失败 gid={} errorType={}", gid, e.getClass().getSimpleName());
|
||||
response.failure("网络波动或其他异常");
|
||||
}
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
/** 页 key 行数与 gidToKey 记录不一致时视为半截缓存,需要重建。 */
|
||||
private boolean isIndexComplete(GidToKey gidToKey) {
|
||||
Integer pages = gidToKey.getPages();
|
||||
if (pages == null || pages <= 0)
|
||||
return false;
|
||||
return imageCacheMapper.countImageKeyCacheByGid(gidToKey.getGid()) == pages;
|
||||
}
|
||||
|
||||
/** 删除某个 gid 已写入的索引(页 key 与 gidToKey),用于回滚或重建。 */
|
||||
private void rollbackImageIndex(String gid) {
|
||||
try {
|
||||
imageCacheMapper.deleteImageKeyCacheByGid(gid);
|
||||
imageCacheMapper.deleteGidToKey(gid);
|
||||
} catch (Exception e) {
|
||||
log.warn("回滚图片索引缓存失败 gid={}", gid, e);
|
||||
GidToKey index = galleryIndexService.ensureIndex(url);
|
||||
return Response.generateResponse().success(objectMapper.valueToTree(index)).toJSONString();
|
||||
} catch (com.lion.lionwebsite.Exception.GalleryIndexException error) {
|
||||
String message = switch (error.getReason()) {
|
||||
case INVALID_LINK -> "链接错误";
|
||||
case UNAVAILABLE -> "该图片已下架或已被删除";
|
||||
case MALFORMED, TRANSIENT -> "网络波动或其他异常";
|
||||
};
|
||||
log.warn("缓存图片索引失败 reason={}", error.getReason());
|
||||
return Response._failure(message);
|
||||
} catch (RuntimeException error) {
|
||||
log.warn("缓存图片索引失败 errorType={}", error.getClass().getSimpleName());
|
||||
return Response._failure("网络波动或其他异常");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,10 +528,24 @@ public class GalleryManageService {
|
||||
|
||||
Gallery[] galleries = galleryMapper.selectUnDoneGalleries();
|
||||
if(galleries != null && galleries.length != 0) {
|
||||
log.info("发送未下载完成图片至节点,{}本", galleries.length);
|
||||
for (Gallery gallery : galleries)
|
||||
// 只重投「确实已提交上游」的任务:提交未知/预占中/待删除的任务重投会变成
|
||||
// 未计量副作用(节点侧显示下载完成但没有任何额度记录)。
|
||||
int sent = 0;
|
||||
int skipped = 0;
|
||||
for (Gallery gallery : galleries) {
|
||||
if (lifecycleService != null
|
||||
&& !lifecycleService.resolveDispatch(gallery.getGid()).dispatchable()) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
remoteService.addGalleryToQueue(gallery);
|
||||
response.success(String.format("发送未下载完成图片至节点,%s本", galleries.length));
|
||||
sent++;
|
||||
}
|
||||
log.info("发送未下载完成图片至节点,{}本(跳过需要对账的{}本)", sent, skipped);
|
||||
if (sent == 0)
|
||||
response.failure("没有可安全重投的任务,需先人工对账提交未知的任务");
|
||||
else
|
||||
response.success(String.format("发送未下载完成图片至节点,%s本", sent));
|
||||
}else{
|
||||
response.failure("当前没有未下载完成的图片");
|
||||
}
|
||||
@@ -536,6 +561,13 @@ public class GalleryManageService {
|
||||
return Response._success("下载完成");
|
||||
if(remoteService.isDead())
|
||||
return Response._failure("节点不在线,无法重试");
|
||||
// 提交结果未知(UNKNOWN)或正在删除的任务不能重试:上游不支持幂等,重发会让
|
||||
// 「可能已受理但未计量」的任务在节点侧变成下载完成却 0 额度。这类任务需人工对账。
|
||||
if (lifecycleService != null) {
|
||||
var decision = lifecycleService.resolveDispatch(gid);
|
||||
if (decision.kind() == GalleryTaskLifecycleService.DispatchDecision.Kind.REFUSE)
|
||||
return Response._failure("该任务提交结果未知或正在删除,请先人工对账后再重试");
|
||||
}
|
||||
|
||||
RemoteService.RetryResult retryResult = remoteService.retryGallery(gallery);
|
||||
if(retryResult.success())
|
||||
|
||||
@@ -0,0 +1,783 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Dao.normal.*;
|
||||
import com.lion.lionwebsite.Domain.*;
|
||||
import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
|
||||
import com.lion.lionwebsite.Message.TaskCommandMessage;
|
||||
import com.lion.lionwebsite.Message.TaskReportMessage;
|
||||
import com.lion.lionwebsite.Util.PreparedGallery;
|
||||
import com.lion.lionwebsite.Util.Response;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
|
||||
|
||||
/**
|
||||
* 下载任务的持久化生命周期(协议 v2)。
|
||||
*
|
||||
* <p>核心约束:
|
||||
* <ol>
|
||||
* <li><b>先保存意图,再产生外部副作用</b>:同一 gid 的唯一 CREATE 意图先落库(占位任务行 + RESERVED 操作记录),
|
||||
* 通过 gid 的主键唯一约束把并发创建收敛成「一个赢家」;</li>
|
||||
* <li><b>SUBMITTING 之后绝不自动重发</b>:上游不支持幂等,提交结果不确定时只能落 UNKNOWN,等显式对账/解决;</li>
|
||||
* <li><b>提交结果与额度用一个短事务写入</b>:Gallery 与 weekUsedAmount 要么一起成功,要么一起回滚,
|
||||
* 事务提交之后才把 DOWNLOAD 命令下发给节点;</li>
|
||||
* <li><b>删除失败保留记录</b>:先落待删除意图,节点确认(或文件已不存在)后才在同一事务里删任务行并留墓碑;
|
||||
* 失败则保留 Gallery 与 DELETE_PENDING 意图,可沿原记录重试且复用同一删除 operationId。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>所有网络调用都发生在事务之外;事务回调内部不做 I/O,也不吞异常(吞掉会错误提交)。
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class GalleryTaskLifecycleService implements TaskReportHandler {
|
||||
|
||||
final GalleryMapper galleryMapper;
|
||||
|
||||
final CollectMapper collectMapper;
|
||||
|
||||
final CustomConfigurationMapper configurationMapper;
|
||||
|
||||
final UserMapper userMapper;
|
||||
|
||||
final TaskOperationRepository taskOperationRepository;
|
||||
|
||||
final UpstreamGateway upstreamGateway;
|
||||
|
||||
final RemoteService remoteService;
|
||||
|
||||
/**
|
||||
* 短全局创建预占锁。
|
||||
*
|
||||
* <p>只有「登记唯一 CREATE 意图」这一小段数据库写入在锁内,网络调用一律在锁外。
|
||||
* 单进程内它把并发同 gid 创建直接收敛成一个赢家;跨进程的正确性仍由
|
||||
* {@code task_operation}/{@code gallery} 的主键唯一约束兜底。
|
||||
*/
|
||||
private final Object reservationLock = new Object();
|
||||
|
||||
/**
|
||||
* 测试钩子:进入上游只读解析之前执行。
|
||||
* 用来让并发删除发生在预占之后、SUBMITTING 之前。生产环境是空实现。
|
||||
*/
|
||||
volatile Runnable beforePrepare = () -> { };
|
||||
|
||||
/**
|
||||
* 测试钩子:SUBMITTING 已持久化之后、真正 POST 上游之前执行。
|
||||
* 用来制造「提交瞬间任务被删除」的 late generation 场景。生产环境是空实现。
|
||||
*/
|
||||
volatile Runnable afterMarkSubmitting = () -> { };
|
||||
|
||||
/** 主库事务模板;测试可注入内存 sqlite 的事务模板。 */
|
||||
TransactionTemplate transactions;
|
||||
|
||||
public GalleryTaskLifecycleService(GalleryMapper galleryMapper,
|
||||
CollectMapper collectMapper,
|
||||
CustomConfigurationMapper configurationMapper,
|
||||
UserMapper userMapper,
|
||||
TaskOperationRepository taskOperationRepository,
|
||||
UpstreamGateway upstreamGateway,
|
||||
RemoteService remoteService,
|
||||
@Qualifier("mainTransactionTemplate") TransactionTemplate transactions) {
|
||||
this.galleryMapper = galleryMapper;
|
||||
this.collectMapper = collectMapper;
|
||||
this.configurationMapper = configurationMapper;
|
||||
this.userMapper = userMapper;
|
||||
this.taskOperationRepository = taskOperationRepository;
|
||||
this.upstreamGateway = upstreamGateway;
|
||||
this.remoteService = remoteService;
|
||||
this.transactions = transactions;
|
||||
}
|
||||
|
||||
/** 测试用:替换事务模板或在没有 Spring 事务时退化为直连执行。 */
|
||||
void setTransactions(TransactionTemplate transactions) {
|
||||
this.transactions = transactions;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次「登记创建意图」的结果。
|
||||
*
|
||||
* <p>只有赢家能继续做外部副作用;输家(已存在有效意图 / 数据库竞争失败)
|
||||
* 一律转成业务失败,绝不再 POST 上游。
|
||||
*/
|
||||
private enum ReserveOutcome {
|
||||
/** 已成功登记唯一 CREATE 意图,可以做只读解析与提交。 */
|
||||
RESERVED,
|
||||
/** 同一 gid 已存在有效意图(或任务行)。 */
|
||||
DUPLICATE,
|
||||
/** 登记时的数据库竞争/失败:没有产生任何副作用,必须转业务失败。 */
|
||||
CONTENDED
|
||||
}
|
||||
|
||||
/** 一次「登记创建意图」的返回:结果 + 赢家持有的操作记录。 */
|
||||
private record Reservation(ReserveOutcome outcome, TaskOperation operation) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 单次下发决策。
|
||||
*
|
||||
* <p>把「能不能把这个任务交给节点」从「下载是否已提交」里拆出来:
|
||||
* 未确认上游提交结果(UNKNOWN / 预占 / 待删除 / 删除墓碑)绝不能下发,
|
||||
* 否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。
|
||||
*/
|
||||
public static final class DispatchDecision {
|
||||
|
||||
public enum Kind {
|
||||
/** 当前 operation 已是 SUBMITTED:可以按稳定 operationId 下发。 */
|
||||
READY,
|
||||
/** 迁移前的旧任务行:没有操作记录但历史上确实提交过下载。 */
|
||||
LEGACY,
|
||||
/** 不可下发:状态不明或被删除,下发会产生未计量副作用。 */
|
||||
REFUSE
|
||||
}
|
||||
|
||||
private final Kind kind;
|
||||
|
||||
private final String operationId;
|
||||
|
||||
private DispatchDecision(Kind kind, String operationId) {
|
||||
this.kind = kind;
|
||||
this.operationId = operationId;
|
||||
}
|
||||
|
||||
public Kind kind() {
|
||||
return kind;
|
||||
}
|
||||
|
||||
public String operationId() {
|
||||
return operationId;
|
||||
}
|
||||
|
||||
/** 只有 READY/LEGACY 允许真正下发到节点。 */
|
||||
public boolean dispatchable() {
|
||||
return kind != Kind.REFUSE && operationId != null;
|
||||
}
|
||||
|
||||
/** 拒绝下发(供调用方与测试构造)。 */
|
||||
public static DispatchDecision refuse() {
|
||||
return REFUSE;
|
||||
}
|
||||
|
||||
/** 可直接下发(供测试构造)。 */
|
||||
public static DispatchDecision ready(String operationId) {
|
||||
return new DispatchDecision(Kind.READY, operationId);
|
||||
}
|
||||
}
|
||||
|
||||
private static final DispatchDecision REFUSE = new DispatchDecision(DispatchDecision.Kind.REFUSE, null);
|
||||
|
||||
/**
|
||||
* 一次删除登记的返回。
|
||||
*
|
||||
* <p>用显式结果替代「null 即退避到裸删除」的旧语义:
|
||||
* 被拒绝时调用方必须回业务失败,不能绕过待删除意图直接删节点文件。
|
||||
*/
|
||||
public record DeleteIntent(boolean accepted, String deleteOperationId, String message) {
|
||||
|
||||
static DeleteIntent refused(String message) {
|
||||
return new DeleteIntent(false, null, message);
|
||||
}
|
||||
|
||||
static DeleteIntent accepted(String deleteOperationId) {
|
||||
return new DeleteIntent(true, deleteOperationId, null);
|
||||
}
|
||||
}
|
||||
|
||||
/** 在事务里执行;没有配置事务模板时候直接执行(仅便于构造 mock 测试)。 */
|
||||
private <T> T inTransaction(java.util.function.Supplier<T> body) {
|
||||
TransactionTemplate template = transactions;
|
||||
if (template == null)
|
||||
return body.get();
|
||||
return template.execute(status -> {
|
||||
try {
|
||||
return body.get();
|
||||
} catch (RuntimeException e) {
|
||||
// 回调里必须把异常继续抛出,否则 Spring 会把只读了一半的事务当成正常提交。
|
||||
status.setRollbackOnly();
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建任务的持久化前置:在同一个事务里登记唯一 CREATE 意图。
|
||||
*
|
||||
* <p>同时写入可见的占位任务行(状态「提交中」),这样提交期间任务列表就能解析该 gid,
|
||||
* 不会出现「上游已受理但本地查不到」的空窗。
|
||||
*
|
||||
* @return 已登记意图(赢家)返回它;同一 gid 已有有效意图返回 {@code null}(输家)。
|
||||
*/
|
||||
private Reservation reserveCreateIntent(int gid, String placeholderName, int downloaderId) {
|
||||
// 锁内只做数据库写入:把并发同 gid 创建串行化,网络调用留在锁外。
|
||||
synchronized (reservationLock) {
|
||||
try {
|
||||
TaskOperation operation = inTransaction(() -> {
|
||||
TaskOperation existingOperation = taskOperationRepository.selectByGid(gid);
|
||||
Gallery existingGallery = galleryMapper.selectGalleryByGid(gid);
|
||||
if (existingOperation != null) {
|
||||
// 只有「删除已完成」的墓碑才允许用全新 operationId 重新创建(显式重建)。
|
||||
if (!TaskOperation.STATE_DELETED.equals(existingOperation.getState()))
|
||||
return null;
|
||||
if (existingGallery != null)
|
||||
return null;
|
||||
String operationId = TaskProtocol.newDownloadOperationId();
|
||||
taskOperationRepository.reactivate(gid, operationId, TaskOperation.STATE_RESERVED, now());
|
||||
insertPlaceholder(gid, placeholderName, downloaderId);
|
||||
return taskOperationRepository.selectByGid(gid);
|
||||
}
|
||||
if (existingGallery != null)
|
||||
return null;
|
||||
TaskOperation created = new TaskOperation();
|
||||
created.setGid(gid);
|
||||
created.setOperationId(TaskProtocol.newDownloadOperationId());
|
||||
created.setState(TaskOperation.STATE_RESERVED);
|
||||
created.setUpdateTime(now());
|
||||
taskOperationRepository.insert(created);
|
||||
insertPlaceholder(gid, placeholderName, downloaderId);
|
||||
return created;
|
||||
});
|
||||
return operation == null
|
||||
? new Reservation(ReserveOutcome.DUPLICATE, null)
|
||||
: new Reservation(ReserveOutcome.RESERVED, operation);
|
||||
} catch (RuntimeException e) {
|
||||
// 数据库竞争/失败发生在任何外部副作用之前,绝不能穿透成 500,也不能继续提交。
|
||||
log.warn("登记创建意图失败,按业务失败处理 gid={} error={}", gid, e.getClass().getSimpleName());
|
||||
return new Reservation(ReserveOutcome.CONTENDED, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 写入占位任务行:让提交期间的任务在列表里可见。 */
|
||||
private void insertPlaceholder(int gid, String name, int downloaderId) {
|
||||
Gallery placeholder = new Gallery();
|
||||
placeholder.setGid(gid);
|
||||
placeholder.setName(name);
|
||||
placeholder.setStatus(TaskStatus.PENDING_SUBMIT);
|
||||
placeholder.setDownloader(downloaderId);
|
||||
placeholder.setCreateTime(System.currentTimeMillis() / 1000);
|
||||
placeholder.setProceeding(0);
|
||||
galleryMapper.insertGallery(placeholder);
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记为正在提交,并用 CAS 确认这一代意图仍然有效。
|
||||
*
|
||||
* <p>只有当前状态与 operationId 都匹配(RESERVED → SUBMITTING)才返回 true;
|
||||
* 期间被删除或重建返回 false,调用方必须停下,绝不能再 POST 上游。
|
||||
*
|
||||
* @return true 表示已持久化 SUBMITTING,可以安全产生上游副作用
|
||||
*/
|
||||
boolean markSubmitting(String operationId, int gid) {
|
||||
return inTransaction(() -> taskOperationRepository.compareAndSetCreateState(gid, operationId,
|
||||
TaskOperation.STATE_RESERVED, TaskOperation.STATE_SUBMITTING, now()) == 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交结果未知(超时/断连/中断)。保留意图,等待人工对账,不自动重发。
|
||||
*
|
||||
* <p>TaskOperation 与 Gallery 状态在同一个短事务里更新:
|
||||
* 操作落 UNKNOWN 而任务行仍显示「提交中」会让运营侧误以为还能自动重试,
|
||||
* 因此两边必须一致地表达「不确定,需要人工对账」。
|
||||
*/
|
||||
void markUnknown(String operationId, int gid) {
|
||||
inTransaction(() -> {
|
||||
int advanced = taskOperationRepository.compareAndSetCreateState(gid, operationId,
|
||||
TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now());
|
||||
// 只有本次这一代确实从 SUBMITTING 推进到 UNKNOWN 时才同步任务行:
|
||||
// 若期间已被删除/重建(CAS 返回 0),绝不能再改那条记录。
|
||||
if (advanced != 1)
|
||||
return null;
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
|
||||
if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus()))
|
||||
galleryMapper.updateGalleryStatus(gid, TaskStatus.UNKNOWN);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交成功后的「一次性短事务」:写入 Gallery 行、累计周用量、把操作推进为 SUBMITTED。
|
||||
* 三者同一个事务,任何一个失败都一起回滚,避免出现「任务存在但用量未计」这类不一致。
|
||||
*/
|
||||
/**
|
||||
* 提交成功后的落库(带世代校验)。
|
||||
*
|
||||
* @param expectedOperationId 本次提交所归属的 operationId;只有它仍是当前有效意图时才能落库,
|
||||
* 否则(被删除/重建)必须抛错,避免复活已删 gid 或污染新世代
|
||||
*/
|
||||
int persistSubmitted(Gallery gallery, int downloaderId, String expectedOperationId) {
|
||||
return inTransaction(() -> {
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(gallery.getGid());
|
||||
// 严格匹配本次世代:任务被删除、被重建或换了 operationId 时都不能落库。
|
||||
if (operation == null || TaskOperation.STATE_DELETED.equals(operation.getState())
|
||||
|| TaskOperation.STATE_DELETE_PENDING.equals(operation.getState()))
|
||||
throw new IllegalStateException("任务已删除或没有有效创建意图,禁止落库复活 gid=" + gallery.getGid());
|
||||
if (expectedOperationId == null || !expectedOperationId.equals(operation.getOperationId()))
|
||||
throw new IllegalStateException("创建意图世代不匹配,禁止落库 gid=" + gallery.getGid());
|
||||
// 幂等:同一意图重复落库(例如上游已提交后重跑)不得把用量累计两次。
|
||||
boolean alreadyCounted = TaskOperation.STATE_SUBMITTED.equals(operation.getState());
|
||||
gallery.setDownloader(downloaderId);
|
||||
gallery.set_download(true);
|
||||
if (!TaskStatus.SUBMITTED.equals(gallery.getStatus()))
|
||||
gallery.setStatus(TaskStatus.SUBMITTED);
|
||||
Gallery existing = galleryMapper.selectGalleryByGid(gallery.getGid());
|
||||
if (existing == null)
|
||||
galleryMapper.insertGallery(gallery);
|
||||
else
|
||||
galleryMapper.updateGallery(gallery);
|
||||
if (!alreadyCounted)
|
||||
configurationMapper.incrementConfiguration(CustomConfiguration.WEEK_USED_AMOUNT, gallery.getFileSize());
|
||||
taskOperationRepository.updateState(gallery.getGid(), TaskOperation.STATE_SUBMITTED, now());
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 预占式创建:登记 gid 唯一意图(提交任务行占位)→ 只读解析上游 → 标 SUBMITTING → 真正提交。
|
||||
*
|
||||
* <p>上游提交发生在事务之外,且只在 SUBMITTING 已持久化之后执行一次。网络异常一律落 UNKNOWN。
|
||||
*/
|
||||
public Response createTask(int gid, String placeholderName, String link, String targetResolution, int downloaderId) {
|
||||
Response response = Response.generateResponse();
|
||||
Reservation reservation = reserveCreateIntent(gid, placeholderName, downloaderId);
|
||||
if (reservation.outcome() == ReserveOutcome.DUPLICATE) {
|
||||
response.failure("任务队列已有此任务,请点击查找任务");
|
||||
return response;
|
||||
}
|
||||
if (reservation.outcome() == ReserveOutcome.CONTENDED) {
|
||||
// 预占阶段的数据库竞争不产生任何副作用;转业务失败,绝不下发上游、绝不穿透成 500。
|
||||
response.failure("任务提交失败,请稍后重试");
|
||||
return response;
|
||||
}
|
||||
|
||||
// 解析上游是只读步骤;真正的 POST 只在 SUBMITTING 持久化之后发生。
|
||||
// 生命周期约束:RESERVED 阶段被中断是安全的(可释放重来),SUBMITTING 之后一律按 UNKNOWN 处理。
|
||||
return submitPreparedIntent(gid, reservation.operation(), link, targetResolution, downloaderId);
|
||||
}
|
||||
|
||||
private Response submitPreparedIntent(int gid, TaskOperation operation, String link, String targetResolution,
|
||||
int downloaderId) {
|
||||
Response response = Response.generateResponse();
|
||||
beforePrepare.run();
|
||||
PreparedGallery prepared;
|
||||
try {
|
||||
prepared = upstreamGateway.prepare(link);
|
||||
} catch (IOException e) {
|
||||
log.warn("解析任务链接失败 gid={}", gid, e);
|
||||
releaseReservation(gid, operation.getOperationId());
|
||||
response.failure("IO错误,可能是网络波动");
|
||||
return response;
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("解析任务链接异常 gid={}", gid, e);
|
||||
releaseReservation(gid, operation.getOperationId());
|
||||
response.failure("提交任务失败,未知原因,请检查链接是否正常");
|
||||
return response;
|
||||
}
|
||||
if (prepared == null || prepared.gallery() == null) {
|
||||
releaseReservation(gid, operation.getOperationId());
|
||||
response.failure("提交任务失败,未知原因,请检查链接是否正常");
|
||||
return response;
|
||||
}
|
||||
|
||||
Gallery gallery = prepared.gallery();
|
||||
gallery.setGid(gid);
|
||||
|
||||
// CAS 确认这一代意图仍然有效(未被并发删除/重建),并把状态持久化为 SUBMITTING。
|
||||
// 只有成功之后才允许真正 POST;失败说明任务已不在当前生命周期里,直接放弃。
|
||||
if (!markSubmitting(operation.getOperationId(), gid)) {
|
||||
log.warn("创建意图已失效(可能被并发删除),放弃提交 gid={} operationId={}",
|
||||
gid, operation.getOperationId());
|
||||
response.failure("任务已被删除或状态已变化,请刷新任务列表");
|
||||
return response;
|
||||
}
|
||||
afterMarkSubmitting.run();
|
||||
|
||||
Gallery submitted;
|
||||
try {
|
||||
submitted = upstreamGateway.submit(prepared, targetResolution);
|
||||
} catch (ResolutionNotMatchException e) {
|
||||
releaseReservation(gid, operation.getOperationId());
|
||||
response.failure("提交失败,分辨率不存在");
|
||||
return response;
|
||||
} catch (IOException | RuntimeException e) {
|
||||
// 超时或断连时上游可能已经受理,无法确认:只能记 UNKNOWN,绝不自动重发。
|
||||
log.warn("上游提交结果未知 gid={} operationId={}", gid, operation.getOperationId(), e);
|
||||
markUnknown(operation.getOperationId(), gid);
|
||||
response.failure("上游提交结果未知,请稍后在任务列表中确认,不要重复提交");
|
||||
return response;
|
||||
}
|
||||
if (submitted == null || !TaskStatus.SUBMITTED.equals(submitted.getStatus())) {
|
||||
markUnknown(operation.getOperationId(), gid);
|
||||
response.failure("上游提交结果未知,请稍后在任务列表中确认,不要重复提交");
|
||||
return response;
|
||||
}
|
||||
submitted.setGid(gid);
|
||||
try {
|
||||
persistSubmitted(submitted, downloaderId, operation.getOperationId());
|
||||
} catch (RuntimeException e) {
|
||||
// 上游已受理但本地落库失败:保留 SUBMITTING 意图并标 UNKNOWN,任务行仍在列表里可对账。
|
||||
log.error("提交成功后落库失败 gid={} operationId={}", gid, operation.getOperationId(), e);
|
||||
markUnknown(operation.getOperationId(), gid);
|
||||
response.failure("任务已提交,但本地保存失败,请在任务列表中确认");
|
||||
return response;
|
||||
}
|
||||
// 只有事务提交之后才向节点下发命令;重复下发由节点按 operationId 去重。
|
||||
byte dispatchResult = dispatchDownload(gid, submitted.getName());
|
||||
Gallery current = galleryMapper.selectGalleryByGid(gid);
|
||||
Gallery payload = current == null ? submitted : current;
|
||||
if (dispatchResult == 0)
|
||||
response.success(objectMapper.valueToTree(payload));
|
||||
else
|
||||
response.failure("任务已保存,但节点未确认接收;请刷新任务列表后重试");
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交成功后的落库 + 下发。落库(含用量)在一个短事务里完成,事务提交后才向节点发命令。
|
||||
*
|
||||
* <p>先解析当前世代再落库:世代不存在(未提交过 / 已删除)时不落库也不下发。
|
||||
*
|
||||
* @return 已持久化的 Gallery;数据库失败或世代无效时抛出异常,调用方回业务失败且不下发节点。
|
||||
*/
|
||||
public Gallery persistAndDispatch(Gallery gallery, int downloaderId) {
|
||||
DispatchDecision decision = resolveDispatch(gallery.getGid());
|
||||
if (!decision.dispatchable())
|
||||
throw new IllegalStateException("任务没有可下发的有效世代 gid=" + gallery.getGid());
|
||||
persistSubmitted(gallery, downloaderId, decision.operationId());
|
||||
dispatchDownload(gallery.getGid(), gallery.getName());
|
||||
return gallery;
|
||||
}
|
||||
|
||||
/** 向节点下发 DOWNLOAD 命令(复用当前有效 operationId),返回节点结果码。 */
|
||||
public byte dispatchDownload(int gid, String name) {
|
||||
DispatchDecision decision = resolveDispatch(gid);
|
||||
if (!decision.dispatchable())
|
||||
return -1;
|
||||
return remoteService.sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(), gid, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个 gid 能否下发给节点,并为迁移前的旧任务补齐稳定 operationId。
|
||||
*
|
||||
* <p>规则:
|
||||
* <ul>
|
||||
* <li>v2 任务只有 {@code SUBMITTED} 可下发;{@code UNKNOWN} 必须人工对账,
|
||||
* 否则未计量的未知任务会在节点侧变成「下载完成却 0 额度」;</li>
|
||||
* <li>预占中的 {@code RESERVED}/{@code SUBMITTING}、待删除与删除墓碑都不下发;</li>
|
||||
* <li>没有任何操作记录、但任务行确实存在且状态已是「已提交」及之后的历史任务,
|
||||
* 补建一个稳定 operationId 后再下发(这是唯一的 legacy 兼容路径)。</li>
|
||||
* </ul>
|
||||
*/
|
||||
public DispatchDecision resolveDispatch(int gid) {
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(gid);
|
||||
if (operation != null && TaskOperation.STATE_SUBMITTED.equals(operation.getState())
|
||||
&& operation.getOperationId() != null)
|
||||
return new DispatchDecision(DispatchDecision.Kind.READY, operation.getOperationId());
|
||||
// 没有操作记录,或只有「旧世代」记录(缺 operationId):必须先确认任务行确实已提交过,
|
||||
// 再补一个稳定 operationId;UNKNOWN/预占/待删除一律拒绝。
|
||||
if (operation != null && !TaskOperation.STATE_SUBMITTED.equals(operation.getState()))
|
||||
return REFUSE;
|
||||
if (!isLegacySubmitted(gid))
|
||||
return REFUSE;
|
||||
String freshId = ensureStableOperationId(gid);
|
||||
return freshId == null ? REFUSE : new DispatchDecision(DispatchDecision.Kind.LEGACY, freshId);
|
||||
}
|
||||
|
||||
/** 迁移前的旧任务行:没有操作记录,但状态表明上游确实已受理过下载,可安全重投。 */
|
||||
private boolean isLegacySubmitted(int gid) {
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
|
||||
if (gallery == null || gallery.getStatus() == null)
|
||||
return false;
|
||||
return switch (gallery.getStatus()) {
|
||||
case TaskStatus.SUBMITTED, TaskStatus.DOWNLOADING, TaskStatus.WAITING_COMPRESS,
|
||||
TaskStatus.COMPRESSING -> true;
|
||||
// 提交未知 / 提交中 / 待删除 / 删除失败 一律不算已提交,不能下发。
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 为一个已经真正提交过的旧任务持久化稳定 operationId。
|
||||
*
|
||||
* <p>必须是短事务写入、先落库后返回,之后重投/重连才复用同一个 ID;
|
||||
* 用 {@code insert or ignore} + 回读,避免与并发创建互相覆盖。
|
||||
*
|
||||
* @return 稳定 operationId;无法安全建立时返回 null(调用方不下发)
|
||||
*/
|
||||
String ensureStableOperationId(int gid) {
|
||||
// 与创建预占共用同一把短锁(锁内只有数据库写),并发为同一 gid 补 ID 时只能有一个赢家。
|
||||
synchronized (reservationLock) {
|
||||
String candidate = TaskProtocol.newDownloadOperationId();
|
||||
return inTransaction(() -> {
|
||||
if (taskOperationRepository.selectByGid(gid) == null)
|
||||
taskOperationRepository.insertIfAbsent(gid, candidate, TaskOperation.STATE_SUBMITTED, now());
|
||||
else
|
||||
taskOperationRepository.assignOperationIdIfMissing(gid, candidate, now());
|
||||
TaskOperation current = taskOperationRepository.selectByGid(gid);
|
||||
if (current == null || current.getOperationId() == null)
|
||||
return null;
|
||||
// 只有确实处于 SUBMITTED 的世代才可下发;并发写入的其他状态一律拒绝。
|
||||
return TaskOperation.STATE_SUBMITTED.equals(current.getState()) ? current.getOperationId() : null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消预占(解析失败、分辨率不存在等:还没有产生任何上游副作用)。
|
||||
*
|
||||
* <p>只删除与本次创建 intent 匹配的 RESERVED 记录:若期间该 gid 已被删除并重建
|
||||
* (新的 operationId / 墓碑),这里绝不能把新世代记录抹掉。占位行同理只删「提交中」。
|
||||
*/
|
||||
private void releaseReservation(int gid, String operationId) {
|
||||
inTransaction(() -> {
|
||||
// 只删与本代 operationId 匹配且仍停在 RESERVED/SUBMITTING 的记录。
|
||||
int removed = taskOperationRepository.deleteReservationIfSameGeneration(gid, operationId);
|
||||
// 只有确实删掉了本代预占记录,才能删对应的占位行:
|
||||
// 若期间已出现新的 RESERVED 代,删除返回 0,此时绝不能误删新占位。
|
||||
if (removed == 1) {
|
||||
Gallery existing = galleryMapper.selectGalleryByGid(gid);
|
||||
if (existing != null && TaskStatus.PENDING_SUBMIT.equals(existing.getStatus()))
|
||||
galleryMapper.deleteGalleryByGid(gid);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除的持久化部分。
|
||||
*
|
||||
* <p>正在创建意图(RESERVED/SUBMITTING)的任务不能删除:此刻 DELETE 与创建 POST 会同时发生,
|
||||
* 随后创建落库还会把已删/重建的 gid 写回。这里直接拒绝,由用户在提交结果明确后再操作。
|
||||
*
|
||||
* @return 显式结果:被拒时调用方必须回业务失败,不能退避到裸删除
|
||||
*/
|
||||
public DeleteIntent beginDelete(int gid) {
|
||||
return inTransaction(() -> {
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
|
||||
if (gallery == null)
|
||||
return DeleteIntent.refused("删除失败,该图片不存在");
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(gid);
|
||||
if (operation != null && (TaskOperation.STATE_RESERVED.equals(operation.getState())
|
||||
|| TaskOperation.STATE_SUBMITTING.equals(operation.getState())))
|
||||
return DeleteIntent.refused("任务正在提交中,请等待提交结果明确后再删除");
|
||||
if (operation == null) {
|
||||
// 历史任务行没有操作记录:直接落一条 DELETE_PENDING 意图,避免中间态缺字段。
|
||||
TaskOperation fresh = new TaskOperation();
|
||||
fresh.setGid(gid);
|
||||
fresh.setOperationId(TaskProtocol.newDownloadOperationId());
|
||||
fresh.setDeleteOperationId(TaskProtocol.newDeleteOperationId());
|
||||
fresh.setState(TaskOperation.STATE_DELETE_PENDING);
|
||||
fresh.setUpdateTime(now());
|
||||
taskOperationRepository.insert(fresh);
|
||||
galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING);
|
||||
return DeleteIntent.accepted(fresh.getDeleteOperationId());
|
||||
}
|
||||
if (operation.getDeleteOperationId() == null) {
|
||||
operation.setDeleteOperationId(TaskProtocol.newDeleteOperationId());
|
||||
operation.setUpdateTime(now());
|
||||
}
|
||||
taskOperationRepository.updateDeleteIntent(gid, operation.getDeleteOperationId(),
|
||||
TaskOperation.STATE_DELETE_PENDING, now());
|
||||
galleryMapper.updateGalleryStatus(gid, TaskStatus.DELETE_PENDING);
|
||||
return DeleteIntent.accepted(operation.getDeleteOperationId());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除的完成部分:节点已确认(或文件已不存在)后,在一个短事务里删任务行、收藏并留墓碑。
|
||||
*
|
||||
* <p>事务提交前若失败,任务行与 DELETE_PENDING 意图都保留,可以继续重试。
|
||||
* 删除意图 ID 必须匹配(防止并发删除/重建时抹掉别的一代),随后只把 DELETE_PENDING
|
||||
* 世代推进为 DELETED;墓碑(DELETED 且无删除意图)不允许在这里被重新激活。
|
||||
*
|
||||
* @return true 表示本次删除意图已完成并留下墓碑
|
||||
*/
|
||||
public boolean completeDelete(int gid, String deleteOperationId) {
|
||||
return inTransaction(() -> {
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(gid);
|
||||
if (operation == null || operation.getDeleteOperationId() == null
|
||||
|| !operation.getDeleteOperationId().equals(deleteOperationId))
|
||||
return false;
|
||||
// 先清收藏再删任务行:收藏一侧在部分历史库上带有指向 gallery 的外键。
|
||||
collectMapper.deleteByGid(gid);
|
||||
galleryMapper.deleteGalleryByGid(gid);
|
||||
int updated = taskOperationRepository.completeDeleteIfSameGeneration(gid, deleteOperationId,
|
||||
TaskOperation.STATE_DELETED, now());
|
||||
return updated == 1;
|
||||
});
|
||||
}
|
||||
|
||||
/** 删除失败:保留任务行并记一条可重试的失败状态。 */
|
||||
public void failDelete(int gid) {
|
||||
inTransaction(() -> {
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(gid);
|
||||
if (gallery == null)
|
||||
return null;
|
||||
gallery.setStatus(TaskStatus.DELETE_FAILED);
|
||||
galleryMapper.updateGallery(gallery);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动恢复:把上次中断在 SUBMITTING 的意图判为 UNKNOWN,并安全释放还停在 RESERVED 的预占。
|
||||
*
|
||||
* <p>这段恢复必须发生在任何自动重投之前:
|
||||
* <ul>
|
||||
* <li>{@code SUBMITTING}:上游可能已经受理但本地无法确认,只能落 UNKNOWN(操作与任务行同时
|
||||
* 更新为「提交未知」),交人工对账,绝不自动再 POST;</li>
|
||||
* <li>{@code RESERVED}:还没有产生任何上游副作用,重启后可以直接释放预占
|
||||
* (删掉匹配的 RESERVED 记录与「提交中」占位行),让用户重新提交。</li>
|
||||
* </ul>
|
||||
*
|
||||
* @return 本次恢复(判未知 + 释放预占)处理的任务数
|
||||
*/
|
||||
public int recoverInterruptedSubmissions() {
|
||||
return inTransaction(() -> {
|
||||
int recovered = 0;
|
||||
for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_SUBMITTING)) {
|
||||
int advanced = taskOperationRepository.compareAndSetCreateState(operation.getGid(), operation.getOperationId(),
|
||||
TaskOperation.STATE_SUBMITTING, TaskOperation.STATE_UNKNOWN, now());
|
||||
if (advanced != 1)
|
||||
continue;
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid());
|
||||
if (gallery != null && !TaskStatus.UNKNOWN.equals(gallery.getStatus()))
|
||||
galleryMapper.updateGalleryStatus(operation.getGid(), TaskStatus.UNKNOWN);
|
||||
recovered++;
|
||||
}
|
||||
for (TaskOperation operation : taskOperationRepository.selectByState(TaskOperation.STATE_RESERVED)) {
|
||||
// 还没 POST:释放匹配的预占记录与占位行,允许重启后重新提交。
|
||||
// 只有确实删掉了这条本代 RESERVED 记录,才删其占位行;否则不动(可能已换代)。
|
||||
int removed = taskOperationRepository.deleteReservationIfSameGeneration(operation.getGid(),
|
||||
operation.getOperationId());
|
||||
if (removed == 1) {
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(operation.getGid());
|
||||
if (gallery != null && TaskStatus.PENDING_SUBMIT.equals(gallery.getStatus()))
|
||||
galleryMapper.deleteGalleryByGid(operation.getGid());
|
||||
}
|
||||
recovered++;
|
||||
}
|
||||
return recovered;
|
||||
});
|
||||
}
|
||||
|
||||
/** 已完成状态上报的处理:丢弃旧 revision、旧 operation、待删除任务与重复上报。 */
|
||||
@Override
|
||||
public CommittedReport apply(TaskReportMessage report) {
|
||||
if (report == null || report.getReportId() == null || report.getGalleryTasks() == null)
|
||||
return null;
|
||||
Applied applied;
|
||||
try {
|
||||
applied = inTransaction(() -> applyInTransaction(report));
|
||||
} catch (RuntimeException e) {
|
||||
// 数据库失败:不返回可回执结果,节点会保留快照并稍后重投;失败写入绝不清任务。
|
||||
log.warn("应用节点上报失败,未回执 reportId={}", report.getReportId(), e);
|
||||
return null;
|
||||
}
|
||||
if (applied == null)
|
||||
return null;
|
||||
return new CommittedReport(report.getReportId(), applied.newlyCompleted(), applied.acceptedTasks());
|
||||
}
|
||||
|
||||
/** 事务内应用上报的结果:新完成的任务与真正被接受的进度更新。 */
|
||||
private record Applied(List<Gallery> newlyCompleted, List<GalleryTask> acceptedTasks) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 在一个事务里应用上报。
|
||||
*
|
||||
* @return 非 null 表示已经提交(可以回 ACK,重复上报也会返回结果以停止重放)。
|
||||
*/
|
||||
private Applied applyInTransaction(TaskReportMessage report) {
|
||||
long revision = report.getRevision();
|
||||
List<Gallery> newlyCompleted = new ArrayList<>();
|
||||
List<GalleryTask> acceptedTasks = new ArrayList<>();
|
||||
for (GalleryTask task : report.getGalleryTasks()) {
|
||||
if (task == null)
|
||||
continue;
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(task.getGid());
|
||||
Gallery gallery = galleryMapper.selectGalleryByGid(task.getGid());
|
||||
if (gallery == null) {
|
||||
// 未知 gid(任务已删除):忽略但 ACK,让节点停止重放。
|
||||
log.debug("忽略节点上报的未知任务 gid={}", task.getGid());
|
||||
continue;
|
||||
}
|
||||
String expectedOperationId = report.getOperationIds() == null
|
||||
? null : report.getOperationIds().get(task.getGid());
|
||||
if (operation == null) {
|
||||
// 旧任务行(迁移前创建)没有操作记录:按显式生成策略补建并采纳上报里的 operationId。
|
||||
operation = adoptGeneration(task.getGid(), expectedOperationId);
|
||||
} else if (TaskOperation.STATE_DELETED.equals(operation.getState())) {
|
||||
// 墓碑必须永久忽略旧上报:删除已完成的任务只能由显式 CREATE 重新建立新世代。
|
||||
// (gallery 已被删除时上面的 null 分支已经提前忽略,这里防御性兜底。)
|
||||
log.debug("忽略已删除 gid 的迟到上报 gid={}", task.getGid());
|
||||
continue;
|
||||
} else if (TaskOperation.STATE_DELETE_PENDING.equals(operation.getState())) {
|
||||
log.info("忽略待删除任务的迟到上报 gid={}", task.getGid());
|
||||
continue;
|
||||
} else if (operation.getLastRevision() > 0 && revision <= operation.getLastRevision()) {
|
||||
log.debug("丢弃过期上报 gid={} revision={} lastRevision={}",
|
||||
task.getGid(), revision, operation.getLastRevision());
|
||||
continue;
|
||||
} else if (operation.getOperationId() != null) {
|
||||
// v2 任务必须有归属:缺 operationId 或与当前世代不符都不能改当前任务,
|
||||
// 否则「删除后重建」的新世代会被旧上报污染。
|
||||
if (expectedOperationId == null || !operation.getOperationId().equals(expectedOperationId)) {
|
||||
log.info("丢弃无法归属/旧世代的迟到上报 gid={} 上报={} 当前={}",
|
||||
task.getGid(), expectedOperationId, operation.getOperationId());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
String status = TaskStatus.fromReport(task.getStatus());
|
||||
if (status == null) {
|
||||
log.warn("未知上报状态码 gid={} status={}", task.getGid(), task.getStatus());
|
||||
continue;
|
||||
}
|
||||
gallery.setProceeding(task.getProceeding());
|
||||
if (task.getName() != null)
|
||||
gallery.setName(task.getName());
|
||||
boolean justCompleted = TaskStatus.COMPLETE.equals(status)
|
||||
&& !TaskStatus.COMPLETE.equals(gallery.getStatus());
|
||||
gallery.setStatus(status);
|
||||
galleryMapper.updateGallery(gallery);
|
||||
taskOperationRepository.updateReportReceipt(task.getGid(), revision, report.getReportId(), now());
|
||||
acceptedTasks.add(task);
|
||||
if (justCompleted)
|
||||
newlyCompleted.add(gallery);
|
||||
}
|
||||
return new Applied(newlyCompleted, acceptedTasks);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为迁移前创建、尚无操作记录的历史任务建立一次「世代」:优先采纳上报里的 operationId;
|
||||
* 旧节点不带 operationId 时保留 null,让后续同世代的进度上报仍能被接受。
|
||||
*
|
||||
* <p>不在这里凭空生成随机 ID:那会让第一份进度上报之后的所有旧上报都因「无法归属」被丢弃。
|
||||
* 真正的稳定 operationId 会在首次自动下发时用 {@code ensureStableOperationId} 落库,
|
||||
* 之后该世代即可被严格匹配。不会覆盖任何已有任务状态。
|
||||
*/
|
||||
private TaskOperation adoptGeneration(int gid, String reportedOperationId) {
|
||||
TaskOperation operation = new TaskOperation();
|
||||
operation.setGid(gid);
|
||||
operation.setOperationId(reportedOperationId);
|
||||
operation.setState(TaskOperation.STATE_SUBMITTED);
|
||||
operation.setUpdateTime(now());
|
||||
taskOperationRepository.insert(operation);
|
||||
return operation;
|
||||
}
|
||||
|
||||
private static long now() {
|
||||
return System.currentTimeMillis() / 1000;
|
||||
}
|
||||
|
||||
/** 供装配层解析当前有效 operationId(避免装配代码访问包私有字段)。 */
|
||||
public String currentOperationId(int gid) {
|
||||
TaskOperation operation = taskOperationRepository.selectByGid(gid);
|
||||
return operation == null ? null : operation.getOperationId();
|
||||
}
|
||||
}
|
||||
@@ -54,35 +54,16 @@ public class QueryService {
|
||||
}
|
||||
|
||||
Document parse = Jsoup.parse(result);
|
||||
Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr");
|
||||
|
||||
ArrayList<GalleryForQuery> galleries = new ArrayList<>();
|
||||
ListIterator<Element> elementListIterator = elements.listIterator();
|
||||
java.util.List<GalleryForQuery> galleries;
|
||||
try {
|
||||
elementListIterator.next(); //skip firstlink
|
||||
}catch (NullPointerException | NoSuchElementException e){
|
||||
response.failure("没有搜索到结果");
|
||||
return response.toJSONString();
|
||||
galleries = com.lion.lionwebsite.Util.GallerySearchParser.parse(parse);
|
||||
} catch (IOException | RuntimeException error) {
|
||||
log.warn("上游搜索页面格式不支持: {}", error.getClass().getSimpleName());
|
||||
return response.failure("上游搜索页面格式不支持").toJSONString();
|
||||
}
|
||||
if (galleries.isEmpty()) return response.failure("没有搜索到结果").toJSONString();
|
||||
|
||||
while (elementListIterator.hasNext()){
|
||||
Element element = elementListIterator.next();
|
||||
GalleryForQuery gallery = new GalleryForQuery();
|
||||
gallery.setType(element.child(0).text()); //type
|
||||
String src = element.child(1).select("img").attr("src");
|
||||
if (src.startsWith("data"))
|
||||
src = element.child(1).select("img").attr("data-src");
|
||||
src = src.replace("https://s.exhentai.org", "");
|
||||
gallery.setThumbnailUrl(src); //thumbnailSrc
|
||||
gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime
|
||||
gallery.setLink(element.child(2).child(0).attr("href")); //link
|
||||
gallery.setGid(GalleryUtil.parseGid(gallery.getLink()) + ""); //gid
|
||||
gallery.setName(element.child(2).child(0).child(0).text()); //name
|
||||
gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page
|
||||
galleries.add(gallery);
|
||||
}
|
||||
|
||||
response.success(objectMapper.valueToTree(galleries).toString());
|
||||
response.success(objectMapper.valueToTree(galleries));
|
||||
Elements nextLink = parse.select("#unext");
|
||||
if(nextLink.hasAttr("href"))
|
||||
response.set("next", nextLink.attr("href"));
|
||||
|
||||
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@Data
|
||||
@@ -76,12 +77,131 @@ public class RemoteService {
|
||||
|
||||
final SubscriptionStandbySnapshotService subscriptionStandbySnapshotService;
|
||||
|
||||
/**
|
||||
* 协议 v2 上报处理器。用 setter 注入而不是构造参数,避免 RemoteService 与
|
||||
* GalleryTaskLifecycleService 之间的依赖环(后者需要 RemoteService 下发命令)。
|
||||
*/
|
||||
volatile TaskReportHandler taskReportHandler;
|
||||
|
||||
/**
|
||||
* gid → 当前有效的下载 operationId。由生命周期服务在装配期注册(保持依赖方向单向)。
|
||||
* 未注册或返回 null 时,重投/重试退化为旧的无 operationId 命令。
|
||||
*/
|
||||
volatile java.util.function.IntFunction<String> downloadOperationIdResolver;
|
||||
|
||||
/**
|
||||
* gid → 下发判定(是否允许下发 + 用哪个稳定 operationId)。
|
||||
*
|
||||
* <p>替代「能解析出 operationId 就下发」的旧语义:UNKNOWN / 预占中 / 待删除的任务
|
||||
* 必须被明确拒绝,否则未计量的人工对账任务可能在节点侧变成「下载完成却 0 额度」。
|
||||
*/
|
||||
volatile java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> dispatchResolver;
|
||||
|
||||
/**
|
||||
* 生命周期装配是否已经完成(handler/resolver 已绑定 + 启动恢复已执行)。
|
||||
*
|
||||
* <p>在它为 true 之前,绝不自动重投未完成任务:启动早期还不知道哪些任务是
|
||||
* SUBMITTED、哪些是 UNKNOWN,盲目下发会制造未计量副作用。
|
||||
*/
|
||||
volatile boolean lifecycleReady;
|
||||
|
||||
/**
|
||||
* 启动恢复是否安全完成。恢复失败时拒绝自动重投,避免在状态不明的库上继续派发。
|
||||
*/
|
||||
volatile boolean recoverySafe;
|
||||
|
||||
/**
|
||||
* 节点是否声明了 v2 可靠任务能力。只有协商成功之后才发送类型 11..13,
|
||||
* 否则继续走旧的无确认上报,保持向后兼容。
|
||||
*/
|
||||
volatile boolean v2Negotiated;
|
||||
|
||||
/** 本次连接的 Hello 协商结果;收到节点 Hello(或超时)后完成。 */
|
||||
volatile CompletableFuture<Boolean> helloNegotiation;
|
||||
|
||||
/** 明确「未下发」的结果码:与 -1(发送失败/超时)区分开。 */
|
||||
public static final byte REFUSED = -3;
|
||||
|
||||
/** 事务提交之后的通知/推送执行器:外部延迟(Telegram、慢浏览器)不得阻塞状态提交。 */
|
||||
final ExecutorService notificationExecutor = Executors.newSingleThreadExecutor(r -> {
|
||||
Thread thread = new Thread(r, "gallery-notify");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
/**
|
||||
* 协议握手的专用线程。hello 等待响应是阻塞调用,绝不能放在 Netty 事件循环上,
|
||||
* 否则响应本身要由同一个事件循环投递,会直接死锁。
|
||||
*/
|
||||
final ExecutorService handshakeExecutor = Executors.newSingleThreadExecutor(r -> {
|
||||
Thread thread = new Thread(r, "node-protocol-handshake");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
final ExecutorService subscriptionSyncExecutor = Executors.newSingleThreadExecutor(r -> {
|
||||
Thread thread = new Thread(r, "subscription-standby-sync");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
/** 测试/装配用:绑定上报处理器。 */
|
||||
public void setTaskReportHandler(TaskReportHandler taskReportHandler) {
|
||||
this.taskReportHandler = taskReportHandler;
|
||||
}
|
||||
|
||||
public void setDownloadOperationIdResolver(java.util.function.IntFunction<String> resolver) {
|
||||
this.downloadOperationIdResolver = resolver;
|
||||
}
|
||||
|
||||
public void setDispatchResolver(java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> resolver) {
|
||||
this.dispatchResolver = resolver;
|
||||
}
|
||||
|
||||
/**
|
||||
* 由装配层在「绑定 handler/resolver + 完成启动恢复」之后调用:标记可以安全派发并建立连接。
|
||||
*
|
||||
* <p>初始连接不再放在 {@code @PostConstruct}:那时 handler/resolver 都还没绑定,
|
||||
* 连接成功会把尚未确认的 UNKNOWN/预占任务当作普通任务重投。
|
||||
*
|
||||
* @param recoverySafe 启动恢复是否成功;false 时只允许连接,不允许自动重投
|
||||
*/
|
||||
public void startAfterWiring(boolean recoverySafe) {
|
||||
this.recoverySafe = recoverySafe;
|
||||
this.lifecycleReady = true;
|
||||
initChannel();
|
||||
}
|
||||
|
||||
/** 把 gid 解析成下发判定;未注册解析器或解析失败时按「拒绝下发」处理。 */
|
||||
private GalleryTaskLifecycleService.DispatchDecision resolveDispatchDecision(int gid) {
|
||||
java.util.function.IntFunction<GalleryTaskLifecycleService.DispatchDecision> resolver = dispatchResolver;
|
||||
if (resolver == null)
|
||||
return null;
|
||||
try {
|
||||
return resolver.apply(gid);
|
||||
} catch (Exception e) {
|
||||
log.warn("解析任务下发判定失败 gid={}", gid, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveDownloadOperationId(int gid) {
|
||||
java.util.function.IntFunction<String> resolver = downloadOperationIdResolver;
|
||||
if (resolver == null)
|
||||
return null;
|
||||
try {
|
||||
return resolver.apply(gid);
|
||||
} catch (Exception e) {
|
||||
log.warn("解析任务 operationId 失败 gid={}", gid, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 节点是否已完成 v2 能力协商(供健康检查/测试观察)。 */
|
||||
public boolean isReliableProtocolReady() {
|
||||
return v2Negotiated;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理节点上报的任务状态。原先这些 JDBC 读写直接跑在 Netty 的 IO 线程上,
|
||||
* N 条上报会阻塞该 IO 线程,连带拖慢心跳与请求响应;改为单线程顺序执行,
|
||||
@@ -120,10 +240,14 @@ public class RemoteService {
|
||||
@Value("${subscription.standby.sync-enabled:false}")
|
||||
boolean subscriptionSyncEnabled;
|
||||
|
||||
@PostConstruct
|
||||
void init() {
|
||||
initChannel();
|
||||
}
|
||||
/**
|
||||
* 这里刻意不再自动连接。
|
||||
*
|
||||
* <p>原实现在 PostConstruct 里立即 initChannel(),而那一刻 TaskLifecycleWiring
|
||||
* 还没绑定上报处理器、operationId 解析器,也没执行启动恢复:连接成功后会马上把
|
||||
* 未完成任务(含 UNKNOWN/预占)当作普通任务重投给节点。
|
||||
* 现在改为由装配层在完成绑定与恢复后调用 startAfterWiring(boolean)。
|
||||
*/
|
||||
|
||||
public boolean initChannel(){
|
||||
if (stopping || !connecting.compareAndSet(false, true))
|
||||
@@ -175,12 +299,32 @@ public class RemoteService {
|
||||
channel = channelFuture.channel();
|
||||
closeMonitorSocket();
|
||||
channel.writeAndFlush(new IdentityMessage("lionwebsite"));
|
||||
|
||||
//子节点上线时,发送未完成的任务
|
||||
resetUndone();
|
||||
// 节点刚上线时可能没有任何快照(例如刚重启),必须推一次;
|
||||
// 这同时会刷新节点「最近收到快照」的时刻,避免内容未变时被判过期。
|
||||
requestSubscriptionSync();
|
||||
// 协议 v2:先发旧的 Identity,再发 hello 做能力协商。
|
||||
// 协商完成之前不允许下发任何任务,也不允许自动重投未完成任务。
|
||||
CompletableFuture<Boolean> negotiation = queueHelloHandshake();
|
||||
// 在协商线程上等待结果,再决定重投;绝不阻塞 Netty 事件循环。
|
||||
handshakeExecutor.execute(() -> {
|
||||
boolean reliable = awaitNegotiation(negotiation);
|
||||
// 订阅快照与任务重投都要求生命周期装配已完成;装配前只保持连接。
|
||||
requestSubscriptionSync();
|
||||
if (!reliable) {
|
||||
log.info("节点未确认 v2 能力,跳过自动重投未完成任务");
|
||||
return;
|
||||
}
|
||||
if (!lifecycleReady) {
|
||||
log.info("生命周期装配未就绪,跳过自动重投未完成任务");
|
||||
return;
|
||||
}
|
||||
if (!recoverySafe) {
|
||||
log.error("启动恢复未安全完成,拒绝自动重投未完成任务");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
resetUndone();
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("重投未完成任务失败", e);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}catch (Exception e){
|
||||
log.error("connect node failed, wait for node back online", e);
|
||||
@@ -297,6 +441,8 @@ public class RemoteService {
|
||||
failPendingRequests();
|
||||
if (channel != null) channel.close();
|
||||
statusApplyExecutor.shutdownNow();
|
||||
notificationExecutor.shutdownNow();
|
||||
handshakeExecutor.shutdownNow();
|
||||
subscriptionSyncExecutor.shutdownNow();
|
||||
networkGroup.shutdownGracefully();
|
||||
eventLoopGroup.shutdownGracefully();
|
||||
@@ -315,7 +461,11 @@ public class RemoteService {
|
||||
}
|
||||
|
||||
public void resetUndone(){
|
||||
if (channelFuture.channel() == null || !channelFuture.channel().isActive())
|
||||
// 只有「连接在线 + v2 已协商 + 生命周期装配完成 + 启动恢复安全完成」才允许批量重投;
|
||||
// 任何一项不满足都说明还不知道任务真实状态,盲目下发会制造未计量副作用。
|
||||
if (channelFuture == null || channelFuture.channel() == null || !channelFuture.channel().isActive())
|
||||
return;
|
||||
if (!v2Negotiated || !lifecycleReady || !recoverySafe)
|
||||
return;
|
||||
|
||||
Gallery[] galleries = galleryMapper.selectUnDoneGalleries();
|
||||
@@ -328,15 +478,135 @@ public class RemoteService {
|
||||
}
|
||||
|
||||
public byte addGalleryToQueue(Gallery gallery){
|
||||
GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid());
|
||||
if (decision != null) {
|
||||
// 明确拒绝(UNKNOWN/预占/待删除等):绝不能退化为无 operationId 的旧命令。
|
||||
if (!decision.dispatchable())
|
||||
return REFUSED;
|
||||
return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, decision.operationId(),
|
||||
gallery.getGid(), gallery.getName());
|
||||
}
|
||||
// 兼容:装配层未注册判定解析器时(如单元测试),退回只按 operationId 解析的旧行为。
|
||||
String operationId = resolveDownloadOperationId(gallery.getGid());
|
||||
if (operationId != null)
|
||||
return sendTaskCommand(TaskCommandMessage.ACTION_DOWNLOAD, operationId, gallery.getGid(), gallery.getName());
|
||||
return legacyDownloadCommand(gallery.getGid(), gallery.getName());
|
||||
}
|
||||
|
||||
/** 旧的无 operationId 下载命令,供未启用 v2 或无法解析 operationId 时退避使用。 */
|
||||
private byte legacyDownloadCommand(int gid, String name) {
|
||||
GalleryTask galleryTask = new GalleryTask();
|
||||
galleryTask.setGid(gallery.getGid());
|
||||
galleryTask.setName(gallery.getName());
|
||||
galleryTask.setGid(gid);
|
||||
galleryTask.setName(name);
|
||||
DownloadPostMessage message = new DownloadPostMessage();
|
||||
message.setGalleryTask(galleryTask);
|
||||
return sendRequest(message, 10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 在专用线程上执行 hello 协商,并返回一个在「收到节点 Hello」或「等待超时」后完成的 future。
|
||||
*
|
||||
* <p>节点的应答是 Hello 而不是 ResponseMessage,因此绝不能走 sendRequest 的响应等待器
|
||||
* (那会白等 10 秒超时,排在后面的自动重投也只能 10 秒后才开始)。
|
||||
* 这里直接 writeAndFlush,并让 channelRead 的 Hello 分支完成 future;
|
||||
* 超时用 completeOnTimeout 处理,写失败立刻完成。
|
||||
*/
|
||||
private CompletableFuture<Boolean> queueHelloHandshake() {
|
||||
// 10 秒内没等到节点 Hello 就按「未协商」完成,避免后续重投被无限期挂起。
|
||||
CompletableFuture<Boolean> result = new CompletableFuture<Boolean>()
|
||||
.completeOnTimeout(false, 10, TimeUnit.SECONDS);
|
||||
helloNegotiation = result;
|
||||
if (stopping) {
|
||||
result.complete(false);
|
||||
return result;
|
||||
}
|
||||
try {
|
||||
handshakeExecutor.execute(() -> {
|
||||
v2Negotiated = false;
|
||||
try {
|
||||
Channel target = channel;
|
||||
if (target == null || !target.isActive()) {
|
||||
result.complete(false);
|
||||
return;
|
||||
}
|
||||
// Hello 不写入 promiseHashMap:节点回的是 Hello,没有 ResponseMessage 等待者。
|
||||
target.writeAndFlush(new ProtocolHelloMessage()).addListener(future -> {
|
||||
if (!future.isSuccess()) {
|
||||
if (!stopping)
|
||||
log.warn("协议协商发送失败: {}", future.cause() == null
|
||||
? "unknown" : future.cause().getClass().getSimpleName());
|
||||
result.complete(false);
|
||||
}
|
||||
});
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("协议协商失败", e);
|
||||
result.complete(false);
|
||||
}
|
||||
});
|
||||
} catch (java.util.concurrent.RejectedExecutionException e) {
|
||||
if (!stopping) log.warn("安排协议协商失败", e);
|
||||
result.complete(false);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 等待协商结果;超时或中断都按「未协商」处理(宁可不下发,也不盲目重投)。 */
|
||||
boolean awaitNegotiation(CompletableFuture<Boolean> negotiation) {
|
||||
if (negotiation == null)
|
||||
return v2Negotiated;
|
||||
try {
|
||||
// future 自身已在 10 秒时按「未协商」完成,这里留少量余量即可。
|
||||
return negotiation.get(12, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 下发一条带稳定 operationId 的任务命令;未协商 v2 时退化为旧的 DownloadPostMessage,
|
||||
* 保证新旧节点组合仍可用(旧节点上无法保证可靠 ACK,这是已知的兼容边界)。
|
||||
*/
|
||||
public byte sendTaskCommand(String action, String operationId, int gid, String name) {
|
||||
if (!v2Negotiated)
|
||||
return TaskCommandMessage.ACTION_DELETE.equals(action)
|
||||
? legacyDeleteCommand(gid, name)
|
||||
: legacyDownloadCommand(gid, name);
|
||||
TaskCommandMessage command = new TaskCommandMessage();
|
||||
command.setOperationId(operationId);
|
||||
command.setAction(action);
|
||||
command.setGid(gid);
|
||||
command.setName(name);
|
||||
return sendRequest(command, 10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/** 删除命令:带独立稳定 operationId,重试复用同一个 ID。 */
|
||||
public byte sendDeleteCommand(String operationId, int gid, String name) {
|
||||
if (!v2Negotiated)
|
||||
return legacyDeleteCommand(gid, name);
|
||||
TaskCommandMessage command = new TaskCommandMessage();
|
||||
command.setOperationId(operationId);
|
||||
command.setAction(TaskCommandMessage.ACTION_DELETE);
|
||||
command.setGid(gid);
|
||||
command.setName(name);
|
||||
return sendRequest(command, 10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/** 旧的无 operationId 删除命令,供未协商 v2 时退避使用。 */
|
||||
private byte legacyDeleteCommand(int gid, String name) {
|
||||
Gallery gallery = new Gallery();
|
||||
gallery.setGid(gid);
|
||||
gallery.setName(name);
|
||||
return deleteGallery(gallery);
|
||||
}
|
||||
|
||||
public RetryResult retryGallery(Gallery gallery){
|
||||
// 提交结果未知或被删除的任务不能重试:节点可能已经把未计量任务当成新下载。
|
||||
GalleryTaskLifecycleService.DispatchDecision decision = resolveDispatchDecision(gallery.getGid());
|
||||
if (decision != null && !decision.dispatchable())
|
||||
return new RetryResult(false, "该任务提交结果未知或正在删除,需要人工对账后再重试");
|
||||
CompletableFuture<String> statusFuture = new CompletableFuture<>();
|
||||
retryStatusWaiters.computeIfAbsent(gallery.getGid(), ignored -> new CopyOnWriteArrayList<>())
|
||||
.add(statusFuture);
|
||||
@@ -447,6 +717,18 @@ public class RemoteService {
|
||||
}
|
||||
});
|
||||
}
|
||||
else if(msg instanceof ProtocolHelloMessage hello) {
|
||||
// 节点声明了 v2 可靠任务能力;此后才允许使用类型 11..13。
|
||||
v2Negotiated = hello.supportsReliableTasks();
|
||||
log.info("节点协议协商完成 v2={} version={}", v2Negotiated, hello.getProtocolVersion());
|
||||
CompletableFuture<Boolean> negotiation = helloNegotiation;
|
||||
if (negotiation != null)
|
||||
negotiation.complete(v2Negotiated);
|
||||
}
|
||||
else if(msg instanceof TaskReportMessage report) {
|
||||
// 事务提交与 ACK 都放在单线程消费队列里顺序执行,避免并发写库。
|
||||
statusApplyExecutor.execute(() -> consumeV2Report(report));
|
||||
}
|
||||
else if(msg instanceof ResponseMessage rsm) {
|
||||
Promise<AbstractMessage> promise = promiseHashMap.remove(rsm.messageId);
|
||||
if(promise != null)
|
||||
@@ -456,6 +738,55 @@ public class RemoteService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 消费协议 v2 上报:短事务提交 → 事务成功后回 ACK → 再用独立执行器做通知/推送。
|
||||
*
|
||||
* <p>顺序很关键:数据库失败时既不能回 ACK,也不能清任务;慢通知不能阻塞这里,
|
||||
* 否则会拖住后续上报的提交。
|
||||
*/
|
||||
private void consumeV2Report(TaskReportMessage report) {
|
||||
try {
|
||||
TaskReportHandler handler = taskReportHandler;
|
||||
if (handler == null) {
|
||||
log.warn("收到 v2 上报但未装配处理器,暂不回执 reportId={}", report.getReportId());
|
||||
return;
|
||||
}
|
||||
TaskReportHandler.CommittedReport committed = handler.apply(report);
|
||||
if (committed == null) {
|
||||
// 提交失败:不回 ACK,节点保留快照后重投。
|
||||
log.warn("v2 上报未提交,等待节点重投 reportId={}", report.getReportId());
|
||||
return;
|
||||
}
|
||||
// 提交成功之后才回执,让节点清理快照。
|
||||
Channel target = channel;
|
||||
if (target != null && target.isActive())
|
||||
target.writeAndFlush(new TaskReportAckMessage(committed.reportId(), TaskReportAckMessage.RESULT_SUCCESS));
|
||||
dispatchNotifications(report, committed);
|
||||
} catch (Exception e) {
|
||||
log.warn("处理 v2 上报失败 reportId={}", report.getReportId(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/** 通知/推送在独立执行器里执行,外部延迟不会阻塞状态提交。 */
|
||||
private void dispatchNotifications(TaskReportMessage report, TaskReportHandler.CommittedReport committed) {
|
||||
// 只推送事务里真正被接受的更新(含下载中/压缩中等中间状态);
|
||||
// 过期、旧世代、待删除任务已被过滤掉,不会把它们带到前端。
|
||||
GalleryTask[] accepted = committed.acceptedTasks() == null
|
||||
? new GalleryTask[0]
|
||||
: committed.acceptedTasks().toArray(new GalleryTask[0]);
|
||||
List<Gallery> newlyCompleted = committed.newlyCompleted();
|
||||
notificationExecutor.execute(() -> {
|
||||
try {
|
||||
for (Gallery gallery : newlyCompleted)
|
||||
pushService.downloadComplete(gallery);
|
||||
if (accepted.length > 0)
|
||||
webSocketService.updateTaskProcessing(accepted);
|
||||
} catch (Exception e) {
|
||||
log.warn("任务完成通知/推送失败", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** 把节点上报的任务状态写回库里并推送给前端,按上报顺序逐个处理。 */
|
||||
private void applyReportedStatus(GalleryTask[] galleryTasks) {
|
||||
for (GalleryTask galleryTask : galleryTasks) {
|
||||
|
||||
@@ -17,6 +17,10 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -39,6 +43,10 @@ import java.util.function.Supplier;
|
||||
final RemoteService remoteService;
|
||||
final SubscriptionStateCoordinator stateCoordinator;
|
||||
|
||||
@Autowired
|
||||
@Qualifier("mainTransactionTemplate")
|
||||
TransactionTemplate transactions;
|
||||
|
||||
/**
|
||||
* 归属地查询是同步外呼(最长十几秒),且失败只影响记录里的一列展示。
|
||||
* 若放在订阅分发路径上,用户取一次订阅就得先等它完成,因此改为后台补齐。
|
||||
@@ -48,11 +56,12 @@ import java.util.function.Supplier;
|
||||
Executor locationExecutor = newFixedLocationExecutor();
|
||||
|
||||
private static ExecutorService newFixedLocationExecutor() {
|
||||
return Executors.newFixedThreadPool(2, r -> {
|
||||
return new java.util.concurrent.ThreadPoolExecutor(2, 2, 0, java.util.concurrent.TimeUnit.MILLISECONDS,
|
||||
new java.util.concurrent.ArrayBlockingQueue<>(128), r -> {
|
||||
Thread thread = new Thread(r, "sub-location-resolver");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
}, new java.util.concurrent.ThreadPoolExecutor.AbortPolicy());
|
||||
}
|
||||
|
||||
public String insertSubscriptionAccount(String name, String upstreamKey, boolean filterHighMultiplier, boolean enabled) {
|
||||
@@ -211,6 +220,16 @@ import java.util.function.Supplier;
|
||||
}
|
||||
|
||||
public void updateSub(HttpServletResponse response, HttpServletRequest request, String client, String key) {
|
||||
Lock lock = stateCoordinator.readLock();
|
||||
lock.lock();
|
||||
try {
|
||||
distributeSubscription(response, request, client, key);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void distributeSubscription(HttpServletResponse response, HttpServletRequest request, String client, String key) {
|
||||
if (key == null || client == null)
|
||||
return;
|
||||
SubBind subBind = subMapper.selectSubBind(key);
|
||||
@@ -221,7 +240,7 @@ import java.util.function.Supplier;
|
||||
String ip = resolveClientIp(request);
|
||||
String ua = request.getHeader("User-Agent");
|
||||
if (ua == null) return;
|
||||
recordUpdate(subBind.getUser(), ip, ua);
|
||||
recordUpdate(subBind, ip, ua);
|
||||
if (!"v2".equals(client) && !"cat".equals(client)) {
|
||||
sendStatus(response, HttpServletResponse.SC_BAD_REQUEST, "client error");
|
||||
return;
|
||||
@@ -259,24 +278,34 @@ import java.util.function.Supplier;
|
||||
|| "0:0:0:0:0:0:0:1".equals(address);
|
||||
}
|
||||
|
||||
private void recordUpdate(String user, String ip, String ua) {
|
||||
private void recordUpdate(SubBind binding, String ip, String ua) {
|
||||
String user = binding.getUser();
|
||||
// 先落一条 unknown 记录并立即返回,归属地随后在后台补齐;
|
||||
// 这样取订阅的延迟不再取决于第三方定位站点的响应时间。
|
||||
subMapper.insertSubUpdateRecord(new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown"));
|
||||
SubUpdateRecord record = new SubUpdateRecord(0, user, ip, ua, new Date(), "unknown");
|
||||
record.setUserId(binding.getUserId());
|
||||
subMapper.insertSubUpdateRecord(record);
|
||||
if (subMapper.selectUpdateRecordCount(user) > 10)
|
||||
subMapper.deleteSubUpdateRecordById(subMapper.selectMinUpdateRecordId(user));
|
||||
locationExecutor.execute(() -> enrichLocation(user, ip));
|
||||
// Capture the generated row ID: another request may arrive before the lookup finishes.
|
||||
if (record.getId() > 0) {
|
||||
try {
|
||||
locationExecutor.execute(() -> enrichLocation(record.getId(), ip));
|
||||
} catch (java.util.concurrent.RejectedExecutionException busy) {
|
||||
log.debug("位置查询队列已满,保留 unknown recordId={}", record.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 后台补齐最近一条记录的归属地;任何失败都只记日志,不影响已落库的记录。 */
|
||||
private void enrichLocation(String user, String ip) {
|
||||
/** 后台按记录 ID 补齐归属地;任何失败都只记日志,不影响已落库的记录。 */
|
||||
private void enrichLocation(int recordId, String ip) {
|
||||
try {
|
||||
String location = resolveLocation(ip);
|
||||
if (location == null || location.isBlank() || "unknown".equals(location))
|
||||
return;
|
||||
subMapper.updateLatestSubUpdateRecordLocation(user, location);
|
||||
subMapper.updateSubUpdateRecordLocation(recordId, location);
|
||||
} catch (Exception e) {
|
||||
log.debug("补齐订阅更新记录归属地失败 user={}", user, e);
|
||||
log.debug("补齐订阅更新记录归属地失败 recordId={}", recordId, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,20 +339,26 @@ import java.util.function.Supplier;
|
||||
Lock stateLock = stateCoordinator.writeLock();
|
||||
stateLock.lock();
|
||||
try {
|
||||
return action.get();
|
||||
return transactions == null ? action.get() : transactions.execute(ignored -> action.get());
|
||||
} finally {
|
||||
stateLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdownLocationResolver() {
|
||||
if (locationExecutor instanceof ExecutorService service)
|
||||
service.shutdownNow();
|
||||
}
|
||||
|
||||
private SubscriptionAccount firstEnabledAccount() {
|
||||
return subMapper.selectAllSubscriptionAccounts().stream().filter(SubscriptionAccount::isEnabled).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
private String accountJson(SubscriptionAccount account) {
|
||||
private tools.jackson.databind.JsonNode accountJson(SubscriptionAccount account) {
|
||||
account.setV2Url(refreshService.v2Url(account));
|
||||
account.setClashUrl(refreshService.clashUrl(account));
|
||||
return CustomUtil.objectMapper.valueToTree(account).toString();
|
||||
return CustomUtil.objectMapper.valueToTree(account);
|
||||
}
|
||||
|
||||
private static void sendStatus(HttpServletResponse response, int status, String message) {
|
||||
|
||||
@@ -2,7 +2,9 @@ package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Dao.normal.SubMapper;
|
||||
import com.lion.lionwebsite.Domain.SubscriptionAccount;
|
||||
import com.lion.lionwebsite.Util.ClashSubscriptionParser;
|
||||
import com.lion.lionwebsite.Util.SubscriptionClientProfile;
|
||||
import com.lion.lionwebsite.Util.SubscriptionCacheStore;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet;
|
||||
@@ -57,6 +59,12 @@ public class SubscriptionRefreshService {
|
||||
/** 时间源,测试可替换以固定成功时刻。 */
|
||||
Clock clock = Clock.systemDefaultZone();
|
||||
|
||||
/**
|
||||
* 分代缓存存储。默认按 {@link #cacheRoot} 懒构造;测试可覆写为受控子类来模拟写入失败。
|
||||
* 保持为普通字段而不是构造参数,是为了不改动既有两参构造器(Spring 与老测试都在用)。
|
||||
*/
|
||||
SubscriptionCacheStore cacheStore;
|
||||
|
||||
private long refreshSequence;
|
||||
// Accessed only while holding the coordinator write lock.
|
||||
private final Map<Integer, Long> latestRefresh = new HashMap<>();
|
||||
@@ -78,18 +86,19 @@ public class SubscriptionRefreshService {
|
||||
|
||||
// Network access and parsing never hold the shared subscription lock.
|
||||
try {
|
||||
boolean filter = account.isFilterHighMultiplier();
|
||||
String v2 = processV2(firstLine(download(v2Url(account), SubscriptionClientProfile.forAccount(accountId, false))),
|
||||
account.isFilterHighMultiplier(), highMultiplierThreshold);
|
||||
List<String> clash = processClash(download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)),
|
||||
account.isFilterHighMultiplier(), highMultiplierThreshold);
|
||||
filter, highMultiplierThreshold);
|
||||
// 按 YAML 语义过滤高倍节点并同步清理 proxy-groups 引用,不再靠行缩进猜测。
|
||||
String clash = ClashSubscriptionParser.transform(
|
||||
download(clashUrl(account), SubscriptionClientProfile.forAccount(accountId, true)),
|
||||
filter, highMultiplierThreshold);
|
||||
stateLock.lock();
|
||||
try {
|
||||
if (!isCurrent(account, version))
|
||||
return false;
|
||||
Path dir = Paths.get(cacheRoot, String.valueOf(accountId));
|
||||
Files.createDirectories(dir);
|
||||
atomicWrite(dir.resolve("v2ray.txt"), v2.getBytes(StandardCharsets.UTF_8));
|
||||
atomicWrite(dir.resolve("clash.yaml"), String.join("\n", clash).concat("\n").getBytes(StandardCharsets.UTF_8));
|
||||
// 两份内容写进同一分代,校验通过后指针最后发布:任一步失败都保留上次完整版本。
|
||||
store().publish(accountId, v2.getBytes(StandardCharsets.UTF_8), clash.getBytes(StandardCharsets.UTF_8));
|
||||
subMapper.markSubscriptionRefreshSuccess(accountId, clock.millis());
|
||||
return true;
|
||||
} finally {
|
||||
@@ -162,56 +171,6 @@ public class SubscriptionRefreshService {
|
||||
return URLDecoder.decode(node.substring(hash + 1), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private List<String> processClash(List<String> source, boolean filter, double threshold) {
|
||||
Set<String> removed = new HashSet<>();
|
||||
List<String> result = new ArrayList<>();
|
||||
boolean inProxies = false;
|
||||
boolean skipNode = false;
|
||||
for (String line : source) {
|
||||
if (line.equals("proxies:")) {
|
||||
inProxies = true;
|
||||
skipNode = false;
|
||||
result.add(line);
|
||||
continue;
|
||||
}
|
||||
if (line.equals("proxy-groups:")) {
|
||||
inProxies = false;
|
||||
skipNode = false;
|
||||
result.add(line);
|
||||
continue;
|
||||
}
|
||||
if (inProxies && line.matches("^\\s{2}-\\s+name:.*")) {
|
||||
String name = clashName(line);
|
||||
skipNode = filter && isHigh(name, threshold);
|
||||
if (skipNode)
|
||||
removed.add(name);
|
||||
else
|
||||
result.add(line);
|
||||
continue;
|
||||
}
|
||||
if (skipNode)
|
||||
continue;
|
||||
if (!inProxies && !removed.isEmpty() && line.trim().startsWith("- ")) {
|
||||
String ref = line.trim().substring(2).trim();
|
||||
if (removed.contains(unquote(ref)))
|
||||
continue;
|
||||
}
|
||||
result.add(line);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String clashName(String line) {
|
||||
int index = line.indexOf("name:");
|
||||
return unquote(line.substring(index + 5).trim());
|
||||
}
|
||||
|
||||
private static String unquote(String value) {
|
||||
if (value.length() >= 2 && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'"))))
|
||||
return value.substring(1, value.length() - 1);
|
||||
return value;
|
||||
}
|
||||
|
||||
private static boolean isHigh(String name, double threshold) {
|
||||
Matcher matcher = MULTIPLIER.matcher(name);
|
||||
return matcher.find() && Double.parseDouble(matcher.group(1)) > threshold;
|
||||
@@ -251,22 +210,17 @@ public class SubscriptionRefreshService {
|
||||
}
|
||||
}
|
||||
|
||||
private static void atomicWrite(Path target, byte[] data) throws IOException {
|
||||
Path temp = target.resolveSibling(target.getFileName() + ".tmp");
|
||||
Files.write(temp, data);
|
||||
try {
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (AtomicMoveNotSupportedException e) {
|
||||
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 当前应分发的缓存文件。只在一整个版本有效时返回真实路径;否则返回一个保证不存在的
|
||||
* 哨兵路径,使分发端按「订阅暂不可用」处理,而不是发出半份旧缓存。
|
||||
*/
|
||||
public Path cachedPath(Integer accountId, String client) {
|
||||
return Paths.get(cacheRoot, String.valueOf(accountId), client.equals("v2") ? "v2ray.txt" : "clash.yaml");
|
||||
SubscriptionCacheStore store = store();
|
||||
return store.resolve(accountId, client).orElseGet(() -> store.unavailablePath(accountId, client));
|
||||
}
|
||||
|
||||
public boolean hasCompleteCache(Integer accountId) {
|
||||
return Files.isRegularFile(cachedPath(accountId, "v2")) && Files.isRegularFile(cachedPath(accountId, "cat"));
|
||||
return store().hasComplete(accountId);
|
||||
}
|
||||
|
||||
public void invalidateCache(Integer accountId) {
|
||||
@@ -274,12 +228,15 @@ public class SubscriptionRefreshService {
|
||||
stateLock.lock();
|
||||
try {
|
||||
latestRefresh.remove(accountId); // In-flight responses must not restore invalidated content.
|
||||
Files.deleteIfExists(cachedPath(accountId, "v2"));
|
||||
Files.deleteIfExists(cachedPath(accountId, "cat"));
|
||||
} catch (IOException e) {
|
||||
log.warn("清理失效订阅缓存失败 accountId={}", accountId, e);
|
||||
store().invalidate(accountId);
|
||||
} finally {
|
||||
stateLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized SubscriptionCacheStore store() {
|
||||
if (cacheStore == null)
|
||||
cacheStore = new SubscriptionCacheStore(cacheRoot);
|
||||
return cacheStore;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.lion.lionwebsite.Domain.SubBind;
|
||||
import com.lion.lionwebsite.Domain.SubscriptionAccount;
|
||||
import com.lion.lionwebsite.Message.*;
|
||||
import com.lion.lionwebsite.Util.CustomUtil;
|
||||
import com.lion.lionwebsite.Util.SubscriptionCacheStore;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -16,8 +17,6 @@ import javax.crypto.spec.SecretKeySpec;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.*;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
@@ -39,6 +38,12 @@ public class SubscriptionStandbySnapshotService {
|
||||
@Value("${subscription.cache-root:sub/accounts}")
|
||||
String cacheRoot;
|
||||
|
||||
/**
|
||||
* 与 {@link SubscriptionRefreshService} 共用的分代缓存解析器。
|
||||
* 测试可直接覆写;未设置时按 {@link #cacheRoot} 懒构造。
|
||||
*/
|
||||
SubscriptionCacheStore cacheStore;
|
||||
|
||||
@PostConstruct
|
||||
void validateConfiguration() {
|
||||
if (syncEnabled && (syncSecret == null || syncSecret.isBlank()))
|
||||
@@ -51,12 +56,14 @@ public class SubscriptionStandbySnapshotService {
|
||||
try {
|
||||
Map<Integer, SubscriptionAccountSnapshot> accountMap = new HashMap<>();
|
||||
for (SubscriptionAccount account : subMapper.selectAllSubscriptionAccounts()) {
|
||||
Path v2Path = cachedPath(account.getId(), "v2");
|
||||
Path clashPath = cachedPath(account.getId(), "cat");
|
||||
if (!account.isEnabled() || !Files.isRegularFile(v2Path) || !Files.isRegularFile(clashPath))
|
||||
if (!account.isEnabled())
|
||||
continue;
|
||||
byte[] v2 = Files.readAllBytes(v2Path);
|
||||
byte[] clash = Files.readAllBytes(clashPath);
|
||||
// 从同一分代读出成对内容,保证快照里两份文件同版本;无完整版本则跳过该账号。
|
||||
Optional<SubscriptionCacheStore.Pair> pair = store().read(account.getId());
|
||||
if (pair.isEmpty())
|
||||
continue;
|
||||
byte[] v2 = pair.get().v2();
|
||||
byte[] clash = pair.get().clash();
|
||||
SubscriptionAccountSnapshot snapshot = new SubscriptionAccountSnapshot();
|
||||
snapshot.setAccountId(account.getId());
|
||||
snapshot.setEnabled(true);
|
||||
@@ -132,7 +139,9 @@ public class SubscriptionStandbySnapshotService {
|
||||
|
||||
private static String hex(byte[] input) { return HexFormat.of().formatHex(input); }
|
||||
|
||||
private Path cachedPath(Integer accountId, String client) {
|
||||
return Path.of(cacheRoot, String.valueOf(accountId), "v2".equals(client) ? "v2ray.txt" : "clash.yaml");
|
||||
private synchronized SubscriptionCacheStore store() {
|
||||
if (cacheStore == null)
|
||||
cacheStore = new SubscriptionCacheStore(cacheRoot);
|
||||
return cacheStore;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
/** 协议 v2 的操作 ID 生成与解析。稳定 ID 是「节点重发/重试不重复执行副作用」的依据。 */
|
||||
public final class TaskProtocol {
|
||||
|
||||
private TaskProtocol() {
|
||||
}
|
||||
|
||||
/** 生成一个新的稳定操作 ID(下载或删除),形如 {@code op-<uuid>}。 */
|
||||
public static String newOperationId(String action) {
|
||||
return action + "-" + UUID.randomUUID();
|
||||
}
|
||||
|
||||
public static String newDownloadOperationId() {
|
||||
return newOperationId("dl");
|
||||
}
|
||||
|
||||
public static String newDeleteOperationId() {
|
||||
return newOperationId("del");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Domain.Gallery;
|
||||
import com.lion.lionwebsite.Domain.GalleryTask;
|
||||
import com.lion.lionwebsite.Message.TaskReportMessage;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 协议 v2 状态上报的消费入口。
|
||||
*
|
||||
* <p>定义成接口是为了断开依赖环:RemoteService 需要把上报交给生命周期服务,
|
||||
* 而生命周期服务需要有 RemoteService 才能下发命令。RemoteService 通过可选的 setter
|
||||
* 注入本接口(见 {@code RemoteService#setTaskReportHandler}),因此构造期没有环。
|
||||
*/
|
||||
public interface TaskReportHandler {
|
||||
|
||||
/**
|
||||
* 提交一份上报。
|
||||
*
|
||||
* @return 已成功提交(可回 ACK)时返回提交结果;数据库失败等「不可回 ACK」的情况返回 null。
|
||||
* 返回 null 时节点会保留快照并稍后重投,这正是「失败写入不得清任务」的要求。
|
||||
*/
|
||||
CommittedReport apply(TaskReportMessage report);
|
||||
|
||||
/**
|
||||
* 已提交的结果。
|
||||
*
|
||||
* @param reportId 本次上报 ID(回执用)
|
||||
* @param newlyCompleted 本次「首次」完成的任务,通知与 WebSocket 由独立执行器在事务之后处理,
|
||||
* 因此慢通知不会阻塞数据库消费线程;重复上报不会重复出现在这里
|
||||
* @param acceptedTasks 事务内实际被接受的进度更新(含下载中/压缩中等中间状态,已排除过期、
|
||||
* 旧世代与待删除任务),WebSocket 只推这一批,避免前端看到被丢弃的迟到状态
|
||||
*/
|
||||
record CommittedReport(String reportId, List<Gallery> newlyCompleted, List<GalleryTask> acceptedTasks) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Domain.GalleryTask;
|
||||
|
||||
/**
|
||||
* 任务业务状态码到展示文案的唯一映射点。
|
||||
*
|
||||
* <p>状态以稳定代码表达,中文文案只出现在展示层(WebSocket/列表),
|
||||
* 避免把「失败/待重试/结果未知」塞进成功失败提示字符串。
|
||||
*/
|
||||
public final class TaskStatus {
|
||||
|
||||
public static final String SUBMITTED = "已提交";
|
||||
/** 已登记创建意图、尚未确认上游提交;用于让任务列表在提交期间也能看到占位任务。 */
|
||||
public static final String PENDING_SUBMIT = "提交中";
|
||||
public static final String DOWNLOADING = "下载中";
|
||||
public static final String WAITING_COMPRESS = "等待压缩";
|
||||
public static final String COMPRESSING = "压缩中";
|
||||
public static final String COMPLETE = "下载完成";
|
||||
public static final String UNKNOWN = "提交未知";
|
||||
public static final String DELETE_PENDING = "待删除";
|
||||
public static final String DELETE_FAILED = "删除失败";
|
||||
|
||||
private TaskStatus() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 节点上报的进度状态映射为业务文案;未知状态码返回 null,由调用方忽略。
|
||||
*
|
||||
* <p>GalleryTask 里的状态码是可变 static byte,不是编译期常量,因此不能用 switch case;
|
||||
* 这里用整数比较,避免为改常量引入风险。
|
||||
*/
|
||||
public static String fromReport(byte reportStatus) {
|
||||
int status = reportStatus;
|
||||
if (status == GalleryTask.DOWNLOADING)
|
||||
return DOWNLOADING;
|
||||
if (status == GalleryTask.DOWNLOAD_COMPLETE)
|
||||
return WAITING_COMPRESS;
|
||||
if (status == GalleryTask.COMPRESSING)
|
||||
return COMPRESSING;
|
||||
if (status == GalleryTask.COMPRESS_COMPLETE)
|
||||
return COMPLETE;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.lion.lionwebsite.Service;
|
||||
|
||||
import com.lion.lionwebsite.Domain.Gallery;
|
||||
import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
|
||||
import com.lion.lionwebsite.Util.GalleryUtil;
|
||||
import com.lion.lionwebsite.Util.PreparedGallery;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* 上游图站的读写边界。
|
||||
*
|
||||
* <p>分成两步是协议 v2 的核心约束:
|
||||
* <ol>
|
||||
* <li>{@link #prepare} 只做读取(页面、分辨率),绝不产生下载副作用;</li>
|
||||
* <li>调用方必须在持久化 SUBMITTING 之后才调用 {@link #submit},因为那一步才真正 POST。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>拆成接口是为了让生命周期服务可以在测试里注入假实现,也方便后续把抓取/解析/提交
|
||||
* 进一步分开;默认实现 {@link GalleryUtilAdapter} 直接转发到现有 GalleryUtil。
|
||||
*/
|
||||
public interface UpstreamGateway {
|
||||
|
||||
/** 只读解析:失败返回 null 表示链接无效或页面不可用,不产生任何副作用。 */
|
||||
PreparedGallery prepare(String link) throws IOException;
|
||||
|
||||
/** 真正提交下载请求。抛异常表示结果未知,由调用方落 UNKNOWN,不做自动重发。 */
|
||||
Gallery submit(PreparedGallery prepared, String targetResolution)
|
||||
throws IOException, ResolutionNotMatchException;
|
||||
|
||||
/** 默认实现:转发现有 GalleryUtil 的拆分方法。 */
|
||||
@Component
|
||||
class GalleryUtilAdapter implements UpstreamGateway {
|
||||
|
||||
@Override
|
||||
public PreparedGallery prepare(String link) throws IOException {
|
||||
return GalleryUtil.prepare(link);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Gallery submit(PreparedGallery prepared, String targetResolution)
|
||||
throws IOException, ResolutionNotMatchException {
|
||||
return GalleryUtil.submit(prepared, targetResolution);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package com.lion.lionwebsite.Service;
|
||||
import com.lion.lionwebsite.Dao.normal.CollectMapper;
|
||||
import com.lion.lionwebsite.Dao.normal.GalleryMapper;
|
||||
import com.lion.lionwebsite.Dao.normal.UserMapper;
|
||||
import com.lion.lionwebsite.Dao.normal.SubMapper;
|
||||
import com.lion.lionwebsite.Domain.User;
|
||||
import com.lion.lionwebsite.Interceptor.TaskHandlerInterceptor;
|
||||
import com.lion.lionwebsite.Util.CustomUtil;
|
||||
@@ -10,8 +11,13 @@ import com.lion.lionwebsite.Util.Response;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
|
||||
|
||||
@@ -28,6 +34,11 @@ public class UserService{
|
||||
|
||||
final TaskHandlerInterceptor taskHandlerInterceptor;
|
||||
|
||||
@Autowired SubMapper subMapper;
|
||||
@Autowired SubscriptionStateCoordinator subscriptionState;
|
||||
@Autowired RemoteService remoteService;
|
||||
@Autowired @Qualifier("mainTransactionTemplate") TransactionTemplate transactions;
|
||||
|
||||
public String addAuthCode(String targetAuthCode, String people) {
|
||||
Response response = Response.generateResponse();
|
||||
|
||||
@@ -66,12 +77,22 @@ public class UserService{
|
||||
Response response = Response.generateResponse();
|
||||
|
||||
try{
|
||||
if(userMapper.isExist(targetAuthCode) != 0){
|
||||
userMapper.updateUsername(targetAuthCode, newUsername);
|
||||
response.success("修改成功");
|
||||
}else {
|
||||
response.failure("授权码不存在");
|
||||
}
|
||||
return coordinated(() -> {
|
||||
if (userMapper.isExist(targetAuthCode) == 0)
|
||||
return response.failure("授权码不存在").toJSONString();
|
||||
if (newUsername == null || newUsername.isBlank())
|
||||
return response.failure("用户名不能为空").toJSONString();
|
||||
String name = newUsername.trim();
|
||||
User current = userMapper.selectUserByAuthCode(targetAuthCode);
|
||||
User existing = userMapper.selectUserByUsername(name);
|
||||
if (existing != null && (current == null || existing.getId() != current.getId()))
|
||||
return response.failure("用户名已存在").toJSONString();
|
||||
userMapper.updateUsername(targetAuthCode, name);
|
||||
if (subMapper != null && current != null)
|
||||
subMapper.renameBoundUser(current.getId(), current.getUsername(), name);
|
||||
if (remoteService != null) remoteService.requestSubscriptionSync();
|
||||
return response.success("修改成功").toJSONString();
|
||||
});
|
||||
}catch (Exception e){
|
||||
log.error("修改用户名失败", e);
|
||||
response.failure("修改失败");
|
||||
@@ -85,17 +106,22 @@ public class UserService{
|
||||
|
||||
if(userMapper.isExist(targetAuthCode) != 0)
|
||||
try{
|
||||
User user = userMapper.selectUserByAuthCode(targetAuthCode);
|
||||
|
||||
//修改下载人以及取消收藏
|
||||
ArrayList<Integer> galleries = collectMapper.selectGidByCollector(user.getId());
|
||||
for(Integer gallery: galleries)
|
||||
collectMapper.disCollect(gallery, user.getId());
|
||||
galleryMapper.updateGalleryDownloader(3, user.getId());
|
||||
|
||||
userMapper.deleteUserByAuthCode(targetAuthCode);
|
||||
coordinated(() -> {
|
||||
User user = userMapper.selectUserByAuthCode(targetAuthCode);
|
||||
ArrayList<Integer> galleries = collectMapper.selectGidByCollector(user.getId());
|
||||
for(Integer gallery: galleries)
|
||||
collectMapper.disCollect(gallery, user.getId());
|
||||
galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, user.getId());
|
||||
if (subMapper != null) {
|
||||
subMapper.deleteSubUpdateRecord(user.getUsername());
|
||||
subMapper.deleteSubBind(user.getUsername());
|
||||
}
|
||||
userMapper.deleteUserByAuthCode(targetAuthCode);
|
||||
return "deleted";
|
||||
});
|
||||
response.success("删除成功");
|
||||
taskHandlerInterceptor.updateAuthCodes();
|
||||
if (remoteService != null) remoteService.requestSubscriptionSync();
|
||||
}catch (Exception e){
|
||||
log.error("删除授权码失败", e);
|
||||
response.failure("删除失败");
|
||||
@@ -108,27 +134,29 @@ public class UserService{
|
||||
}
|
||||
|
||||
public String alterStatus(String AuthCode, boolean isEnable){
|
||||
Response response = Response.generateResponse();
|
||||
|
||||
if(userMapper.isExist(AuthCode) == 1){
|
||||
int id = userMapper.selectUserByAuthCode(AuthCode).getId();
|
||||
userMapper.updateIsEnableById(id, isEnable);
|
||||
galleryMapper.updateGalleryDownloader(3, id);
|
||||
response.success("修改用户状态成功");
|
||||
taskHandlerInterceptor.updateAuthCodes();
|
||||
try {
|
||||
String result = coordinated(() -> {
|
||||
if (userMapper.isExist(AuthCode) != 1)
|
||||
return Response._failure("该用户不存在");
|
||||
int id = userMapper.selectUserByAuthCode(AuthCode).getId();
|
||||
userMapper.updateIsEnableById(id, isEnable);
|
||||
if (!isEnable) galleryMapper.updateGalleryDownloader(ADMIN_USER_ID, id);
|
||||
return Response._success("修改用户状态成功");
|
||||
});
|
||||
// Refresh the in-memory identity view only after the database commits.
|
||||
if (result.contains("\"result\":\"success\"")) taskHandlerInterceptor.updateAuthCodes();
|
||||
return result;
|
||||
} catch (RuntimeException error) {
|
||||
log.warn("修改用户状态失败: {}", error.getClass().getSimpleName());
|
||||
return Response._failure("修改用户状态失败");
|
||||
}
|
||||
else{
|
||||
response.failure("该用户不存在");
|
||||
}
|
||||
|
||||
return response.toJSONString();
|
||||
}
|
||||
|
||||
public String getAllUser() {
|
||||
Response response = Response.generateResponse();
|
||||
User[] users = userMapper.selectAllUser();
|
||||
|
||||
response.success(objectMapper.valueToTree(users).toString());
|
||||
response.success(objectMapper.valueToTree(users));
|
||||
|
||||
return response.toJSONString();
|
||||
}
|
||||
@@ -150,4 +178,14 @@ public class UserService{
|
||||
User user = userMapper.selectUserByAuthCode(AuthCode);
|
||||
return user != null && user.getId() == ADMIN_USER_ID;
|
||||
}
|
||||
|
||||
private String coordinated(Supplier<String> action) {
|
||||
Lock lock = subscriptionState == null ? null : subscriptionState.writeLock();
|
||||
if (lock != null) lock.lock();
|
||||
try {
|
||||
return transactions == null ? action.get() : transactions.execute(ignored -> action.get());
|
||||
} finally {
|
||||
if (lock != null) lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
import org.yaml.snakeyaml.DumperOptions;
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** YAML semantics rather than indentation determine nodes and references. */
|
||||
public final class ClashSubscriptionParser {
|
||||
private static final Pattern MULTIPLIER = Pattern.compile("(\\d+(?:\\.\\d+)?)x\\s*$", Pattern.CASE_INSENSITIVE);
|
||||
private ClashSubscriptionParser() { }
|
||||
|
||||
public static String transform(List<String> lines, boolean filter, double threshold) {
|
||||
Object loaded = new Yaml().load(String.join("\n", lines));
|
||||
if (!(loaded instanceof Map<?, ?> source)) throw new IllegalArgumentException("Clash 上游必须为 YAML 对象");
|
||||
if (!(source.get("proxies") instanceof List<?> proxies)) throw new IllegalArgumentException("Clash 上游缺少 proxies 列表");
|
||||
Map<Object, Object> output = new LinkedHashMap<>(source);
|
||||
Set<String> removed = new HashSet<>();
|
||||
List<Object> kept = new ArrayList<>();
|
||||
for (Object node : proxies) {
|
||||
if (!(node instanceof Map<?, ?> fields) || !(fields.get("name") instanceof String name))
|
||||
throw new IllegalArgumentException("Clash 节点缺少名称");
|
||||
var multiplier = MULTIPLIER.matcher(name);
|
||||
if (filter && multiplier.find() && Double.parseDouble(multiplier.group(1)) > threshold)
|
||||
removed.add(name);
|
||||
else kept.add(node);
|
||||
}
|
||||
output.put("proxies", kept);
|
||||
if (source.get("proxy-groups") instanceof List<?> groups) {
|
||||
List<Object> result = new ArrayList<>();
|
||||
for (Object group : groups) {
|
||||
if (!(group instanceof Map<?, ?> fields)) throw new IllegalArgumentException("Clash 分组必须为对象");
|
||||
Map<Object, Object> copy = new LinkedHashMap<>(fields);
|
||||
if (fields.get("proxies") instanceof List<?> references)
|
||||
copy.put("proxies", references.stream().filter(ref -> !removed.contains(ref)).toList());
|
||||
result.add(copy);
|
||||
}
|
||||
output.put("proxy-groups", result);
|
||||
}
|
||||
DumperOptions options = new DumperOptions();
|
||||
options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
|
||||
options.setPrettyFlow(true);
|
||||
return new Yaml(options).dump(output);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import com.lion.lionwebsite.Domain.Gallery;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Pure parsing: upstream requests and download submission live outside this class. */
|
||||
public final class GalleryPageParser {
|
||||
private GalleryPageParser() { }
|
||||
|
||||
public static Gallery metadata(Document document, String link, int gid) throws IOException {
|
||||
String title = document.select("#gn").text();
|
||||
if (title.isBlank()) title = document.select("#gj").text();
|
||||
if (title.isBlank()) throw new IOException("上游页面缺少画廊名称,可能是错误页");
|
||||
Map<String, String> fields = new HashMap<>();
|
||||
for (Element row : document.select("#gdd tr")) {
|
||||
var cells = row.select("td");
|
||||
if (cells.size() < 2) continue;
|
||||
String name = cells.get(0).text().replace(":", "").strip().toLowerCase(Locale.ROOT);
|
||||
fields.put(name, cells.get(1).text().strip());
|
||||
}
|
||||
var pages = Pattern.compile("\\d+").matcher(fields.getOrDefault("length", ""));
|
||||
if (!pages.find()) throw new IOException("上游页面缺少画廊页数");
|
||||
Gallery result = new Gallery();
|
||||
result.setGid(gid); result.setLink(link); result.setName(title + " [" + gid + "]");
|
||||
result.setCreateTime(System.currentTimeMillis() / 1000); result.setProceeding(0);
|
||||
result.setLanguage(fields.getOrDefault("language", "").replace("\u00a0", " "));
|
||||
try {
|
||||
result.setPages(Integer.parseInt(pages.group()));
|
||||
result.setFileSize(CustomUtil.stringToFileSize(fields.getOrDefault("file size", "").replace(" ", "")));
|
||||
} catch (RuntimeException failure) { throw new IOException("上游画廊页数或文件大小格式不支持", failure); }
|
||||
if (result.getPages() <= 0) throw new IOException("上游画廊页数必须为正数");
|
||||
result.setDisplayFileSize(CustomUtil.fileSizeToString(result.getFileSize()));
|
||||
Element thumbnail = document.selectFirst("#gd1 [style*=url]");
|
||||
var image = Pattern.compile("url\\(\\s*['\"]?(.*?)['\"]?\\s*\\)").matcher(thumbnail == null ? "" : thumbnail.attr("style"));
|
||||
String imageUrl = image.find() ? image.group(1) : document.select("#gd1 img").attr("src");
|
||||
result.setThumb_link(imageUrl.replace("https://s.exhentai.org", ""));
|
||||
return result;
|
||||
}
|
||||
|
||||
public static String archiveLink(Document document) throws IOException {
|
||||
for (Element anchor : document.select("#gd5 a")) {
|
||||
var matcher = Pattern.compile("https?://[^'\"\\s]+").matcher(anchor.attr("onclick"));
|
||||
if (matcher.find()) return matcher.group();
|
||||
String href = anchor.absUrl("href");
|
||||
if (!href.isBlank() && (anchor.text().toLowerCase(Locale.ROOT).contains("download") || href.contains("archive")))
|
||||
return href;
|
||||
}
|
||||
throw new IOException("上游页面缺少归档入口");
|
||||
}
|
||||
|
||||
public static Map<String, String> resolutions(Document document) throws IOException {
|
||||
var expression = Pattern.compile("^(\\d+x|Original)\\s+([\\d.]+)\\s*([A-Za-z]+)$", Pattern.CASE_INSENSITIVE);
|
||||
TreeMap<Integer, String> sizes = new TreeMap<>();
|
||||
String original = null;
|
||||
for (Element paragraph : document.select("#db td p")) {
|
||||
var matcher = expression.matcher(paragraph.text().strip());
|
||||
if (!matcher.matches()) continue;
|
||||
String size = matcher.group(2) + matcher.group(3);
|
||||
if (matcher.group(1).equalsIgnoreCase("Original")) original = size;
|
||||
else sizes.put(Integer.parseInt(matcher.group(1).replace("x", "")), size);
|
||||
}
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
sizes.forEach((width, size) -> result.put(width + "x", size));
|
||||
if (original != null) result.put("Original", original);
|
||||
if (result.isEmpty()) throw new IOException("上游页面没有可用分辨率");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import com.lion.lionwebsite.Domain.GalleryForQuery;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Search rows are located by gallery links, independent of table nesting and headings. */
|
||||
public final class GallerySearchParser {
|
||||
private GallerySearchParser() { }
|
||||
public static List<GalleryForQuery> parse(Document document) throws IOException {
|
||||
List<GalleryForQuery> result = new ArrayList<>();
|
||||
for (Element row : document.select("tr")) {
|
||||
Element link = row.selectFirst("a[href*=/g/]");
|
||||
if (link == null) continue;
|
||||
String href = link.attr("href");
|
||||
Integer gid = GalleryUtil.parseGid(href);
|
||||
if (gid == null) continue;
|
||||
GalleryForQuery gallery = new GalleryForQuery();
|
||||
gallery.setGid(gid.toString()); gallery.setLink(href);
|
||||
gallery.setName(link.select(".glink").isEmpty() ? link.text() : link.select(".glink").text());
|
||||
Element type = row.selectFirst(".cn, .cs");
|
||||
gallery.setType(type == null ? row.select("td").first().text() : type.text());
|
||||
Element image = row.selectFirst("img");
|
||||
String thumbnail = image == null ? "" : image.attr("src");
|
||||
if (thumbnail.isBlank() || thumbnail.startsWith("data:")) thumbnail = image == null ? "" : image.attr("data-src");
|
||||
gallery.setThumbnailUrl(thumbnail.replace("https://s.exhentai.org", ""));
|
||||
var pages = Pattern.compile("(\\d+)\\s+pages?", Pattern.CASE_INSENSITIVE).matcher(row.text());
|
||||
if (!pages.find()) throw new IOException("上游搜索结果缺少页数 gid=" + gid);
|
||||
gallery.setPage(Integer.parseInt(pages.group(1)));
|
||||
Element date = row.selectFirst("[id^=posted_]");
|
||||
if (date != null) gallery.setUploadTime(date.text());
|
||||
else {
|
||||
var dates = Pattern.compile("\\d{4}-\\d{2}-\\d{2}(?:\\s+\\d{2}:\\d{2})?").matcher(row.text());
|
||||
gallery.setUploadTime(dates.find() ? dates.group() : "");
|
||||
}
|
||||
if (gallery.getName().isBlank()) throw new IOException("上游搜索结果缺少名称 gid=" + gid);
|
||||
result.add(gallery);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -82,6 +82,12 @@ public class GalleryUtil {
|
||||
* @throws ResolutionNotMatchException 没有目标分辨率
|
||||
*/
|
||||
public static Gallery parse(String url, boolean isDownload, String targetResolution) throws IOException, ResolutionNotMatchException {
|
||||
PreparedGallery prepared = prepare(url);
|
||||
if (prepared == null) return null;
|
||||
return isDownload ? submit(prepared, targetResolution) : prepared.gallery();
|
||||
}
|
||||
|
||||
public static PreparedGallery prepare(String url) throws IOException {
|
||||
String origin = verifyLink(url);
|
||||
|
||||
if(origin == null){
|
||||
@@ -94,66 +100,26 @@ public class GalleryUtil {
|
||||
log.warn("链接无法解析出 gid,按无效链接处理");
|
||||
return null;
|
||||
}
|
||||
Gallery gallery = new Gallery();
|
||||
gallery.setLink(url);
|
||||
gallery.setCreateTime(System.currentTimeMillis()/1000);
|
||||
gallery.setGid(gid);
|
||||
gallery.setProceeding(0);
|
||||
|
||||
//访问图片页面
|
||||
String galleryPage = requests(url, GET, null, null);
|
||||
Document galleryDoc = Jsoup.parse(galleryPage);
|
||||
|
||||
//收集图片基本信息
|
||||
gallery.setName(galleryDoc.select("#gn").text() + " [" + gallery.getGid() + ']');
|
||||
gallery.setLanguage(galleryDoc.select("#gdd > table > tbody > tr:nth-child(4) > td.gdt2").text().replace("%nbps", ""));
|
||||
gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0]));
|
||||
gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", "")));
|
||||
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
|
||||
String thumb_link = galleryDoc.select("#gd1 > div").getFirst().attr("style");
|
||||
gallery.setThumb_link(thumb_link.substring(thumb_link.indexOf("url(") + 4, thumb_link.lastIndexOf(")")).replace("https://s.exhentai.org", ""));
|
||||
|
||||
//查找下载页面链接并初始化参数
|
||||
String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1];
|
||||
Document galleryDoc = Jsoup.parse(requests(url, GET, null, null), url);
|
||||
Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
|
||||
String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
|
||||
HashMap<String, String> headers = new HashMap<>();
|
||||
headers.put("origin", origin);
|
||||
headers.put("referer", archiveLink);
|
||||
Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
|
||||
String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
|
||||
if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
|
||||
gallery.setAvailableResolution(GalleryPageParser.resolutions(downloadDoc));
|
||||
gallery.setStatus("等待确认下载");
|
||||
return new PreparedGallery(gallery, download_link, origin);
|
||||
}
|
||||
|
||||
public static Gallery submit(PreparedGallery prepared, String targetResolution) throws IOException, ResolutionNotMatchException {
|
||||
Gallery gallery = prepared.gallery();
|
||||
String download_link = prepared.formUrl();
|
||||
HashMap<String, String> headers = new HashMap<>();
|
||||
headers.put("origin", prepared.origin());
|
||||
headers.put("referer", download_link);
|
||||
|
||||
//访问下载页面,获取分辨率
|
||||
String downloadPage = requests(download_link, GET, headers, null);
|
||||
Document downloadDoc = Jsoup.parse(downloadPage);
|
||||
Elements resolutions = downloadDoc.select("#db > div > table > tbody > tr > td");
|
||||
download_link = downloadDoc.select("#hathdl_form").attr("action");
|
||||
|
||||
Map<String, String> availableResolution = new LinkedHashMap<>();
|
||||
Map<String, String> tempResolutions = new LinkedHashMap<>();
|
||||
ArrayList<Integer> tempResolutionKeys = new ArrayList<>();
|
||||
|
||||
for(Element element: resolutions){
|
||||
String[] parameter = element.select("td > p").text().split(" ");
|
||||
if(parameter[1].contains("N/A"))
|
||||
continue;
|
||||
tempResolutions.put(parameter[0], parameter[1] + parameter[2]);
|
||||
if(parameter[0].contains("x")){
|
||||
tempResolutionKeys.add(Integer.valueOf(parameter[0].replace("x", "")));
|
||||
}
|
||||
}
|
||||
|
||||
tempResolutionKeys.sort(Comparator.comparingInt(o -> o));
|
||||
|
||||
for(Integer resolution: tempResolutionKeys){
|
||||
availableResolution.put(resolution.toString()+"x", tempResolutions.get(resolution+"x"));
|
||||
}
|
||||
|
||||
availableResolution.put("Original", tempResolutions.get("Original"));
|
||||
gallery.setAvailableResolution(availableResolution);
|
||||
|
||||
//如果不是下载,则直接返回
|
||||
if(!isDownload){
|
||||
gallery.setStatus("等待确认下载");
|
||||
return gallery;
|
||||
}
|
||||
|
||||
//如果目标分辨率不存在,抛出错误
|
||||
if(!gallery.getAvailableResolution().containsKey(targetResolution)){
|
||||
log.warn("目标分辨率不存在,可用分辨率: {}", gallery.getAvailableResolution());
|
||||
@@ -161,15 +127,15 @@ public class GalleryUtil {
|
||||
}
|
||||
|
||||
gallery.setResolution(targetResolution);
|
||||
gallery.setFileSize(CustomUtil.stringToFileSize(availableResolution.get(targetResolution).replace(" ", "")));
|
||||
gallery.setFileSize(CustomUtil.stringToFileSize(gallery.getAvailableResolution().get(targetResolution).replace(" ", "")));
|
||||
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
|
||||
|
||||
//提交下载请求
|
||||
HashMap<String, String> body = new HashMap<>();
|
||||
body.put("payload", FORM_DATA);
|
||||
body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
|
||||
downloadPage = requests(download_link, POST, headers, body);
|
||||
downloadDoc = Jsoup.parse(downloadPage);
|
||||
String downloadPage = requests(download_link, POST, headers, body);
|
||||
Document downloadDoc = Jsoup.parse(downloadPage);
|
||||
|
||||
//判断下载请求是否提交成功
|
||||
if(downloadDoc.select("#db > p:nth-child(2) > strong").text().startsWith("#"))
|
||||
@@ -192,14 +158,12 @@ public class GalleryUtil {
|
||||
if(content.trim().isEmpty())
|
||||
return null;
|
||||
Document document = Jsoup.parse(content);
|
||||
if(document.select("body > script").size() < 2)
|
||||
return null;
|
||||
Element script = document.select("body > script").get(1);
|
||||
String[] scripts = script.html().split("\n");
|
||||
String script = document.select("script").stream().map(Element::data).collect(java.util.stream.Collectors.joining("\n"));
|
||||
if (!script.contains("imagelist") || !script.contains("mpvkey")) return null;
|
||||
ArrayList<ImageKeyCache> imageKeyCaches = new ArrayList<>();
|
||||
AtomicInteger page = new AtomicInteger(1);
|
||||
gid2MpvKey.put(gid, scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", ""));
|
||||
JsonNode nodes = parseImagelist(scripts[2]);
|
||||
gid2MpvKey.put(gid, JavaScriptAssignments.string(script, "mpvkey"));
|
||||
JsonNode nodes = objectMapper.readTree(JavaScriptAssignments.array(script, "imagelist"));
|
||||
nodes.forEach((n) -> {
|
||||
ImageKeyCache imageKeyCache = new ImageKeyCache();
|
||||
imageKeyCache.setGid(gid);
|
||||
@@ -247,12 +211,14 @@ public class GalleryUtil {
|
||||
gid2MpvKey.remove(parseGid(url) + "");
|
||||
return;
|
||||
}
|
||||
Document document = Jsoup.parse(content);
|
||||
Element script = document.select("body > script").get(1);
|
||||
String[] scripts = script.html().split("\n");
|
||||
String mpvKey = scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", "");
|
||||
gid2MpvKey.put(parseGid(url) + "", mpvKey);
|
||||
log.info("刷新key:{}", mpvKey);
|
||||
try {
|
||||
String scripts = Jsoup.parse(content).select("script").stream().map(Element::data)
|
||||
.collect(java.util.stream.Collectors.joining("\n"));
|
||||
gid2MpvKey.put(parseGid(url) + "", JavaScriptAssignments.string(scripts, "mpvkey"));
|
||||
} catch (IOException e) {
|
||||
gid2MpvKey.remove(parseGid(url) + "");
|
||||
log.warn("上游mpv脚本无法解析,缓存已失效");
|
||||
}
|
||||
}
|
||||
|
||||
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Extracts data assignments without depending on script position or line layout. */
|
||||
public final class JavaScriptAssignments {
|
||||
private JavaScriptAssignments() { }
|
||||
|
||||
public static String string(String source, String name) throws IOException {
|
||||
var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*(['\"])(.*?)\\1", Pattern.DOTALL).matcher(source);
|
||||
if (!matcher.find()) throw new IOException("上游脚本缺少 " + name);
|
||||
return matcher.group(2);
|
||||
}
|
||||
|
||||
public static String array(String source, String name) throws IOException {
|
||||
var matcher = Pattern.compile("\\b" + Pattern.quote(name) + "\\s*=\\s*\\[").matcher(source);
|
||||
if (!matcher.find()) throw new IOException("上游脚本缺少 " + name);
|
||||
int start = matcher.end() - 1, depth = 0; char quote = 0; boolean escaped = false;
|
||||
for (int i = start; i < source.length(); i++) {
|
||||
char c = source.charAt(i);
|
||||
if (quote != 0) {
|
||||
if (escaped) escaped = false;
|
||||
else if (c == '\\') escaped = true;
|
||||
else if (c == quote) quote = 0;
|
||||
continue;
|
||||
}
|
||||
if (c == '"' || c == '\'') quote = c;
|
||||
else if (c == '[') depth++;
|
||||
else if (c == ']' && --depth == 0) return source.substring(start, i + 1);
|
||||
}
|
||||
throw new IOException("上游脚本的 " + name + " 不完整");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import com.lion.lionwebsite.Domain.Gallery;
|
||||
|
||||
/** A read-only upstream preparation; creating this value never submits a download. */
|
||||
public record PreparedGallery(Gallery gallery, String formUrl, String origin) { }
|
||||
@@ -0,0 +1,328 @@
|
||||
package com.lion.lionwebsite.Util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.Comparator;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* 订阅缓存的分代存储:v2ray 与 clash 两份内容必须同时生效。
|
||||
*
|
||||
* <p>旧实现把两份文件各自原子替换,第二份失败时用户会拿到「新 v2ray + 旧 clash」的
|
||||
* 混合版本。这里把两份文件写进同一个不可变分代目录,校验通过后用一次原子替换发布
|
||||
* 指针(manifest),因此抓取、解析、写入或发布任何一步失败,都只会保留上一个完整分代。
|
||||
*
|
||||
* <p>目录结构:
|
||||
* <pre>
|
||||
* <cacheRoot>/<accountId>/current 指针(原子替换,最后发布)
|
||||
* <cacheRoot>/<accountId>/generations/<generation>/v2ray.txt
|
||||
* <cacheRoot>/<accountId>/generations/<generation>/clash.yaml
|
||||
* <cacheRoot>/<accountId>/v2ray.txt + clash.yaml 旧结构,两份都在才按 legacy 读
|
||||
* </pre>
|
||||
*
|
||||
* <p>读取始终从指针指向的同一个分代里取两份文件,所以拿到的一定是同版本内容。
|
||||
*/
|
||||
@Slf4j
|
||||
public class SubscriptionCacheStore {
|
||||
private static final String MANIFEST_FILE = "current";
|
||||
private static final String GENERATION_DIR = "generations";
|
||||
private static final String V2_FILE = "v2ray.txt";
|
||||
private static final String CLASH_FILE = "clash.yaml";
|
||||
/** 保留目录名,结构上永远不存在,用于表达「当前没有可分发版本」。 */
|
||||
private static final String UNAVAILABLE_DIR = ".unavailable";
|
||||
private static final String MANIFEST_GENERATION = "generation";
|
||||
private static final String MANIFEST_V2_SHA = "v2-sha256";
|
||||
private static final String MANIFEST_CLASH_SHA = "clash-sha256";
|
||||
/** 旧结构读出来的版本的 generation 标记;真实分代名是定宽时间戳,不会与之碰撞。 */
|
||||
private static final String LEGACY_GENERATION = "legacy";
|
||||
/** 保留分代数:当前与上一个,足够回退且不会无限增长。 */
|
||||
private static final int RETAINED_GENERATIONS = 2;
|
||||
private static final AtomicLong SEQUENCE = new AtomicLong();
|
||||
|
||||
private final Path root;
|
||||
|
||||
public SubscriptionCacheStore(String cacheRoot) {
|
||||
this(Paths.get(cacheRoot));
|
||||
}
|
||||
|
||||
public SubscriptionCacheStore(Path root) {
|
||||
this.root = root;
|
||||
}
|
||||
|
||||
/** 同一个分代目录里读出的一对内容;两份文件必然同版本。 */
|
||||
public record Pair(String generation, byte[] v2, byte[] clash) { }
|
||||
|
||||
public Path accountDir(int accountId) {
|
||||
return root.resolve(String.valueOf(accountId));
|
||||
}
|
||||
|
||||
/** 旧结构的账号根文件路径。仅供兼容测试定位 legacy 文件,不作为回退目标。 */
|
||||
public Path legacyPath(int accountId, String client) {
|
||||
return accountDir(accountId).resolve(fileName(client));
|
||||
}
|
||||
|
||||
/**
|
||||
* 没有完整可分发版本时返回的哨兵路径。
|
||||
*
|
||||
* <p>它保证不存在,因此调用方用 {@code Files.isRegularFile} 判定就会走「订阅暂不可用」
|
||||
* 分支,而不会被半份 legacy 文件骗过。
|
||||
*/
|
||||
public Path unavailablePath(int accountId, String client) {
|
||||
return accountDir(accountId).resolve(UNAVAILABLE_DIR).resolve(fileName(client));
|
||||
}
|
||||
|
||||
private static String fileName(String client) {
|
||||
return "v2".equals(client) ? V2_FILE : CLASH_FILE;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入一份新分代并在最后发布指针。
|
||||
*
|
||||
* <p>失败时抛出异常,且不改变指针与上一个分代,调用方据此保留上次的完整版本。
|
||||
*/
|
||||
public Pair publish(int accountId, byte[] v2, byte[] clash) throws IOException {
|
||||
Path account = accountDir(accountId);
|
||||
Files.createDirectories(account);
|
||||
String generation = nextGeneration();
|
||||
Path dir = account.resolve(GENERATION_DIR).resolve(generation);
|
||||
boolean published = false;
|
||||
try {
|
||||
Files.createDirectories(dir);
|
||||
writeFile(dir.resolve(V2_FILE), v2);
|
||||
writeFile(dir.resolve(CLASH_FILE), clash);
|
||||
// 发布前校验:两份文件都必须能按写入内容读回,避免把半截内容挂上指针。
|
||||
verify(dir.resolve(V2_FILE), v2);
|
||||
verify(dir.resolve(CLASH_FILE), clash);
|
||||
publishManifest(account, generation, sha256(v2), sha256(clash));
|
||||
published = true;
|
||||
} finally {
|
||||
if (!published)
|
||||
deleteRecursively(dir);
|
||||
}
|
||||
prune(account, generation);
|
||||
removeLegacy(account);
|
||||
return new Pair(generation, v2, clash);
|
||||
}
|
||||
|
||||
/** 读取当前发布内容;只有构成一整个有效版本时才返回。 */
|
||||
public Optional<Pair> read(int accountId) {
|
||||
return Optional.ofNullable(load(accountDir(accountId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析当前应分发的单个文件。
|
||||
*
|
||||
* <p>与 {@link #read}、{@link #hasComplete} 使用同一份配对契约:只有两份文件
|
||||
* 齐全且哈希校验通过时才返回路径,因此分发路径不会发出半份或已损坏的缓存。
|
||||
*/
|
||||
public Optional<Path> resolve(int accountId, String client) {
|
||||
Path account = accountDir(accountId);
|
||||
Pair pair = load(account);
|
||||
if (pair == null)
|
||||
return Optional.empty();
|
||||
Path path = LEGACY_GENERATION.equals(pair.generation())
|
||||
? account.resolve(fileName(client))
|
||||
: account.resolve(GENERATION_DIR).resolve(pair.generation()).resolve(fileName(client));
|
||||
return Files.isRegularFile(path) ? Optional.of(path) : Optional.empty();
|
||||
}
|
||||
|
||||
/** 是否已有可分发的一对内容(分代两份都在且哈希匹配,或旧结构两份都在)。 */
|
||||
public boolean hasComplete(int accountId) {
|
||||
return load(accountDir(accountId)) != null;
|
||||
}
|
||||
|
||||
/** 删除该账号的全部缓存(指针、分代、旧结构文件),使后续请求判定为无缓存。 */
|
||||
public void invalidate(int accountId) {
|
||||
Path account = accountDir(accountId);
|
||||
deleteIfExists(account.resolve(MANIFEST_FILE));
|
||||
deleteRecursively(account.resolve(GENERATION_DIR));
|
||||
deleteIfExists(account.resolve(V2_FILE));
|
||||
deleteIfExists(account.resolve(CLASH_FILE));
|
||||
}
|
||||
|
||||
/** 写入接缝:测试覆写以模拟第二份文件写入失败。 */
|
||||
protected void writeFile(Path target, byte[] data) throws IOException {
|
||||
atomicWrite(target, data);
|
||||
}
|
||||
|
||||
/** 指针发布接缝:测试覆写以模拟 manifest 发布失败。 */
|
||||
protected void publishManifest(Path accountDir, String generation, String v2Sha256, String clashSha256)
|
||||
throws IOException {
|
||||
atomicWrite(accountDir.resolve(MANIFEST_FILE),
|
||||
manifestText(generation, v2Sha256, clashSha256).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取「一整个版本」:一次读入指针快照,再按该快照定位分代并校验双文件哈希。
|
||||
*
|
||||
* <p>指针只读一次,因此并发 {@link #publish} 切换指针时不会出现「用新分代名配旧哈希」
|
||||
* 或哈希字段缺失导致的空指针;要么拿到完整的新版本,要么拿到完整的旧版本。
|
||||
*/
|
||||
private Pair load(Path account) {
|
||||
Properties manifest = readManifest(account);
|
||||
if (manifest == null) {
|
||||
// 指针文件不存在才是「旧结构」。指针存在但读不出来说明它已损坏,同样按无缓存处理。
|
||||
return Files.isRegularFile(account.resolve(MANIFEST_FILE)) ? null : loadLegacy(account);
|
||||
}
|
||||
Pair generation = loadGeneration(account, manifest);
|
||||
if (generation != null)
|
||||
return generation;
|
||||
// 指针存在却读不出完整版本(半写、被篡改、内容缺失)时不得回退到 legacy:
|
||||
// 那会发出一个比指针更旧的版本,掩盖损坏。此时按无可用缓存处理。
|
||||
return null;
|
||||
}
|
||||
|
||||
private Pair loadGeneration(Path account, Properties manifest) {
|
||||
String generation = manifest.getProperty(MANIFEST_GENERATION);
|
||||
if (generation == null || generation.isBlank() || generation.contains("/") || generation.contains("..")) {
|
||||
log.warn("订阅缓存指针内容非法");
|
||||
return null;
|
||||
}
|
||||
String v2Sha256 = manifest.getProperty(MANIFEST_V2_SHA);
|
||||
String clashSha256 = manifest.getProperty(MANIFEST_CLASH_SHA);
|
||||
if (v2Sha256 == null || clashSha256 == null) {
|
||||
log.warn("订阅缓存指针缺少哈希字段 generation={}", generation);
|
||||
return null;
|
||||
}
|
||||
Path dir = account.resolve(GENERATION_DIR).resolve(generation);
|
||||
byte[] v2 = readFile(dir.resolve(V2_FILE));
|
||||
byte[] clash = readFile(dir.resolve(CLASH_FILE));
|
||||
if (v2 == null || clash == null) {
|
||||
log.warn("订阅缓存分代内容缺失 generation={}", generation);
|
||||
return null;
|
||||
}
|
||||
if (!matches(v2, v2Sha256) || !matches(clash, clashSha256)) {
|
||||
log.warn("订阅缓存分代内容校验失败 generation={}", generation);
|
||||
return null;
|
||||
}
|
||||
return new Pair(generation, v2, clash);
|
||||
}
|
||||
|
||||
/** 旧结构:只有账号根下两份文件都在才构成一个有效版本。 */
|
||||
private Pair loadLegacy(Path account) {
|
||||
byte[] v2 = readFile(account.resolve(V2_FILE));
|
||||
byte[] clash = readFile(account.resolve(CLASH_FILE));
|
||||
if (v2 == null || clash == null)
|
||||
return null;
|
||||
return new Pair(LEGACY_GENERATION, v2, clash);
|
||||
}
|
||||
|
||||
/** 指针读取接缝:包内可见,便于测试统计「每次查询只读一次指针」。 */
|
||||
Properties readManifest(Path account) {
|
||||
Path manifest = account.resolve(MANIFEST_FILE);
|
||||
if (!Files.isRegularFile(manifest))
|
||||
return null;
|
||||
try {
|
||||
Properties properties = new Properties();
|
||||
properties.load(new StringReader(Files.readString(manifest, StandardCharsets.UTF_8)));
|
||||
return properties;
|
||||
} catch (IOException e) {
|
||||
log.warn("读取订阅缓存指针失败: {}", e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] readFile(Path path) {
|
||||
try {
|
||||
return Files.isRegularFile(path) ? Files.readAllBytes(path) : null;
|
||||
} catch (IOException e) {
|
||||
log.warn("读取订阅缓存文件失败 path={} error={}", path, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void verify(Path path, byte[] expected) throws IOException {
|
||||
byte[] written = Files.readAllBytes(path);
|
||||
if (written.length != expected.length || !MessageDigest.isEqual(written, expected))
|
||||
throw new IOException("订阅缓存写入校验失败 " + path.getFileName());
|
||||
}
|
||||
|
||||
private static boolean matches(byte[] content, String expectedSha256) {
|
||||
return expectedSha256 != null && expectedSha256.equals(sha256(content));
|
||||
}
|
||||
|
||||
private static String manifestText(String generation, String v2Sha256, String clashSha256) {
|
||||
return MANIFEST_GENERATION + "=" + generation + "\n"
|
||||
+ MANIFEST_V2_SHA + "=" + v2Sha256 + "\n"
|
||||
+ MANIFEST_CLASH_SHA + "=" + clashSha256 + "\n";
|
||||
}
|
||||
|
||||
/** 分代名定宽可排序:时间戳在前保证单调,序号消除同毫秒碰撞。 */
|
||||
private static String nextGeneration() {
|
||||
return String.format("%013d-%06d", System.currentTimeMillis(), SEQUENCE.incrementAndGet() % 1_000_000);
|
||||
}
|
||||
|
||||
private void prune(Path account, String current) {
|
||||
Path dir = account.resolve(GENERATION_DIR);
|
||||
List<Path> generations;
|
||||
try (Stream<Path> entries = Files.list(dir)) {
|
||||
generations = entries.filter(Files::isDirectory)
|
||||
.sorted(Comparator.comparing(path -> path.getFileName().toString(), Comparator.reverseOrder()))
|
||||
.toList();
|
||||
} catch (IOException e) {
|
||||
log.warn("清理订阅缓存分代失败 path={} error={}", dir, e.getMessage());
|
||||
return;
|
||||
}
|
||||
for (int index = RETAINED_GENERATIONS; index < generations.size(); index++) {
|
||||
Path stale = generations.get(index);
|
||||
if (!stale.getFileName().toString().equals(current))
|
||||
deleteRecursively(stale);
|
||||
}
|
||||
}
|
||||
|
||||
/** 分代发布成功后旧结构文件被取代,避免两份来源同时存在。 */
|
||||
private void removeLegacy(Path account) {
|
||||
deleteIfExists(account.resolve(V2_FILE));
|
||||
deleteIfExists(account.resolve(CLASH_FILE));
|
||||
}
|
||||
|
||||
private static void atomicWrite(Path target, byte[] data) throws IOException {
|
||||
Path temp = target.resolveSibling(target.getFileName() + ".tmp");
|
||||
Files.write(temp, data);
|
||||
try {
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (AtomicMoveNotSupportedException e) {
|
||||
Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
}
|
||||
|
||||
private static void deleteIfExists(Path path) {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException e) {
|
||||
log.warn("删除订阅缓存文件失败 path={} error={}", path, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static void deleteRecursively(Path path) {
|
||||
if (!Files.exists(path))
|
||||
return;
|
||||
try (Stream<Path> entries = Files.walk(path)) {
|
||||
entries.sorted(Comparator.reverseOrder()).forEach(SubscriptionCacheStore::deleteIfExists);
|
||||
} catch (IOException e) {
|
||||
log.warn("删除订阅缓存目录失败 path={} error={}", path, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static String sha256(byte[] input) {
|
||||
try {
|
||||
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(input));
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user