17 changed files with 1044 additions and 18 deletions
+30
View File
@@ -0,0 +1,30 @@
# 备用订阅更新事件
主站 `/sub/{client}/{key}` 接受网关的签名事件;`GET` 在客户端类型与缓存路径有效后,同一个主库事务写更新记录和长期收据。签名有效的 `HEAD` 不写记录。普通请求以及无效签名仍走原来的更新记录逻辑。
`POST /internal/subscription-events` 有界读取原始请求(最多 256 KiB),批次最多 100 条。签名算法为 HMAC-SHA256,同步密钥来自 `SUBSCRIPTION_SYNC_SECRET`;签名输入是 `v1\n` + epoch 秒时间戳 + `\n` + 原始请求体 SHA256 十六进制。时间偏差最多 300 秒。请求头为 `X-Subscription-Timestamp` 和 `X-Subscription-Signature`。主站 GET 将单个事件的原始 JSON 使用无 padding 的 base64url 编码放入 `X-Subscription-Event`,同样对原始事件 JSON 签名。
事件字段:`eventId`(规范小写 UUID)、`keyHash`(公开 key 的小写 SHA256)、`client`(v2 或 cat)、`clientIp`(IPv4/IPv6 字面值)、`userAgent`(最多 2048 字符)、`occurredAt`(epoch 毫秒)。允许历史事件,事件时间不可超过未来 5 分钟。签名中的 IP 为权威来源,不能由未经签名的 XFF 覆盖。
响应 `acknowledged` 只在事务已提交后返回,包括已存在的相同事件。冲突事件 ID 返回 400;签名无效返回 403;数据库异常返回 5xx,无确认列表。未知 hash 不确认;历史映射中用户已删除则写终态收据并确认,不猜测用户归属。可见记录按事件时间和 ID 倒序保留每用户最近 10 条,收据独立长期保留。批次任意数据库写入失败会全部回滚。
## 显式迁移
运行前先执行稳定 user_id 关联迁移。新表不会在启动时创建。
1. 先在数据库副本上演练:`python3 scripts/migrate_subscription_events.py /path/to/copy.db`。
2. 停止生产主站服务,确认没有其他写入者。
3. 执行 `python3 scripts/migrate_subscription_events.py /path/to/main.db --apply --service-stopped`。
4. 检查输出 integrity 为 ok,保留输出中的 SQLite 备份,然后启动新程序。
脚本在写入前使用 SQLite backup API 生成权限 0600 的备份,在单个事务中建表并初始化历史 hash,重复执行保留既有收据和历史。不将公开 key 写入历史表或输出;原数据库备份含既有敏感数据,须保留原有保护措施。
快照构建记住所有现有绑定;重置或删除绑定前也保存历史 hash。用户改名时回传事件按 stable user_id 读取当前名字。合法旧订阅标签的 user_id 可以为 NULL,历史映射保留标签并按原标签写回,不创建用户,也不猜测同名新用户归属。可见日志删除不删除历史映射或收据。
Java 编译与测试必须在部署手册指定的 192.168.5.30 执行:
```sh
mvn -Dtest=SubscriptionEventIntegrationTest,SubServiceTest,SubscriptionStandbySnapshotServiceTest,MyBatisNativeConfigurationTest test
```
Python 迁移验证可本机执行:`python3 -m unittest discover -s scripts -p 'test_subscription_events.py' -v`。
+2
View File
@@ -15,6 +15,8 @@
<description>LionWebsite</description> <description>LionWebsite</description>
<properties> <properties>
<java.version>21</java.version> <java.version>21</java.version>
<tomcat.version>11.0.27</tomcat.version>
<jackson-bom.version>3.1.7</jackson-bom.version>
</properties> </properties>
<dependencies> <dependencies>
<dependency> <dependency>
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Create durable subscription event receipts/history, with a SQLite backup.
Stop the production backend before --apply and pass --service-stopped to confirm.
No automatic startup DDL exists. Dry run is read-only; only hashes enter history.
Receipt retention is independent of the most recent ten visible update records.
"""
import argparse
import datetime
import hashlib
import json
from pathlib import Path
import sqlite3
DDL = (
"""CREATE TABLE IF NOT EXISTS subscription_event_receipt (
event_id TEXT PRIMARY KEY NOT NULL, key_hash TEXT NOT NULL,
client TEXT NOT NULL, occurred_at INTEGER NOT NULL, source TEXT NOT NULL,
ingested_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, outcome TEXT NOT NULL
)""",
"""CREATE TABLE IF NOT EXISTS subscription_event_binding (
key_hash TEXT PRIMARY KEY NOT NULL, user_id INTEGER,
user_name TEXT NOT NULL, first_seen_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL
)""",
"CREATE INDEX IF NOT EXISTS idx_subscription_event_binding_user ON subscription_event_binding(user_id)",
"CREATE INDEX IF NOT EXISTS idx_sub_record_user_time ON sub_update_record(user_id,time DESC,id DESC)",
)
def migrate(database: Path, apply=False, service_stopped=False):
if not database.is_file():
raise ValueError("database must already exist")
if apply and not service_stopped:
raise ValueError("stop backend and supply --service-stopped before applying")
uri = database.resolve().as_uri() + ("?mode=rw" if apply else "?mode=ro")
with sqlite3.connect(uri, uri=True) as db:
for table in ("User", "sub_bind", "sub_update_record"):
columns = {row[1] for row in db.execute(f'PRAGMA table_info("{table}")')}
if not columns or (table != "User" and "user_id" not in columns):
raise ValueError("stable user links migration is required first")
unresolved = db.execute("SELECT count(*) FROM sub_bind sb LEFT JOIN User u ON u.id=sb.user_id WHERE sb.user_id IS NOT NULL AND u.id IS NULL").fetchone()[0]
if unresolved:
raise ValueError("resolve non-null dangling subscription user IDs before migration")
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
result = {"apply": apply, "tables_to_create": sorted({"subscription_event_receipt", "subscription_event_binding"} - tables)}
if not apply:
return result
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
backup = database.with_name(database.name + ".before-subscription-events." + stamp)
# Create mode 600 before any sensitive database bytes are written.
backup.touch(mode=0o600, exist_ok=False)
with sqlite3.connect(backup) as snapshot:
db.backup(snapshot)
try:
db.execute("BEGIN IMMEDIATE")
for statement in DDL:
db.execute(statement)
now = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000)
for key, user_id, name in db.execute("SELECT sb.key,sb.user_id,coalesce(u.username,sb.user) FROM sub_bind sb LEFT JOIN User u ON u.id=sb.user_id"):
key_hash = hashlib.sha256(key.encode("utf-8")).hexdigest()
db.execute("""INSERT INTO subscription_event_binding VALUES(?,?,?,?,?)
ON CONFLICT(key_hash) DO UPDATE SET user_name=excluded.user_name,last_seen_at=excluded.last_seen_at
WHERE subscription_event_binding.user_id IS excluded.user_id""", (key_hash, user_id, name, now, now))
db.commit()
except BaseException:
db.rollback()
raise
result["backup"] = str(backup)
result["history_bindings"] = db.execute("SELECT count(*) FROM subscription_event_binding").fetchone()[0]
result["integrity"] = db.execute("PRAGMA quick_check").fetchone()[0]
return result
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("database", type=Path)
parser.add_argument("--apply", action="store_true")
parser.add_argument("--service-stopped", action="store_true")
args = parser.parse_args()
print(json.dumps(migrate(args.database, args.apply, args.service_stopped), ensure_ascii=False))
+82
View File
@@ -0,0 +1,82 @@
import importlib.util
from pathlib import Path
import sqlite3
import tempfile
import unittest
import hashlib
spec = importlib.util.spec_from_file_location("migration", Path(__file__).with_name("migrate_subscription_events.py"))
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
class SubscriptionEventsMigrationTest(unittest.TestCase):
def test_explicit_backup_atomic_and_idempotent(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "main.db"
with sqlite3.connect(path) as db:
db.executescript("""
CREATE TABLE User(id INTEGER PRIMARY KEY,username TEXT);
CREATE TABLE sub_bind(key TEXT,user TEXT,user_id INTEGER);
CREATE TABLE sub_update_record(id INTEGER PRIMARY KEY,user_id INTEGER,time INTEGER);
INSERT INTO User VALUES(3,'alice');
INSERT INTO sub_bind VALUES('public-test-key','alice',3);
""")
dry = migration.migrate(path)
self.assertEqual(2, len(dry["tables_to_create"]))
with self.assertRaises(ValueError):
migration.migrate(path, True)
first = migration.migrate(path, True, True)
self.assertEqual("ok", first["integrity"])
self.assertEqual(0o600, Path(first["backup"]).stat().st_mode & 0o777)
with sqlite3.connect(first["backup"]) as db:
self.assertEqual(0, db.execute("SELECT count(*) FROM sqlite_master WHERE name='subscription_event_binding'").fetchone()[0])
with sqlite3.connect(path) as db:
row = db.execute("SELECT key_hash,user_id,user_name FROM subscription_event_binding").fetchone()
self.assertEqual((hashlib.sha256(b"public-test-key").hexdigest(), 3, "alice"), row)
db.execute("INSERT INTO subscription_event_receipt VALUES('id','hash','v2',0,'primary',1,'fingerprint','recorded')")
second = migration.migrate(path, True, True)
self.assertEqual([], second["tables_to_create"])
with sqlite3.connect(path) as db:
self.assertEqual(1, db.execute("SELECT count(*) FROM subscription_event_receipt").fetchone()[0])
self.assertEqual(1, db.execute("SELECT count(*) FROM subscription_event_binding").fetchone()[0])
def test_legacy_labels_without_user_ids_remain_null(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "main.db"
with sqlite3.connect(path) as db:
db.executescript("""
CREATE TABLE User(id INTEGER PRIMARY KEY,username TEXT);
CREATE TABLE sub_bind(key TEXT,user TEXT,user_id INTEGER);
CREATE TABLE sub_update_record(id INTEGER PRIMARY KEY,user_id INTEGER,time INTEGER);
INSERT INTO sub_bind VALUES('legacy-key','legacy-label',NULL);
""")
result = migration.migrate(path, True, True)
self.assertEqual(1, result["history_bindings"])
with sqlite3.connect(path) as db:
row = db.execute("SELECT key_hash,user_id,user_name FROM subscription_event_binding").fetchone()
self.assertEqual((hashlib.sha256(b"legacy-key").hexdigest(), None, "legacy-label"), row)
self.assertIsNone(db.execute("SELECT user_id FROM sub_bind").fetchone()[0])
self.assertEqual(0, db.execute("SELECT count(*) FROM User").fetchone()[0])
migration.migrate(path, True, True)
with sqlite3.connect(path) as db:
self.assertEqual(1, db.execute("SELECT count(*) FROM subscription_event_binding").fetchone()[0])
def test_orphan_binding_aborts_without_ddl(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "main.db"
with sqlite3.connect(path) as db:
db.executescript("""
CREATE TABLE User(id INTEGER PRIMARY KEY,username TEXT);
CREATE TABLE sub_bind(key TEXT,user TEXT,user_id INTEGER);
CREATE TABLE sub_update_record(id INTEGER PRIMARY KEY,user_id INTEGER,time INTEGER);
INSERT INTO sub_bind VALUES('test','deleted',9);
""")
with self.assertRaises(ValueError):
migration.migrate(path, True, True)
with sqlite3.connect(path) as db:
self.assertEqual(0, db.execute("SELECT count(*) FROM sqlite_master WHERE name LIKE 'subscription_event_%'").fetchone()[0])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,33 @@
package com.lion.lionwebsite.Controller;
import com.lion.lionwebsite.Service.SubscriptionEventService;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.Map;
@RestController
public class SubscriptionEventController {
private final SubscriptionEventService events;
public SubscriptionEventController(SubscriptionEventService events) { this.events = events; }
@PostMapping(value = "/internal/subscription-events", produces = "application/json")
public ResponseEntity<Map<String, Object>> ingest(HttpServletRequest request) throws IOException {
// Bound actual bytes even for chunked requests and dishonest Content-Length headers.
byte[] body = request.getInputStream().readNBytes(SubscriptionEventService.MAX_BODY + 1);
if (body.length > SubscriptionEventService.MAX_BODY)
return ResponseEntity.badRequest().body(Map.of("error", "invalid subscription events"));
if (!events.verify(body, request.getHeader("X-Subscription-Timestamp"), request.getHeader("X-Subscription-Signature")))
return ResponseEntity.status(403).body(Map.of("error", "authentication failed"));
try {
return ResponseEntity.ok(events.ingest(events.parseBatch(body)));
} catch (IllegalArgumentException invalid) {
return ResponseEntity.badRequest().body(Map.of("error", "invalid subscription events"));
}
// All infrastructure failures propagate as 5xx; never acknowledge an uncommitted batch.
}
}
@@ -80,7 +80,7 @@ public interface SubMapper {
@Select("select count(user) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") @Select("select count(user) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})")
Integer selectUpdateRecordCount(String user); Integer selectUpdateRecordCount(String user);
@Select("select min(id) from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user})") @Select("select id from sub_update_record where user_id=(select id from User where username=#{user}) or (user_id is null and user=#{user}) order by time asc,id asc limit 1")
Integer selectMinUpdateRecordId(String user); Integer selectMinUpdateRecordId(String user);
/** 后台补齐归属地:只更新该用户最近一条记录,避免覆盖更早的历史记录。 */ /** 后台补齐归属地:只更新该用户最近一条记录,避免覆盖更早的历史记录。 */
@@ -0,0 +1,35 @@
package com.lion.lionwebsite.Dao.normal;
import com.lion.lionwebsite.Domain.SubscriptionEvent;
import com.lion.lionwebsite.Domain.SubUpdateRecord;
import com.lion.lionwebsite.Domain.SubscriptionEventBinding;
import org.apache.ibatis.annotations.*;
@Mapper
public interface SubscriptionEventMapper {
/** Reserve SQLite's writer before the receipt read; a deferred read-to-write upgrade can fail immediately. */
@Update("update subscription_event_receipt set ingested_at=ingested_at where 0")
void reserveWriter();
@Select("select fingerprint from subscription_event_receipt where event_id=#{eventId}")
String fingerprint(String eventId);
@Insert("insert into subscription_event_receipt(event_id,key_hash,client,occurred_at,source,ingested_at,fingerprint,outcome) values(#{event.eventId},#{event.keyHash},#{event.client},#{event.occurredAt},#{source},#{ingestedAt},#{fingerprint},#{outcome})")
void insertReceipt(@Param("event") SubscriptionEvent event, @Param("source") String source,
@Param("ingestedAt") long ingestedAt, @Param("fingerprint") String fingerprint,
@Param("outcome") String outcome);
@Select("select key_hash as keyHash,user_id as userId,user_name as userName from subscription_event_binding where key_hash=#{hash}")
SubscriptionEventBinding binding(String hash);
@Insert("insert into subscription_event_binding(key_hash,user_id,user_name,first_seen_at,last_seen_at) values(#{hash},#{userId},#{userName},#{now},#{now}) on conflict(key_hash) do update set user_name=excluded.user_name,last_seen_at=excluded.last_seen_at where subscription_event_binding.user_id IS excluded.user_id")
void rememberBinding(@Param("hash") String hash, @Param("userId") Integer userId,
@Param("userName") String userName, @Param("now") long now);
@Insert("insert into sub_update_record(user,ip,UA,time,location,user_id) values(#{user},#{ip},#{UA},#{time},#{location},#{userId})")
@Options(useGeneratedKeys = true, keyProperty = "id")
void insertRecord(SubUpdateRecord record);
@Delete("delete from sub_update_record where (user_id=#{userId} or (user_id is null and user=#{userName})) and id not in (select id from sub_update_record where user_id=#{userId} or (user_id is null and user=#{userName}) order by time desc,id desc limit 10)")
void pruneRecords(@Param("userId") Integer userId, @Param("userName") String userName);
}
@@ -0,0 +1,16 @@
package com.lion.lionwebsite.Domain;
import lombok.AllArgsConstructor;
import lombok.Data;
/** Validated, authenticated event. Never contains the public subscription key. */
@Data
@AllArgsConstructor
public class SubscriptionEvent {
private String eventId;
private String keyHash;
private String client;
private String clientIp;
private String userAgent;
private long occurredAt;
}
@@ -0,0 +1,10 @@
package com.lion.lionwebsite.Domain;
import lombok.Data;
@Data
public class SubscriptionEventBinding {
private String keyHash;
private Integer userId;
private String userName;
}
@@ -5,6 +5,7 @@ import com.lion.lionwebsite.Dao.normal.SubMapper;
import com.lion.lionwebsite.Dao.normal.UserMapper; import com.lion.lionwebsite.Dao.normal.UserMapper;
import com.lion.lionwebsite.Domain.SubBind; import com.lion.lionwebsite.Domain.SubBind;
import com.lion.lionwebsite.Domain.SubUpdateRecord; import com.lion.lionwebsite.Domain.SubUpdateRecord;
import com.lion.lionwebsite.Domain.SubscriptionEvent;
import com.lion.lionwebsite.Domain.SubscriptionAccount; import com.lion.lionwebsite.Domain.SubscriptionAccount;
import com.lion.lionwebsite.Util.CustomUtil; import com.lion.lionwebsite.Util.CustomUtil;
import com.lion.lionwebsite.Util.FileDownload; import com.lion.lionwebsite.Util.FileDownload;
@@ -47,6 +48,9 @@ import java.util.function.Supplier;
@Qualifier("mainTransactionTemplate") @Qualifier("mainTransactionTemplate")
TransactionTemplate transactions; TransactionTemplate transactions;
@Autowired(required = false)
SubscriptionEventService subscriptionEvents;
/** /**
* 归属地查询是同步外呼(最长十几秒),且失败只影响记录里的一列展示。 * 归属地查询是同步外呼(最长十几秒),且失败只影响记录里的一列展示。
* 若放在订阅分发路径上,用户取一次订阅就得先等它完成,因此改为后台补齐。 * 若放在订阅分发路径上,用户取一次订阅就得先等它完成,因此改为后台补齐。
@@ -188,6 +192,7 @@ import java.util.function.Supplier;
String key = RandomUtil.randomString(8); String key = RandomUtil.randomString(8);
while (subMapper.selectSubBindExist(key)) while (subMapper.selectSubBindExist(key))
key = RandomUtil.randomString(8); key = RandomUtil.randomString(8);
rememberEventBindings();
subMapper.updateSubBindKey(user, key); subMapper.updateSubBindKey(user, key);
subMapper.deleteSubUpdateRecord(user); subMapper.deleteSubUpdateRecord(user);
remoteService.requestSubscriptionSync(); remoteService.requestSubscriptionSync();
@@ -239,8 +244,9 @@ import java.util.function.Supplier;
} }
String ip = resolveClientIp(request); String ip = resolveClientIp(request);
String ua = request.getHeader("User-Agent"); String ua = request.getHeader("User-Agent");
if (ua == null) return; SubscriptionEvent event = subscriptionEvents == null ? null : subscriptionEvents.requestEvent(request, key, client);
recordUpdate(subBind, ip, ua); if (ua == null && event == null) return;
if (event == null) recordUpdate(subBind, ip, ua);
if (!"v2".equals(client) && !"cat".equals(client)) { if (!"v2".equals(client) && !"cat".equals(client)) {
sendStatus(response, HttpServletResponse.SC_BAD_REQUEST, "client error"); sendStatus(response, HttpServletResponse.SC_BAD_REQUEST, "client error");
return; return;
@@ -250,6 +256,8 @@ import java.util.function.Supplier;
sendStatus(response, HttpServletResponse.SC_SERVICE_UNAVAILABLE, "subscription unavailable"); sendStatus(response, HttpServletResponse.SC_SERVICE_UNAVAILABLE, "subscription unavailable");
return; return;
} }
if (event != null && !"HEAD".equalsIgnoreCase(request.getMethod()))
subscriptionEvents.recordPrimary(event, subBind);
FileDownload.export(request, response, path.toString()); FileDownload.export(request, response, path.toString());
} }
@@ -329,12 +337,17 @@ import java.util.function.Supplier;
} }
private String deleteSubBindUnlocked(String user) { private String deleteSubBindUnlocked(String user) {
rememberEventBindings();
subMapper.deleteSubBind(user); subMapper.deleteSubBind(user);
subMapper.deleteSubUpdateRecord(user); subMapper.deleteSubUpdateRecord(user);
remoteService.requestSubscriptionSync(); remoteService.requestSubscriptionSync();
return Response._success("删除成功"); return Response._success("删除成功");
} }
private void rememberEventBindings() {
if (subscriptionEvents != null) subscriptionEvents.rememberBindings(subMapper.selectAllSubBind());
}
private String withWriteLock(Supplier<String> action) { private String withWriteLock(Supplier<String> action) {
Lock stateLock = stateCoordinator.writeLock(); Lock stateLock = stateCoordinator.writeLock();
stateLock.lock(); stateLock.lock();
@@ -0,0 +1,224 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Dao.normal.SubMapper;
import com.lion.lionwebsite.Dao.normal.SubscriptionEventMapper;
import com.lion.lionwebsite.Dao.normal.UserMapper;
import com.lion.lionwebsite.Domain.*;
import com.lion.lionwebsite.Util.CustomUtil;
import jakarta.annotation.PreDestroy;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import tools.jackson.databind.JsonNode;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.Clock;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.locks.Lock;
@Service
public class SubscriptionEventService {
public static final int MAX_BODY = 256 * 1024;
private final SubscriptionEventMapper events;
private final SubMapper subscriptions;
private final UserMapper users;
private final TransactionTemplate transactions;
private final SubscriptionStateCoordinator coordinator;
@Value("${subscription.standby.sync-secret:}") String syncSecret;
Clock clock = Clock.systemUTC();
private final ExecutorService locations = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(128), runnable -> {
Thread thread = new Thread(runnable, "subscription-event-location");
thread.setDaemon(true);
return thread;
}, new ThreadPoolExecutor.AbortPolicy());
public SubscriptionEventService(SubscriptionEventMapper events, SubMapper subscriptions, UserMapper users,
@Qualifier("mainTransactionTemplate") TransactionTemplate transactions,
SubscriptionStateCoordinator coordinator) {
this.events = events;
this.subscriptions = subscriptions;
this.users = users;
this.transactions = transactions;
this.coordinator = coordinator;
}
/** Auth failure deliberately discloses neither signature nor payload. */
public boolean verify(byte[] body, String timestamp, String signature) {
if (body == null || body.length > MAX_BODY || syncSecret == null || syncSecret.isBlank()
|| timestamp == null || !timestamp.matches("[0-9]{1,12}")
|| signature == null || !signature.matches("[0-9a-fA-F]{64}")) return false;
try {
long seconds = Long.parseLong(timestamp);
if (Math.abs(clock.instant().getEpochSecond() - seconds) > 300) return false;
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(new SecretKeySpec(syncSecret.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
byte[] expected = mac.doFinal(("v1\n" + timestamp + "\n" + sha256(body)).getBytes(StandardCharsets.UTF_8));
return MessageDigest.isEqual(expected, HexFormat.of().parseHex(signature));
} catch (Exception invalid) { return false; }
}
/** Invalid or absent signed headers fall back to the ordinary logging path. */
public SubscriptionEvent requestEvent(HttpServletRequest request, String key, String client) {
String encoded = request.getHeader("X-Subscription-Event");
if (encoded == null || encoded.length() > 8192 || !encoded.matches("[A-Za-z0-9_-]+")) return null;
try {
byte[] body = Base64.getUrlDecoder().decode(encoded);
if (!verify(body, request.getHeader("X-Subscription-Timestamp"), request.getHeader("X-Subscription-Signature"))) return null;
SubscriptionEvent event = parse(CustomUtil.objectMapper.readTree(body));
String ua = Objects.requireNonNullElse(request.getHeader("User-Agent"), "");
if (!event.getKeyHash().equals(sha256(key.getBytes(StandardCharsets.UTF_8)))
|| !event.getClient().equals(client) || !event.getUserAgent().equals(ua)) return null;
return event;
} catch (Exception invalid) { return null; }
}
public List<SubscriptionEvent> parseBatch(byte[] body) {
try {
if (body == null || body.length > MAX_BODY) throw new IllegalArgumentException();
JsonNode root = CustomUtil.objectMapper.readTree(body);
JsonNode list = root.get("events");
if (!root.isObject() || list == null || !list.isArray() || list.size() > 100) throw new IllegalArgumentException();
List<SubscriptionEvent> result = new ArrayList<>();
for (JsonNode item : list) result.add(parse(item));
return result;
} catch (Exception invalid) { throw new IllegalArgumentException("invalid subscription events"); }
}
SubscriptionEvent parse(JsonNode node) {
if (node == null || !node.isObject()) throw new IllegalArgumentException();
String id = text(node, "eventId", 36);
if (!UUID.fromString(id).toString().equals(id)) throw new IllegalArgumentException();
String hash = text(node, "keyHash", 64);
if (!hash.matches("[0-9a-f]{64}")) throw new IllegalArgumentException();
String client = text(node, "client", 3);
if (!client.equals("v2") && !client.equals("cat")) throw new IllegalArgumentException();
String ip = text(node, "clientIp", 64);
// Literal IPv4/IPv6 only: no host names, URL fragments or query injection into geolocation.
if (ip.isEmpty() || !ip.matches("[0-9A-Fa-f:.]+")) throw new IllegalArgumentException();
String ua = text(node, "userAgent", 2048);
if (ua.chars().anyMatch(c -> c < 32 || c == 127)) throw new IllegalArgumentException();
JsonNode time = node.get("occurredAt");
if (time == null || !time.isIntegralNumber() || !time.canConvertToLong()) throw new IllegalArgumentException();
long occurred = time.longValue();
if (occurred < 0 || occurred > clock.millis() + 300_000) throw new IllegalArgumentException();
return new SubscriptionEvent(id, hash, client, ip, ua, occurred);
}
private static String text(JsonNode node, String field, int max) {
JsonNode value = node.get(field);
if (value == null || !value.isString() || value.stringValue().length() > max) throw new IllegalArgumentException();
return value.stringValue();
}
public Map<String, Object> ingest(List<SubscriptionEvent> batch) {
Lock lock = coordinator.readLock();
lock.lock();
try {
// Return acknowledgements only after execute has returned and the entire transaction committed.
List<String> acknowledged = transactions.execute(status -> {
events.reserveWriter();
List<String> ids = new ArrayList<>();
for (SubscriptionEvent event : batch) if (persist(event, "standby", null)) ids.add(event.getEventId());
return ids;
});
return Map.of("acknowledged", Objects.requireNonNull(acknowledged));
} finally { lock.unlock(); }
}
public void recordPrimary(SubscriptionEvent event, SubBind binding) {
transactions.executeWithoutResult(status -> {
events.reserveWriter();
if (!persist(event, "primary", binding)) throw new IllegalStateException("subscription user unavailable");
});
}
private boolean persist(SubscriptionEvent event, String source, SubBind current) {
String fingerprint = sha256((event.getEventId() + "\n" + event.getKeyHash() + "\n" + event.getClient()
+ "\n" + event.getOccurredAt() + "\n" + event.getClientIp() + "\n" + event.getUserAgent()).getBytes(StandardCharsets.UTF_8));
String old = events.fingerprint(event.getEventId());
if (old != null) {
if (!old.equals(fingerprint)) throw new IllegalArgumentException("event ID conflict");
return true;
}
Integer userId = current == null ? null : current.getUserId();
String userName = current == null ? null : current.getUser();
if (current == null) {
boolean found = false;
for (SubBind binding : subscriptions.selectAllSubBind()) {
if (sha256(binding.getKey().getBytes(StandardCharsets.UTF_8)).equals(event.getKeyHash())) {
userId = binding.getUserId();
userName = binding.getUser();
found = true;
break;
}
}
if (!found) {
SubscriptionEventBinding binding = events.binding(event.getKeyHash());
if (binding == null) return false;
userId = binding.getUserId();
userName = binding.getUserName();
}
}
if (userId != null) {
User user = users.selectUserById(userId);
if (user == null) {
events.insertReceipt(event, source, clock.millis(), fingerprint, "user_deleted");
return true;
}
userName = user.getUsername();
}
// NULL user_id is a valid legacy subscription label, not a missing/deleted User.
// Keep it NULL even if someone later creates a User with the same label.
if (userName == null || userName.isBlank()) return false;
events.rememberBinding(event.getKeyHash(), userId, userName, clock.millis());
SubUpdateRecord record = new SubUpdateRecord(0, userName, event.getClientIp(), event.getUserAgent(),
new Date(event.getOccurredAt()), "unknown", userId);
events.insertRecord(record);
events.pruneRecords(userId, userName);
events.insertReceipt(event, source, clock.millis(), fingerprint, "recorded");
// Never perform network calls before commit. A dropped/rolled-back record has no enrichment task.
org.springframework.transaction.support.TransactionSynchronizationManager.registerSynchronization(
new org.springframework.transaction.support.TransactionSynchronization() {
@Override public void afterCommit() { enrich(record.getId(), event.getClientIp()); }
});
return true;
}
public void rememberBindings(Collection<SubBind> bindings) {
for (SubBind binding : bindings) {
Integer userId = binding.getUserId();
String name = binding.getUser();
if (userId != null) {
User user = users.selectUserById(userId);
if (user != null) name = user.getUsername();
}
if (name != null && !name.isBlank()) events.rememberBinding(
sha256(binding.getKey().getBytes(StandardCharsets.UTF_8)), userId, name, clock.millis());
}
}
private void enrich(int recordId, String ip) {
if (recordId <= 0) return;
try { locations.execute(() -> {
String location = SubService.resolveLocation(ip);
if (location != null && !location.isBlank() && !location.equals("unknown")) {
try { subscriptions.updateSubUpdateRecordLocation(recordId, location); }
catch (Exception ignored) { /* Optional display enrichment; no request failure or sensitive log. */ }
}
}); } catch (RejectedExecutionException ignored) { }
}
public static String sha256(byte[] body) {
try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(body)); }
catch (Exception failure) { throw new IllegalStateException("SHA-256 unavailable"); }
}
@PreDestroy void close() { locations.shutdownNow(); }
}
@@ -9,6 +9,7 @@ import com.lion.lionwebsite.Util.CustomUtil;
import com.lion.lionwebsite.Util.SubscriptionCacheStore; import com.lion.lionwebsite.Util.SubscriptionCacheStore;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
@@ -29,6 +30,9 @@ public class SubscriptionStandbySnapshotService {
private final SubscriptionStateCoordinator stateCoordinator; private final SubscriptionStateCoordinator stateCoordinator;
private final ObjectMapper objectMapper = CustomUtil.objectMapper; private final ObjectMapper objectMapper = CustomUtil.objectMapper;
@Autowired(required = false)
SubscriptionEventService subscriptionEvents;
@Value("${subscription.standby.sync-secret:}") @Value("${subscription.standby.sync-secret:}")
String syncSecret; String syncSecret;
@@ -78,7 +82,9 @@ public class SubscriptionStandbySnapshotService {
List<SubscriptionAccountSnapshot> accounts = new ArrayList<>(accountMap.values()); List<SubscriptionAccountSnapshot> accounts = new ArrayList<>(accountMap.values());
accounts.sort(Comparator.comparing(SubscriptionAccountSnapshot::getAccountId)); accounts.sort(Comparator.comparing(SubscriptionAccountSnapshot::getAccountId));
List<SubscriptionBindingSnapshot> bindings = new ArrayList<>(); List<SubscriptionBindingSnapshot> bindings = new ArrayList<>();
for (SubBind bind : subMapper.selectAllSubBind()) { List<SubBind> currentBindings = subMapper.selectAllSubBind();
if (subscriptionEvents != null) subscriptionEvents.rememberBindings(currentBindings);
for (SubBind bind : currentBindings) {
if (bind.getSubscriptionAccountId() == null || !accountMap.containsKey(bind.getSubscriptionAccountId())) if (bind.getSubscriptionAccountId() == null || !accountMap.containsKey(bind.getSubscriptionAccountId()))
continue; continue;
SubscriptionBindingSnapshot snapshot = new SubscriptionBindingSnapshot(); SubscriptionBindingSnapshot snapshot = new SubscriptionBindingSnapshot();
@@ -10,12 +10,13 @@ import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.entity.EntityBuilder; import org.apache.hc.client5.http.entity.EntityBuilder;
import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout; import org.apache.hc.core5.util.Timeout;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
@@ -28,6 +29,8 @@ import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@@ -40,7 +43,7 @@ public class GalleryUtil {
static String POST = "post"; static String POST = "post";
static String GET = "get"; static String GET = "get";
static String FORM_DATA = "formData"; static String FORM_URLENCODED = "formUrlEncoded";
static String JSON = "json"; static String JSON = "json";
@@ -58,6 +61,8 @@ public class GalleryUtil {
/** Reusable HTTP client —不要每次请求新建 */ /** Reusable HTTP client —不要每次请求新建 */
private static final CloseableHttpClient httpClient = HttpClients.custom() private static final CloseableHttpClient httpClient = HttpClients.custom()
// An upstream download POST has no idempotency key; never replay it on 429/503.
.disableAutomaticRetries()
.setDefaultRequestConfig(RequestConfig.custom() .setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(5_000)) .setConnectTimeout(Timeout.ofMilliseconds(5_000))
.setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000)) .setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000))
@@ -104,8 +109,7 @@ public class GalleryUtil {
Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid); Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
String archiveLink = GalleryPageParser.archiveLink(galleryDoc); String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
HashMap<String, String> headers = new HashMap<>(); HashMap<String, String> headers = new HashMap<>();
headers.put("origin", origin); headers.put("referer", url);
headers.put("referer", archiveLink);
Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink); Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
String download_link = downloadDoc.select("#hathdl_form").attr("abs:action"); String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单"); if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
@@ -132,7 +136,8 @@ public class GalleryUtil {
//提交下载请求 //提交下载请求
HashMap<String, String> body = new HashMap<>(); HashMap<String, String> body = new HashMap<>();
body.put("payload", FORM_DATA); // The upstream form has no enctype: match the browser's default encoding.
body.put("payload", FORM_URLENCODED);
body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org")); body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
String downloadPage = requests(download_link, POST, headers, body); String downloadPage = requests(download_link, POST, headers, body);
Document downloadDoc = Jsoup.parse(downloadPage); Document downloadDoc = Jsoup.parse(downloadPage);
@@ -292,9 +297,18 @@ public class GalleryUtil {
if(url.length() < 40) if(url.length() < 40)
return null; return null;
if(!url.contains("/g/") || !url.contains("exhentai")) try {
URI uri = URI.create(url);
String scheme = uri.getScheme();
if ((!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme))
|| !"exhentai.org".equalsIgnoreCase(uri.getHost())
|| uri.getRawUserInfo() != null || uri.getPort() != -1
|| uri.getPath() == null || !uri.getPath().matches("/g/[0-9]+/[A-Za-z0-9]+/?"))
return null; return null;
return ""; return scheme.toLowerCase(Locale.ROOT) + "://exhentai.org";
} catch (IllegalArgumentException invalidUrl) {
return null;
}
} }
/** /**
@@ -329,16 +343,18 @@ public class GalleryUtil {
httpPost.addHeader(header.getKey(), header.getValue()); httpPost.addHeader(header.getKey(), header.getValue());
if(body != null) { if(body != null) {
// Keep the caller's form intact; payload is a local encoding selector.
body = new HashMap<>(body);
String payload; String payload;
if((payload = body.remove("payload")).equals(JSON)) { if((payload = body.remove("payload")).equals(JSON)) {
EntityBuilder entityBuilder = EntityBuilder.create(); EntityBuilder entityBuilder = EntityBuilder.create();
entityBuilder.setContentType(ContentType.APPLICATION_JSON); entityBuilder.setContentType(ContentType.APPLICATION_JSON);
entityBuilder.setText(objectMapper.writeValueAsString(body)); entityBuilder.setText(objectMapper.writeValueAsString(body));
httpPost.setEntity(entityBuilder.build()); httpPost.setEntity(entityBuilder.build());
} else if(payload.equals(FORM_DATA)) { } else if(payload.equals(FORM_URLENCODED)) {
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); var fields = body.entrySet().stream()
body.forEach(multipartEntityBuilder::addTextBody); .map(entry -> new BasicNameValuePair(entry.getKey(), entry.getValue())).toList();
httpPost.setEntity(multipartEntityBuilder.build()); httpPost.setEntity(new UrlEncodedFormEntity(fields, StandardCharsets.UTF_8));
} }
} }
httpResponse = httpClient.execute(httpPost); httpResponse = httpClient.execute(httpPost);
@@ -348,13 +364,15 @@ public class GalleryUtil {
int statusCode = httpResponse.getCode(); int statusCode = httpResponse.getCode();
StringBuilder stringBuilder = new StringBuilder(); StringBuilder stringBuilder = new StringBuilder();
if (statusCode == 200 && responseEntity != null) { if (statusCode == 200 && responseEntity != null) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent()))) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8))) {
String str; String str;
while ((str = reader.readLine()) != null) while ((str = reader.readLine()) != null)
stringBuilder.append(str).append("\n"); stringBuilder.append(str).append("\n");
} }
} else { } else {
log.warn("{}:{}", url, statusCode); // Never log gallery tokens, cookies or an arbitrary upstream error body.
throw new IOException("上游 HTTP " + statusCode + " (" + method.toUpperCase(Locale.ROOT)
+ " " + URI.create(url).getHost() + ")");
} }
return stringBuilder.toString(); return stringBuilder.toString();
} }
@@ -231,6 +231,22 @@ class MyBatisNativeConfigurationTest {
"同名 XML 映射文件应登记"); "同名 XML 映射文件应登记");
} }
@Test
void subscriptionEventMapperRegistersEventAndBindingReflection() {
var processor = new MyBatisNativeConfiguration.MyBatisBeanFactoryInitializationAotProcessor();
var factory = new org.springframework.beans.factory.support.DefaultListableBeanFactory();
var definition = new RootBeanDefinition(MapperFactoryBean.class);
definition.getPropertyValues().add("mapperInterface", com.lion.lionwebsite.Dao.normal.SubscriptionEventMapper.class);
factory.registerBeanDefinition("subscriptionEventMapper", definition);
var hints = new RuntimeHints();
var contribution = processor.processAheadOfTime(factory);
assertNotNull(contribution);
contribution.applyTo(new StubGenerationContext(hints), new NoOpInitializationCode());
assertTrue(RuntimeHintsPredicates.reflection().onType(com.lion.lionwebsite.Domain.SubscriptionEvent.class).test(hints));
assertTrue(RuntimeHintsPredicates.reflection().onType(com.lion.lionwebsite.Domain.SubscriptionEventBinding.class).test(hints));
assertTrue(RuntimeHintsPredicates.proxies().forInterfaces(com.lion.lionwebsite.Dao.normal.SubscriptionEventMapper.class).test(hints));
}
/** applyTo 只用 runtimeHints,初始化代码侧给一个记录型空实现即可。 */ /** applyTo 只用 runtimeHints,初始化代码侧给一个记录型空实现即可。 */
private static final class NoOpInitializationCode private static final class NoOpInitializationCode
implements org.springframework.beans.factory.aot.BeanFactoryInitializationCode { implements org.springframework.beans.factory.aot.BeanFactoryInitializationCode {
@@ -0,0 +1,340 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Controller.SubscriptionEventController;
import com.lion.lionwebsite.Dao.normal.SubMapper;
import com.lion.lionwebsite.Dao.normal.SubscriptionEventMapper;
import com.lion.lionwebsite.Dao.normal.UserMapper;
import com.lion.lionwebsite.Domain.SubscriptionEvent;
import com.lion.lionwebsite.Util.CustomUtil;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.mock.web.MockHttpServletResponse;
import java.nio.file.Files;
import static org.mockito.Mockito.*;
import org.springframework.transaction.support.TransactionTemplate;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.*;
import static org.junit.jupiter.api.Assertions.*;
/** Real SQLite/MyBatis/Spring transactions: these assertions cover committed database state. */
class SubscriptionEventIntegrationTest {
@TempDir Path directory;
private JdbcTemplate db;
private SubscriptionEventService service;
private SubMapper subscriptions;
private static final long NOW = 1_792_555_200_000L;
private static final String SECRET = "integration-test-only-secret";
private static final String KEY = "example-key";
@BeforeEach void setUp() throws Exception {
var source = new DriverManagerDataSource("jdbc:sqlite:" + directory.resolve("events.db") + "?busy_timeout=10000");
db = new JdbcTemplate(source);
db.execute("create table User(id integer primary key,username text,AuthCode text,isEnable integer,lastAccessTime text)");
db.execute("create table subscription_account(id integer primary key,name text,enabled integer,filter_high_multiplier integer)");
db.execute("create table sub_bind(key text primary key,user text,user_id integer,subscription_account_id integer)");
db.execute("create table sub_update_record(id integer primary key autoincrement,user text,ip text,UA text,time integer,location text,user_id integer)");
db.execute("create table subscription_event_receipt(event_id text primary key,key_hash text,client text,occurred_at integer,source text,ingested_at integer,fingerprint text,outcome text)");
db.execute("create table subscription_event_binding(key_hash text primary key,user_id integer,user_name text,first_seen_at integer,last_seen_at integer)");
db.update("insert into User(id,username,isEnable) values(5,'alice',1)");
db.update("insert into subscription_account values(1,'account',1,1)");
db.update("insert into sub_bind values(?,'alice',5,1)", KEY);
var bean = new SqlSessionFactoryBean();
bean.setDataSource(source);
var factory = bean.getObject();
factory.getConfiguration().addMapper(SubMapper.class);
factory.getConfiguration().addMapper(UserMapper.class);
factory.getConfiguration().addMapper(SubscriptionEventMapper.class);
var session = new SqlSessionTemplate(factory);
subscriptions = session.getMapper(SubMapper.class);
service = new SubscriptionEventService(session.getMapper(SubscriptionEventMapper.class), subscriptions,
session.getMapper(UserMapper.class), new TransactionTemplate(new DataSourceTransactionManager(source)),
new SubscriptionStateCoordinator());
service.syncSecret = SECRET;
service.clock = Clock.fixed(Instant.ofEpochMilli(NOW), ZoneOffset.UTC);
service.close(); // Reject optional geolocation work; never make network calls from these tests.
}
@Test void hmacRejectsInvalidExpiredModifiedAndOversizedInputs() throws Exception {
byte[] body = batch(event(NOW - 60_000));
String timestamp = Long.toString(NOW / 1000);
assertTrue(service.verify(body, timestamp, sign(body, timestamp)));
assertFalse(service.verify(body, timestamp, "0".repeat(64)));
assertFalse(service.verify("changed".getBytes(StandardCharsets.UTF_8), timestamp, sign(body, timestamp)));
String expired = Long.toString(NOW / 1000 - 301);
assertFalse(service.verify(body, expired, sign(body, expired)));
assertFalse(service.verify(body, "9".repeat(1024), "0".repeat(64)));
assertFalse(service.verify(new byte[SubscriptionEventService.MAX_BODY + 1], timestamp, "0".repeat(64)));
}
@Test void primaryAndStandbySameEventProduceOneRecordAndPreserveOriginalMetadata() {
SubscriptionEvent event = event(NOW - 86_400_000);
service.recordPrimary(event, subscriptions.selectSubBind(KEY));
assertEquals(List.of(event.getEventId()), service.ingest(List.of(event)).get("acknowledged"));
assertEquals(1, count("sub_update_record"));
assertEquals(1, count("subscription_event_receipt"));
assertEquals("primary", db.queryForObject("select source from subscription_event_receipt", String.class));
Map<String, Object> row = db.queryForMap("select * from sub_update_record");
assertEquals(event.getOccurredAt(), ((Number) row.get("time")).longValue());
assertEquals("2001:db8::7", row.get("ip"));
assertEquals("client/1.2", row.get("UA"));
assertEquals(5, ((Number) row.get("user_id")).intValue());
}
@Test void concurrentPrimaryAndReplayTransactionsReserveTheWriterBeforeReading() throws Exception {
var start = new java.util.concurrent.CountDownLatch(1);
var executor = java.util.concurrent.Executors.newFixedThreadPool(8);
var binding = subscriptions.selectSubBind(KEY);
try {
List<java.util.concurrent.Future<?>> futures = new ArrayList<>();
for (int i = 0; i < 32; i++) {
SubscriptionEvent event = event(NOW - i);
boolean primary = i % 2 == 0;
futures.add(executor.submit(() -> {
try { start.await(); }
catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); throw new RuntimeException(interrupted); }
if (primary) service.recordPrimary(event, binding);
else assertEquals(List.of(event.getEventId()), service.ingest(List.of(event)).get("acknowledged"));
}));
}
start.countDown();
for (var future : futures) future.get(30, java.util.concurrent.TimeUnit.SECONDS);
assertEquals(32, count("subscription_event_receipt"));
assertEquals(10, count("sub_update_record"));
} finally { executor.shutdownNow(); }
}
@Test void keyRotationAndRenameRetainStableIdentity() {
service.rememberBindings(subscriptions.selectAllSubBind());
db.update("update sub_bind set key='rotated-key'");
db.update("update User set username='renamed' where id=5");
SubscriptionEvent old = event(NOW - 30_000);
assertEquals(List.of(old.getEventId()), service.ingest(List.of(old)).get("acknowledged"));
assertEquals("renamed", db.queryForObject("select user from sub_update_record", String.class));
assertEquals(5, db.queryForObject("select user_id from sub_update_record", Integer.class));
assertEquals(SubscriptionEventService.sha256(KEY.getBytes(StandardCharsets.UTF_8)),
db.queryForObject("select key_hash from subscription_event_binding", String.class));
}
@Test void legacyLabelWithoutUserHasPrimaryStandbyReplayAndRotatedKeyHistory() {
db.update("update sub_bind set user='legacy-label',user_id=NULL");
db.update("delete from User");
SubscriptionEvent primary = event(NOW - 10_000);
service.recordPrimary(primary, subscriptions.selectSubBind(KEY));
assertEquals(List.of(primary.getEventId()), service.ingest(List.of(primary)).get("acknowledged"));
assertEquals(1, count("sub_update_record"));
assertEquals("legacy-label", db.queryForObject("select user from sub_update_record", String.class));
assertNull(db.queryForObject("select user_id from sub_update_record", Integer.class));
service.rememberBindings(subscriptions.selectAllSubBind());
db.update("update sub_bind set key='rotated-legacy-key'");
SubscriptionEvent offline = event(NOW - 5_000);
assertEquals(List.of(offline.getEventId()), service.ingest(List.of(offline)).get("acknowledged"));
assertEquals(2, count("sub_update_record"));
assertEquals(0, count("User"));
assertEquals(2, db.queryForObject("select count(*) from sub_update_record where user='legacy-label' and user_id is null", Integer.class));
for (int i = 0; i < 12; i++) service.ingest(List.of(event(NOW - i)));
assertEquals(10, count("sub_update_record"));
assertEquals(14, count("subscription_event_receipt"));
service.ingest(List.of(primary));
assertEquals(10, count("sub_update_record"));
}
@Test void legacyLabelDoesNotAdoptNewSameNamedUser() {
db.update("update sub_bind set user='legacy-label',user_id=NULL");
service.rememberBindings(subscriptions.selectAllSubBind());
db.update("delete from sub_bind");
db.update("insert into User(id,username,isEnable) values(8,'legacy-label',1)");
SubscriptionEvent event = event(NOW);
assertEquals(List.of(event.getEventId()), service.ingest(List.of(event)).get("acknowledged"));
assertNull(db.queryForObject("select user_id from sub_update_record", Integer.class));
}
@Test void unknownBindingIsNotAcknowledgedAndDeletedUserIsTerminal() {
SubscriptionEvent unknown = event(NOW);
unknown.setKeyHash("0".repeat(64));
assertEquals(List.of(), service.ingest(List.of(unknown)).get("acknowledged"));
assertEquals(0, count("subscription_event_receipt"));
service.rememberBindings(subscriptions.selectAllSubBind());
db.update("delete from sub_bind");
db.update("delete from User");
SubscriptionEvent deleted = event(NOW - 1_000);
assertEquals(List.of(deleted.getEventId()), service.ingest(List.of(deleted)).get("acknowledged"));
assertEquals("user_deleted", db.queryForObject("select outcome from subscription_event_receipt", String.class));
assertEquals(0, count("sub_update_record"));
}
@Test void transactionFailureRollsBackWholeBatchAndControllerNeverAcknowledges() throws Exception {
db.execute("create trigger fail_receipt before insert on subscription_event_receipt begin select raise(ABORT,'test rollback'); end");
SubscriptionEvent event = event(NOW);
assertThrows(RuntimeException.class, () -> service.ingest(List.of(event)));
assertEquals(0, count("sub_update_record"));
assertEquals(0, count("subscription_event_receipt"));
var controller = new SubscriptionEventController(service);
assertThrows(RuntimeException.class, () -> controller.ingest(signedPost(batch(event))));
assertEquals(0, count("sub_update_record"));
}
@Test void pruningUsesEventTimeAndReceiptsSurviveVisibleRecordDeletion() {
SubscriptionEvent oldest = event(NOW - 100_000);
service.ingest(List.of(oldest));
List<SubscriptionEvent> newer = new ArrayList<>();
for (int i = 0; i < 12; i++) newer.add(event(NOW - i * 1_000));
service.ingest(newer);
assertEquals(10, count("sub_update_record"));
assertEquals(13, count("subscription_event_receipt"));
assertEquals(NOW - 9_000, db.queryForObject("select min(time) from sub_update_record", Long.class));
assertEquals(List.of(oldest.getEventId()), service.ingest(List.of(oldest)).get("acknowledged"));
assertEquals(10, count("sub_update_record"));
db.update("delete from sub_update_record");
service.recordPrimary(newer.getFirst(), subscriptions.selectSubBind(KEY));
assertEquals(0, count("sub_update_record"));
assertEquals(13, count("subscription_event_receipt"));
}
@Test void signedGetVerifiesKeyClientUaAndUsesAuthenticatedIp() throws Exception {
SubscriptionEvent event = event(NOW);
byte[] body = CustomUtil.objectMapper.writeValueAsBytes(asMap(event));
var request = new MockHttpServletRequest("GET", "/sub/cat/" + KEY);
request.setRemoteAddr("127.0.0.1");
request.addHeader("X-Forwarded-For", "198.51.100.123");
request.addHeader("User-Agent", event.getUserAgent());
request.addHeader("X-Subscription-Event", Base64.getUrlEncoder().withoutPadding().encodeToString(body));
String timestamp = Long.toString(NOW / 1000);
request.addHeader("X-Subscription-Timestamp", timestamp);
request.addHeader("X-Subscription-Signature", sign(body, timestamp));
SubscriptionEvent parsed = service.requestEvent(request, KEY, "cat");
assertNotNull(parsed);
assertEquals("2001:db8::7", parsed.getClientIp());
assertNull(service.requestEvent(request, "wrong-key", "cat"));
assertNull(service.requestEvent(request, KEY, "v2"));
request.removeHeader("User-Agent");
request.addHeader("User-Agent", "different-client");
assertNull(service.requestEvent(request, KEY, "cat"));
}
@Test void signedHeadAndUnavailableCacheDoNotWriteAndGetReplayWritesOnce() throws Exception {
var refresh = mock(SubscriptionRefreshService.class);
var sub = new SubService(subscriptions, mock(UserMapper.class), refresh, mock(RemoteService.class),
new SubscriptionStateCoordinator());
sub.subscriptionEvents = service;
sub.locationExecutor = runnable -> { };
Path cache = directory.resolve("subscription.txt");
when(refresh.cachedPath(1, "cat")).thenReturn(cache);
SubscriptionEvent event = event(NOW);
var unavailable = new MockHttpServletResponse();
sub.updateSub(unavailable, signedGet(event, "GET"), "cat", KEY);
assertEquals(503, unavailable.getStatus());
assertEquals(0, count("sub_update_record"));
Files.writeString(cache, "subscription-content");
var head = new MockHttpServletResponse();
sub.updateSub(head, signedGet(event, "HEAD"), "cat", KEY);
assertEquals(200, head.getStatus());
assertEquals(0, head.getContentAsByteArray().length);
assertEquals(0, count("sub_update_record"));
assertEquals(0, count("subscription_event_receipt"));
for (int i = 0; i < 2; i++) {
var get = new MockHttpServletResponse();
sub.updateSub(get, signedGet(event, "GET"), "cat", KEY);
assertEquals(200, get.getStatus());
assertEquals("subscription-content", get.getContentAsString());
}
assertEquals(1, count("sub_update_record"));
assertEquals(1, count("subscription_event_receipt"));
assertEquals("2001:db8::7", db.queryForObject("select ip from sub_update_record", String.class));
sub.shutdownLocationResolver();
}
@Test void invalidSignedGetStillUsesOrdinaryLogging() throws Exception {
var refresh = mock(SubscriptionRefreshService.class);
var sub = new SubService(subscriptions, mock(UserMapper.class), refresh, mock(RemoteService.class),
new SubscriptionStateCoordinator());
sub.subscriptionEvents = service;
sub.locationExecutor = runnable -> { };
Path cache = directory.resolve("subscription.txt");
Files.writeString(cache, "subscription-content");
when(refresh.cachedPath(1, "cat")).thenReturn(cache);
var request = signedGet(event(NOW), "GET");
request.removeHeader("X-Subscription-Signature");
request.addHeader("X-Subscription-Signature", "0".repeat(64));
request.setRemoteAddr("198.51.100.10");
sub.updateSub(new MockHttpServletResponse(), request, "cat", KEY);
assertEquals(1, count("sub_update_record"));
assertEquals(0, count("subscription_event_receipt"));
assertEquals("198.51.100.10", db.queryForObject("select ip from sub_update_record", String.class));
sub.shutdownLocationResolver();
}
@Test void invalidInputAndAuthenticationReturnNoAcknowledgements() throws Exception {
var controller = new SubscriptionEventController(service);
var unauthenticated = new MockHttpServletRequest("POST", "/internal/subscription-events");
unauthenticated.setContent(batch(event(NOW)));
assertEquals(403, controller.ingest(unauthenticated).getStatusCode().value());
assertEquals(400, controller.ingest(signedPost("{}".getBytes(StandardCharsets.UTF_8))).getStatusCode().value());
assertEquals(400, controller.ingest(signedPost(new byte[SubscriptionEventService.MAX_BODY + 1])).getStatusCode().value());
SubscriptionEvent future = event(NOW + 300_001);
assertThrows(IllegalArgumentException.class, () -> service.parseBatch(batch(future)));
SubscriptionEvent tooLong = event(NOW);
tooLong.setUserAgent("a".repeat(2049));
assertThrows(IllegalArgumentException.class, () -> service.parseBatch(batch(tooLong)));
assertEquals(0, count("subscription_event_receipt"));
}
@Test void conflictingIdCannotAcknowledgeOrWriteChangedEvent() {
SubscriptionEvent original = event(NOW);
service.ingest(List.of(original));
original.setClientIp("192.0.2.7");
assertThrows(IllegalArgumentException.class, () -> service.ingest(List.of(original)));
assertEquals(1, count("sub_update_record"));
}
private int count(String table) { return db.queryForObject("select count(*) from " + table, Integer.class); }
private SubscriptionEvent event(long time) {
return new SubscriptionEvent(UUID.randomUUID().toString(), SubscriptionEventService.sha256(KEY.getBytes(StandardCharsets.UTF_8)),
"cat", "2001:db8::7", "client/1.2", time);
}
private static Map<String, Object> asMap(SubscriptionEvent event) {
return Map.of("eventId", event.getEventId(), "keyHash", event.getKeyHash(), "client", event.getClient(),
"clientIp", event.getClientIp(), "userAgent", event.getUserAgent(), "occurredAt", event.getOccurredAt());
}
private static byte[] batch(SubscriptionEvent event) {
return CustomUtil.objectMapper.writeValueAsBytes(Map.of("events", List.of(asMap(event))));
}
private static String sign(byte[] body, String timestamp) throws Exception {
var mac = Mac.getInstance("HmacSHA256");
mac.init(new SecretKeySpec(SECRET.getBytes(StandardCharsets.UTF_8), "HmacSHA256"));
return HexFormat.of().formatHex(mac.doFinal(("v1\n" + timestamp + "\n" + SubscriptionEventService.sha256(body)).getBytes(StandardCharsets.UTF_8)));
}
private MockHttpServletRequest signedGet(SubscriptionEvent event, String method) throws Exception {
byte[] body = CustomUtil.objectMapper.writeValueAsBytes(asMap(event));
var request = new MockHttpServletRequest(method, "/sub/cat/" + KEY);
request.setRemoteAddr("127.0.0.1");
request.addHeader("User-Agent", event.getUserAgent());
request.addHeader("X-Subscription-Event", Base64.getUrlEncoder().withoutPadding().encodeToString(body));
String timestamp = Long.toString(NOW / 1000);
request.addHeader("X-Subscription-Timestamp", timestamp);
request.addHeader("X-Subscription-Signature", sign(body, timestamp));
return request;
}
private MockHttpServletRequest signedPost(byte[] body) throws Exception {
var request = new MockHttpServletRequest("POST", "/internal/subscription-events");
request.setContent(body);
String timestamp = Long.toString(NOW / 1000);
request.addHeader("X-Subscription-Timestamp", timestamp);
request.addHeader("X-Subscription-Signature", sign(body, timestamp));
return request;
}
}
@@ -0,0 +1,113 @@
package com.lion.lionwebsite.Util;
import com.lion.lionwebsite.Domain.Gallery;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.*;
/** Exercise actual HTTP bytes against loopback, without contacting or queuing upstream. */
class GallerySubmissionHttpTest {
private HttpServer server;
private String endpoint;
private int responseStatus = 200;
private String responseBody = "<div id='db'><p>Queued</p><p><strong>#12345</strong></p></div>";
private final AtomicInteger requestCount = new AtomicInteger();
private final AtomicReference<Received> received = new AtomicReference<>();
record Received(String method, String origin, String referer, String contentType, String body) { }
@BeforeEach
void startServer() throws IOException {
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/submit", exchange -> {
requestCount.incrementAndGet();
var headers = exchange.getRequestHeaders();
received.set(new Received(exchange.getRequestMethod(), headers.getFirst("Origin"),
headers.getFirst("Referer"), headers.getFirst("Content-Type"),
new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)));
byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(responseStatus, bytes.length);
try (var output = exchange.getResponseBody()) { output.write(bytes); }
});
server.start();
endpoint = "http://127.0.0.1:" + server.getAddress().getPort() + "/submit";
}
@AfterEach
void stopServer() { server.stop(0); }
private PreparedGallery prepared(String resolution) {
Gallery gallery = new Gallery();
gallery.setGid(1234567);
var resolutions = new LinkedHashMap<String, String>();
resolutions.put(resolution, "12.5MiB");
gallery.setAvailableResolution(resolutions);
return new PreparedGallery(gallery, endpoint,
GalleryUtil.verifyLink("https://exhentai.org/g/1234567/0123456789/"));
}
@Test
void originalSubmissionMatchesBrowserFormAndHeaders() throws Exception {
Gallery gallery = GalleryUtil.submit(prepared("Original"), "Original");
assertEquals("已提交", gallery.getStatus());
assertEquals(1, requestCount.get());
Received request = received.get();
assertEquals("POST", request.method());
assertEquals("https://exhentai.org", request.origin());
assertEquals(endpoint, request.referer());
assertTrue(request.contentType().startsWith("application/x-www-form-urlencoded"));
assertEquals("hathdl_xres=org", request.body());
}
@Test
void resampledSubmissionSendsNumericWidth() throws Exception {
GalleryUtil.submit(prepared("1280x"), "1280x");
assertEquals("hathdl_xres=1280", received.get().body());
}
@ParameterizedTest
@ValueSource(ints = {403, 429, 503})
void rejectedSubmissionReportsStatusWithoutSecretsOrAutomaticReplay(int status) {
responseStatus = status;
responseBody = "private upstream response";
PreparedGallery initial = prepared("Original");
PreparedGallery withToken = new PreparedGallery(initial.gallery(),
endpoint + "?token=private-gallery-token", initial.origin());
IOException error = assertThrows(IOException.class,
() -> GalleryUtil.submit(withToken, "Original"));
assertTrue(error.getMessage().contains("HTTP " + status));
assertFalse(error.getMessage().contains("private-gallery-token"));
assertFalse(error.getMessage().contains(responseBody));
assertEquals(1, requestCount.get(), "A rejected download POST must never be replayed");
}
@Test
void jsonImageApiRetainsItsEncodingAndDoesNotMutateCaller() throws Exception {
HashMap<String, String> body = new HashMap<>();
body.put("payload", "json");
body.put("method", "imagedispatch");
GalleryUtil.requests(endpoint, "post", null, body);
assertTrue(received.get().contentType().startsWith("application/json"));
assertEquals("imagedispatch", CustomUtil.objectMapper.readTree(received.get().body()).get("method").asText());
assertFalse(received.get().body().contains("payload"));
assertEquals("json", body.get("payload"));
}
}
@@ -41,7 +41,15 @@ class GalleryUtilTest {
void verifyLinkAcceptsWellFormedGalleryUrl() { void verifyLinkAcceptsWellFormedGalleryUrl() {
String url = "https://exhentai.org/g/1234567/0123456789ab/"; String url = "https://exhentai.org/g/1234567/0123456789ab/";
assertNotNull(GalleryUtil.verifyLink(url)); assertNotNull(GalleryUtil.verifyLink(url));
assertEquals("", GalleryUtil.verifyLink(url), "通过校验时返回空串作为 origin"); assertEquals("https://exhentai.org", GalleryUtil.verifyLink(url), "返回真实 origin,不能发送空 Origin");
}
@Test
void verifyLinkRejectsLookalikeHostsAndMalformedUrls() {
assertNull(GalleryUtil.verifyLink("https://exhentai.org.example.com/g/1234567/0123456789ab/"));
assertNull(GalleryUtil.verifyLink("https://example.com/g/1234567/exhentai0123456789/"));
assertNull(GalleryUtil.verifyLink("https://user@exhentai.org/g/1234567/0123456789ab/"));
assertNull(GalleryUtil.verifyLink("https://exhentai.org/g/1234567/invalid key/"));
} }
// ---------- parseGid ---------- // ---------- parseGid ----------