339 lines
17 KiB
Java
339 lines
17 KiB
Java
package lion.Service;
|
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
|
import lion.CustomUtil;
|
|
import lion.Externel.BackupSubServer;
|
|
import lion.Http.HttpRequestHandler;
|
|
import lion.Http.HttpRequestModel;
|
|
import lion.Http.HttpResponseModel;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
import javax.crypto.Mac;
|
|
import javax.crypto.spec.SecretKeySpec;
|
|
import java.io.IOException;
|
|
import java.net.InetAddress;
|
|
import java.net.URI;
|
|
import java.net.URISyntaxException;
|
|
import java.net.http.HttpClient;
|
|
import java.net.http.HttpRequest;
|
|
import java.net.http.HttpResponse;
|
|
import java.nio.ByteBuffer;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Path;
|
|
import java.security.MessageDigest;
|
|
import java.time.Duration;
|
|
import java.util.Base64;
|
|
import java.util.HashSet;
|
|
import java.util.HexFormat;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.UUID;
|
|
import java.util.concurrent.CompletableFuture;
|
|
import java.util.concurrent.CompletionStage;
|
|
import java.util.concurrent.Flow;
|
|
import java.util.concurrent.ScheduledExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
/** storageNode 内置的主站优先订阅入口及离线访问记录回传器。 */
|
|
@Slf4j
|
|
public final class SubscriptionGateway implements HttpRequestHandler, AutoCloseable {
|
|
private static final int MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
|
private static final int MAX_BATCH_BYTES = 256 * 1024;
|
|
private final HttpRequestHandler local;
|
|
private final URI origin;
|
|
private final byte[] secret;
|
|
private final SubscriptionEventJournal journal;
|
|
private final HttpClient client;
|
|
private final Duration timeout;
|
|
private final long cooldownMillis;
|
|
private final ScheduledExecutorService uploader;
|
|
private final Object uploadLock = new Object();
|
|
private long openUntil;
|
|
private long generation;
|
|
private boolean halfOpen;
|
|
private volatile long lastUploadAt;
|
|
private volatile String lastUploadState = "idle";
|
|
private volatile boolean closed;
|
|
|
|
public SubscriptionGateway(SubscriptionSnapshotStore store, String primaryUrl, String secret, Path eventDir)
|
|
throws IOException {
|
|
this(store, primaryUrl, secret, eventDir, Duration.ofSeconds(3), 10_000, true);
|
|
}
|
|
|
|
/** 可配置时间仅供真实 HTTP 测试；生产固定 1s 连接/3s 总时限/10s 熔断。 */
|
|
public SubscriptionGateway(SubscriptionSnapshotStore store, String primaryUrl, String secret, Path eventDir,
|
|
Duration timeout, long cooldownMillis, boolean backgroundUpload) throws IOException {
|
|
this.local = BackupSubServer.handler(store);
|
|
this.origin = validateOrigin(primaryUrl);
|
|
if (secret == null || secret.isBlank()) throw new IllegalArgumentException("Subscription gateway secret required");
|
|
this.secret = secret.getBytes(StandardCharsets.UTF_8);
|
|
this.journal = new SubscriptionEventJournal(eventDir);
|
|
this.timeout = timeout;
|
|
this.cooldownMillis = cooldownMillis;
|
|
this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(1))
|
|
.followRedirects(HttpClient.Redirect.NEVER).build();
|
|
this.uploader = Executors.newSingleThreadScheduledExecutor(r -> {
|
|
Thread thread = new Thread(r, "subscription-events-upload");
|
|
thread.setDaemon(true);
|
|
return thread;
|
|
});
|
|
if (backgroundUpload) uploader.scheduleWithFixedDelay(this::uploadSafely, 10, 10, TimeUnit.SECONDS);
|
|
}
|
|
|
|
private static URI validateOrigin(String value) {
|
|
URI uri = URI.create(value);
|
|
if (!("http".equals(uri.getScheme()) || "https".equals(uri.getScheme()))
|
|
|| uri.getHost() == null || uri.getUserInfo() != null || uri.getQuery() != null
|
|
|| uri.getFragment() != null || !(uri.getPath().isEmpty() || "/".equals(uri.getPath())))
|
|
throw new IllegalArgumentException("SubscriptionPrimaryUrl must be a fixed HTTP(S) origin");
|
|
return uri;
|
|
}
|
|
|
|
@Override
|
|
public HttpResponseModel handle(HttpRequestModel request) {
|
|
String path = request.decodedPath();
|
|
if ("/health/subscription-gateway".equals(path)) {
|
|
if (!HttpRequestModel.GET.equals(request.method()) && !request.isHead()) return HttpResponseModel.status(405);
|
|
try {
|
|
return HttpResponseModel.bytes(200, CustomUtil.objectMapper.writeValueAsBytes(status()),
|
|
"application/json; charset=utf-8", null, Map.of());
|
|
} catch (IOException failure) {
|
|
return HttpResponseModel.status(500);
|
|
}
|
|
}
|
|
if (!HttpRequestModel.GET.equals(request.method()) && !request.isHead()) return local.handle(request);
|
|
String[] parts = path == null ? new String[0] : path.split("/", -1);
|
|
if (parts.length != 4 || !parts[0].isEmpty() || !"sub".equals(parts[1])
|
|
|| !("v2".equals(parts[2]) || "cat".equals(parts[2]))
|
|
|| parts[3].length() < 6 || parts[3].length() > 512)
|
|
return local.handle(request);
|
|
// 内部路径不能成为 URI 目录跳转或带控制字符的请求。
|
|
if (parts[3].codePoints().anyMatch(c -> Character.isISOControl(c))) return HttpResponseModel.status(404);
|
|
ObjectNode event;
|
|
try {
|
|
event = event(request, parts[2], parts[3]);
|
|
} catch (IOException failure) {
|
|
return HttpResponseModel.status(503);
|
|
}
|
|
Attempt attempt = acquire();
|
|
if (attempt != null) {
|
|
try {
|
|
HttpResponseModel response = primary(request, parts, event);
|
|
if (response.status() < 500 && !(response.status() >= 300 && response.status() < 400)) {
|
|
recovered(attempt);
|
|
return response;
|
|
}
|
|
failed(attempt);
|
|
} catch (Exception failure) {
|
|
if (failure instanceof InterruptedException) Thread.currentThread().interrupt();
|
|
failed(attempt);
|
|
// 不记录 URL/公开 key/密钥，失败原因只暴露类别。
|
|
log.debug("订阅主站请求失败: {}", failure.getClass().getSimpleName());
|
|
}
|
|
}
|
|
HttpResponseModel fallback = local.handle(request);
|
|
if (!request.isHead() && (fallback.status() == 200 || fallback.status() == 206)) {
|
|
try {
|
|
journal.append(event);
|
|
} catch (IOException failure) {
|
|
log.error("订阅离线记录无法持久化，拒绝本次更新: {}", failure.getClass().getSimpleName());
|
|
return HttpResponseModel.text(503, "text/plain", "Subscription access journal unavailable");
|
|
}
|
|
}
|
|
return fallback;
|
|
}
|
|
|
|
private ObjectNode event(HttpRequestModel request, String format, String key) throws IOException {
|
|
ObjectNode event = CustomUtil.objectMapper.createObjectNode();
|
|
event.put("eventId", UUID.randomUUID().toString());
|
|
event.put("keyHash", sha256(key.getBytes(StandardCharsets.UTF_8)));
|
|
event.put("client", format);
|
|
event.put("clientIp", clientIp(request));
|
|
String rawAgent = request.header("User-Agent");
|
|
if (rawAgent == null) rawAgent = "";
|
|
// HTTP/1 header值使用Latin-1。正常原始UA保持不变；移除控制字符，
|
|
// 非Latin-1字符替换为?，避免非法header导致每次请求都只能fallback。
|
|
StringBuilder normalized = new StringBuilder();
|
|
rawAgent.codePoints().filter(c -> c >= 32 && c != 127)
|
|
.limit(2048).forEach(c -> normalized.appendCodePoint(c <= 255 ? c : '?'));
|
|
String agent = normalized.toString();
|
|
while (agent.getBytes(StandardCharsets.UTF_8).length > 2048)
|
|
agent = agent.substring(0, agent.length() - 1);
|
|
event.put("userAgent", agent);
|
|
event.put("occurredAt", System.currentTimeMillis());
|
|
// 签名事件头的 base64url 必须 <=4096。保留能装下的原始前缀，避免切断代理对。
|
|
while (Base64.getUrlEncoder().withoutPadding().encodeToString(
|
|
CustomUtil.objectMapper.writeValueAsBytes(event)).length() > 4096) {
|
|
agent = agent.substring(0, agent.offsetByCodePoints(agent.length(), -1));
|
|
event.put("userAgent", agent);
|
|
}
|
|
return event;
|
|
}
|
|
|
|
public static String clientIp(HttpRequestModel request) {
|
|
InetAddress remote = literalAddress(request.clientAddress());
|
|
if (remote != null && remote.isLoopbackAddress()) {
|
|
InetAddress forwarded = literalAddress(request.header("X-Real-IP"));
|
|
if (forwarded != null) return forwarded.getHostAddress();
|
|
}
|
|
return remote == null ? "0.0.0.0" : remote.getHostAddress();
|
|
}
|
|
|
|
private static InetAddress literalAddress(String value) {
|
|
if (value == null || value.length() > 45 || !value.matches("[0-9a-fA-F:.]+")) return null;
|
|
if (!value.contains(":")) {
|
|
if (!value.matches("[0-9]{1,3}(\\.[0-9]{1,3}){3}")) return null;
|
|
for (String part : value.split("\\.")) if (Integer.parseInt(part) > 255) return null;
|
|
}
|
|
try { return InetAddress.getByName(value); } catch (Exception ignored) { return null; }
|
|
}
|
|
|
|
private HttpResponseModel primary(HttpRequestModel request, String[] parts, ObjectNode event) throws Exception {
|
|
URI target = endpoint("/sub/" + parts[2] + "/" + parts[3]);
|
|
HttpRequest.Builder builder = HttpRequest.newBuilder(target).timeout(timeout)
|
|
.method(request.isHead() ? "HEAD" : "GET", HttpRequest.BodyPublishers.noBody())
|
|
.header("User-Agent", event.path("userAgent").asText());
|
|
if (request.rangeHeader() != null) builder.header("Range", request.rangeHeader());
|
|
// HEAD 也必须签名，使主站绕过普通访问记录分支；主站与节点均不记 HEAD。
|
|
byte[] json = CustomUtil.objectMapper.writeValueAsBytes(event);
|
|
builder.header("X-Subscription-Event", Base64.getUrlEncoder().withoutPadding().encodeToString(json));
|
|
sign(builder, json);
|
|
HttpResponse<byte[]> response = send(builder.build(), MAX_RESPONSE_BYTES);
|
|
Map<String, String> headers = new LinkedHashMap<>();
|
|
for (String name : List.of("Content-Range", "Accept-Ranges"))
|
|
response.headers().firstValue(name).ifPresent(value -> headers.put(name, value));
|
|
if (request.isHead()) response.headers().firstValue("Content-Length")
|
|
.ifPresent(value -> headers.put("Content-Length", value));
|
|
return HttpResponseModel.bytes(response.statusCode(), response.body(),
|
|
response.headers().firstValue("Content-Type").orElse("application/octet-stream"), null, headers);
|
|
}
|
|
|
|
/** URI 构造器百分号编码单一路径；从不拼接用户 origin/query 或跟随 redirect。 */
|
|
private URI endpoint(String path) throws URISyntaxException {
|
|
return new URI(origin.getScheme(), null, origin.getHost(), origin.getPort(), path, null, null);
|
|
}
|
|
|
|
private void sign(HttpRequest.Builder builder, byte[] payload) throws Exception {
|
|
String timestamp = Long.toString(System.currentTimeMillis() / 1000);
|
|
Mac mac = Mac.getInstance("HmacSHA256");
|
|
mac.init(new SecretKeySpec(secret, "HmacSHA256"));
|
|
String input = "v1\n" + timestamp + "\n" + sha256(payload);
|
|
builder.header("X-Subscription-Timestamp", timestamp)
|
|
.header("X-Subscription-Signature", HexFormat.of().formatHex(mac.doFinal(input.getBytes(StandardCharsets.UTF_8))));
|
|
}
|
|
|
|
private static String sha256(byte[] bytes) {
|
|
try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); }
|
|
catch (java.security.NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); }
|
|
}
|
|
|
|
/** sendAsync 的整次 future 包含正文消费，避免 ofInputStream 绕过读取时限。 */
|
|
private HttpResponse<byte[]> send(HttpRequest request, int maxBytes) throws Exception {
|
|
CompletableFuture<HttpResponse<byte[]>> future = client.sendAsync(request,
|
|
info -> new BoundedSubscriber(maxBytes));
|
|
try { return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); }
|
|
finally { if (!future.isDone()) future.cancel(true); }
|
|
}
|
|
|
|
private static final class BoundedSubscriber implements HttpResponse.BodySubscriber<byte[]> {
|
|
private final HttpResponse.BodySubscriber<byte[]> delegate = HttpResponse.BodySubscribers.ofByteArray();
|
|
private final int limit;
|
|
private int received;
|
|
private Flow.Subscription subscription;
|
|
BoundedSubscriber(int limit) { this.limit = limit; }
|
|
public CompletionStage<byte[]> getBody() { return delegate.getBody(); }
|
|
public void onSubscribe(Flow.Subscription value) { subscription = value; delegate.onSubscribe(value); }
|
|
public void onNext(List<ByteBuffer> buffers) {
|
|
long size = buffers.stream().mapToLong(ByteBuffer::remaining).sum();
|
|
if (size > limit - received) {
|
|
subscription.cancel();
|
|
delegate.onError(new IOException("Subscription HTTP response exceeds limit"));
|
|
} else { received += (int) size; delegate.onNext(buffers); }
|
|
}
|
|
public void onError(Throwable error) { delegate.onError(error); }
|
|
public void onComplete() { delegate.onComplete(); }
|
|
}
|
|
|
|
private record Attempt(long generation) { }
|
|
private synchronized Attempt acquire() {
|
|
if (openUntil == 0) return new Attempt(generation);
|
|
if (System.currentTimeMillis() < openUntil || halfOpen) return null;
|
|
halfOpen = true;
|
|
return new Attempt(generation);
|
|
}
|
|
private synchronized void failed(Attempt attempt) {
|
|
if (attempt.generation() != generation) return;
|
|
openUntil = System.currentTimeMillis() + cooldownMillis;
|
|
halfOpen = false;
|
|
generation++;
|
|
}
|
|
private synchronized void recovered(Attempt attempt) {
|
|
if (attempt.generation() != generation) return;
|
|
openUntil = 0;
|
|
halfOpen = false;
|
|
}
|
|
|
|
public Map<String, Object> status() {
|
|
String circuit;
|
|
synchronized (this) { circuit = openUntil == 0 ? "closed" : halfOpen ? "half-open" : "open"; }
|
|
return Map.of("pendingCount", journal.size(), "oldestAgeMillis", journal.oldestAgeMillis(),
|
|
"circuit", circuit, "lastUploadAt", lastUploadAt, "lastUploadState", lastUploadState);
|
|
}
|
|
|
|
private void uploadSafely() {
|
|
try { uploadOnce(); }
|
|
catch (Exception failure) {
|
|
lastUploadState = "failed";
|
|
log.debug("订阅访问记录回传失败: {}", failure.getClass().getSimpleName());
|
|
}
|
|
}
|
|
|
|
/** 每轮最多一批、100条、256KiB；所有未明确确认的原始 eventId 均留盘。 */
|
|
public void uploadOnce() throws Exception {
|
|
synchronized (uploadLock) {
|
|
if (closed) return;
|
|
List<SubscriptionEventJournal.Pending> candidates = journal.oldest(100);
|
|
if (candidates.isEmpty()) { lastUploadState = "idle"; return; }
|
|
ObjectNode body = CustomUtil.objectMapper.createObjectNode();
|
|
var array = body.putArray("events");
|
|
java.util.ArrayList<SubscriptionEventJournal.Pending> batch = new java.util.ArrayList<>();
|
|
int total = 13; // {"events":[]} + 逗号安全余量
|
|
for (var pending : candidates) {
|
|
if (total + pending.json().length + 1 > MAX_BATCH_BYTES) break;
|
|
array.add(CustomUtil.objectMapper.readTree(pending.json()));
|
|
batch.add(pending);
|
|
total += pending.json().length + 1;
|
|
}
|
|
if (batch.isEmpty()) throw new IOException("Subscription event exceeds upload batch limit");
|
|
byte[] json = CustomUtil.objectMapper.writeValueAsBytes(body);
|
|
HttpRequest.Builder builder = HttpRequest.newBuilder(endpoint("/internal/subscription-events"))
|
|
.timeout(timeout).header("Content-Type", "application/json")
|
|
.POST(HttpRequest.BodyPublishers.ofByteArray(json));
|
|
sign(builder, json);
|
|
HttpResponse<byte[]> response = send(builder.build(), 64 * 1024);
|
|
if (response.statusCode() != 200) throw new IOException("Subscription event upload rejected");
|
|
JsonNode result = CustomUtil.objectMapper.readTree(response.body());
|
|
if (result == null || !result.path("acknowledged").isArray()) throw new IOException("Invalid subscription event acknowledgement");
|
|
Set<String> ids = new HashSet<>();
|
|
for (JsonNode id : result.path("acknowledged")) {
|
|
if (!id.isTextual()) throw new IOException("Invalid subscription event acknowledgement");
|
|
ids.add(id.asText());
|
|
}
|
|
journal.acknowledge(batch, ids);
|
|
lastUploadAt = System.currentTimeMillis();
|
|
lastUploadState = "ok";
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void close() {
|
|
closed = true;
|
|
uploader.shutdownNow();
|
|
client.close();
|
|
}
|
|
}
|