lion/lionwebsite-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix upstream archive submissions to match browser form encoding

Browse Source
This commit is contained in:
root
2026-10-22 22:27:59 +08:00
parent a72d99ff0a
commit b9358d9817
3 changed files with 153 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java
+31 -13
View File
@@ -10,12 +10,13 @@ import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.entity.EntityBuilder;
import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -28,6 +29,8 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -40,7 +43,7 @@ public class GalleryUtil {
static String POST = "post";
static String GET = "get";
static String FORM_DATA = "formData";
static String FORM_URLENCODED = "formUrlEncoded";
static String JSON = "json";
@@ -58,6 +61,8 @@ public class GalleryUtil {
/** Reusable HTTP client —不要每次请求新建 */
private static final CloseableHttpClient httpClient = HttpClients.custom()
// An upstream download POST has no idempotency key; never replay it on 429/503.
.disableAutomaticRetries()
.setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(5_000))
.setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000))
@@ -104,8 +109,7 @@ public class GalleryUtil {
Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
HashMap<String, String> headers = new HashMap<>();
headers.put("origin", origin);
headers.put("referer", archiveLink);
headers.put("referer", url);
Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
@@ -132,7 +136,8 @@ public class GalleryUtil {
//提交下载请求
HashMap<String, String> body = new HashMap<>();
body.put("payload", FORM_DATA);
// The upstream form has no enctype: match the browser's default encoding.
body.put("payload", FORM_URLENCODED);
body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
String downloadPage = requests(download_link, POST, headers, body);
Document downloadDoc = Jsoup.parse(downloadPage);
@@ -292,9 +297,18 @@ public class GalleryUtil {
if(url.length() < 40)
return null;
if(!url.contains("/g/") || !url.contains("exhentai"))
try {
URI uri = URI.create(url);
String scheme = uri.getScheme();
if ((!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme))
|| !"exhentai.org".equalsIgnoreCase(uri.getHost())
|| uri.getRawUserInfo() != null || uri.getPort() != -1
|| uri.getPath() == null || !uri.getPath().matches("/g/[0-9]+/[A-Za-z0-9]+/?"))
return null;
return scheme.toLowerCase(Locale.ROOT) + "://exhentai.org";
} catch (IllegalArgumentException invalidUrl) {
return null;
return "";
}
}
/**
@@ -329,16 +343,18 @@ public class GalleryUtil {
httpPost.addHeader(header.getKey(), header.getValue());
if(body != null) {
// Keep the caller's form intact; payload is a local encoding selector.
body = new HashMap<>(body);
String payload;
if((payload = body.remove("payload")).equals(JSON)) {
EntityBuilder entityBuilder = EntityBuilder.create();
entityBuilder.setContentType(ContentType.APPLICATION_JSON);
entityBuilder.setText(objectMapper.writeValueAsString(body));
httpPost.setEntity(entityBuilder.build());
} else if(payload.equals(FORM_DATA)) {
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
body.forEach(multipartEntityBuilder::addTextBody);
httpPost.setEntity(multipartEntityBuilder.build());
} else if(payload.equals(FORM_URLENCODED)) {
var fields = body.entrySet().stream()
.map(entry -> new BasicNameValuePair(entry.getKey(), entry.getValue())).toList();
httpPost.setEntity(new UrlEncodedFormEntity(fields, StandardCharsets.UTF_8));
}
}
httpResponse = httpClient.execute(httpPost);
@@ -348,13 +364,15 @@ public class GalleryUtil {
int statusCode = httpResponse.getCode();
StringBuilder stringBuilder = new StringBuilder();
if (statusCode == 200 && responseEntity != null) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent()))) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8))) {
String str;
while ((str = reader.readLine()) != null)
stringBuilder.append(str).append("\n");
}
} else {
log.warn("{}:{}", url, statusCode);
// Never log gallery tokens, cookies or an arbitrary upstream error body.
throw new IOException("上游 HTTP " + statusCode + " (" + method.toUpperCase(Locale.ROOT)
+ " " + URI.create(url).getHost() + ")");
}
return stringBuilder.toString();
}
src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java
+113
View File
@@ -0,0 +1,113 @@
package com.lion.lionwebsite.Util;
import com.lion.lionwebsite.Domain.Gallery;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.*;
/** Exercise actual HTTP bytes against loopback, without contacting or queuing upstream. */
class GallerySubmissionHttpTest {
private HttpServer server;
private String endpoint;
private int responseStatus = 200;
private String responseBody = "<div id='db'><p>Queued</p><p><strong>#12345</strong></p></div>";
private final AtomicInteger requestCount = new AtomicInteger();
private final AtomicReference<Received> received = new AtomicReference<>();
record Received(String method, String origin, String referer, String contentType, String body) { }
@BeforeEach
void startServer() throws IOException {
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/submit", exchange -> {
requestCount.incrementAndGet();
var headers = exchange.getRequestHeaders();
received.set(new Received(exchange.getRequestMethod(), headers.getFirst("Origin"),
headers.getFirst("Referer"), headers.getFirst("Content-Type"),
new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)));
byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(responseStatus, bytes.length);
try (var output = exchange.getResponseBody()) { output.write(bytes); }
});
server.start();
endpoint = "http://127.0.0.1:" + server.getAddress().getPort() + "/submit";
}
@AfterEach
void stopServer() { server.stop(0); }
private PreparedGallery prepared(String resolution) {
Gallery gallery = new Gallery();
gallery.setGid(1234567);
var resolutions = new LinkedHashMap<String, String>();
resolutions.put(resolution, "12.5MiB");
gallery.setAvailableResolution(resolutions);
return new PreparedGallery(gallery, endpoint,
GalleryUtil.verifyLink("https://exhentai.org/g/1234567/0123456789/"));
}
@Test
void originalSubmissionMatchesBrowserFormAndHeaders() throws Exception {
Gallery gallery = GalleryUtil.submit(prepared("Original"), "Original");
assertEquals("已提交", gallery.getStatus());
assertEquals(1, requestCount.get());
Received request = received.get();
assertEquals("POST", request.method());
assertEquals("https://exhentai.org", request.origin());
assertEquals(endpoint, request.referer());
assertTrue(request.contentType().startsWith("application/x-www-form-urlencoded"));
assertEquals("hathdl_xres=org", request.body());
}
@Test
void resampledSubmissionSendsNumericWidth() throws Exception {
GalleryUtil.submit(prepared("1280x"), "1280x");
assertEquals("hathdl_xres=1280", received.get().body());
}
@ParameterizedTest
@ValueSource(ints = {403, 429, 503})
void rejectedSubmissionReportsStatusWithoutSecretsOrAutomaticReplay(int status) {
responseStatus = status;
responseBody = "private upstream response";
PreparedGallery initial = prepared("Original");
PreparedGallery withToken = new PreparedGallery(initial.gallery(),
endpoint + "?token=private-gallery-token", initial.origin());
IOException error = assertThrows(IOException.class,
() -> GalleryUtil.submit(withToken, "Original"));
assertTrue(error.getMessage().contains("HTTP " + status));
assertFalse(error.getMessage().contains("private-gallery-token"));
assertFalse(error.getMessage().contains(responseBody));
assertEquals(1, requestCount.get(), "A rejected download POST must never be replayed");
}
@Test
void jsonImageApiRetainsItsEncodingAndDoesNotMutateCaller() throws Exception {
HashMap<String, String> body = new HashMap<>();
body.put("payload", "json");
body.put("method", "imagedispatch");
GalleryUtil.requests(endpoint, "post", null, body);
assertTrue(received.get().contentType().startsWith("application/json"));
assertEquals("imagedispatch", CustomUtil.objectMapper.readTree(received.get().body()).get("method").asText());
assertFalse(received.get().body().contains("payload"));
assertEquals("json", body.get("payload"));
}
}
src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java
+9 -1
View File
@@ -41,7 +41,15 @@ class GalleryUtilTest {
void verifyLinkAcceptsWellFormedGalleryUrl() {
String url = "https://exhentai.org/g/1234567/0123456789ab/";
assertNotNull(GalleryUtil.verifyLink(url));
assertEquals("", GalleryUtil.verifyLink(url), "通过校验时返回空串作为 origin");
assertEquals("https://exhentai.org", GalleryUtil.verifyLink(url), "返回真实 origin，不能发送空 Origin");
}
@Test
void verifyLinkRejectsLookalikeHostsAndMalformedUrls() {
assertNull(GalleryUtil.verifyLink("https://exhentai.org.example.com/g/1234567/0123456789ab/"));
assertNull(GalleryUtil.verifyLink("https://example.com/g/1234567/exhentai0123456789/"));
assertNull(GalleryUtil.verifyLink("https://user@exhentai.org/g/1234567/0123456789ab/"));
assertNull(GalleryUtil.verifyLink("https://exhentai.org/g/1234567/invalid key/"));
}
// ---------- parseGid ----------