From b9358d9817bb00b93bb59eba4e0466445f70063d Mon Sep 17 00:00:00 2001 From: root Date: Thu, 22 Oct 2026 22:27:59 +0800 Subject: [PATCH] Fix upstream archive submissions to match browser form encoding --- .../lion/lionwebsite/Util/GalleryUtil.java | 44 +++++-- .../Util/GallerySubmissionHttpTest.java | 113 ++++++++++++++++++ .../lionwebsite/Util/GalleryUtilTest.java | 10 +- 3 files changed, 153 insertions(+), 14 deletions(-) create mode 100644 src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java diff --git a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java index 2ddd740..a62a309 100644 --- a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java +++ b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java @@ -10,12 +10,13 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.entity.EntityBuilder; -import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; +import org.apache.hc.client5.http.entity.UrlEncodedFormEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.message.BasicNameValuePair; import org.apache.hc.core5.util.Timeout; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -28,6 +29,8 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -40,7 +43,7 @@ public class GalleryUtil { static String POST = "post"; static String GET = "get"; - static String FORM_DATA = "formData"; + static String FORM_URLENCODED = "formUrlEncoded"; static String JSON = "json"; @@ -58,6 +61,8 @@ public class GalleryUtil { /** Reusable HTTP client —不要每次请求新建 */ private static final CloseableHttpClient httpClient = HttpClients.custom() + // An upstream download POST has no idempotency key; never replay it on 429/503. + .disableAutomaticRetries() .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout(Timeout.ofMilliseconds(5_000)) .setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000)) @@ -104,8 +109,7 @@ public class GalleryUtil { Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid); String archiveLink = GalleryPageParser.archiveLink(galleryDoc); HashMap headers = new HashMap<>(); - headers.put("origin", origin); - headers.put("referer", archiveLink); + headers.put("referer", url); Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink); String download_link = downloadDoc.select("#hathdl_form").attr("abs:action"); if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单"); @@ -132,7 +136,8 @@ public class GalleryUtil { //提交下载请求 HashMap body = new HashMap<>(); - body.put("payload", FORM_DATA); + // The upstream form has no enctype: match the browser's default encoding. + body.put("payload", FORM_URLENCODED); body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org")); String downloadPage = requests(download_link, POST, headers, body); Document downloadDoc = Jsoup.parse(downloadPage); @@ -292,9 +297,18 @@ public class GalleryUtil { if(url.length() < 40) return null; - if(!url.contains("/g/") || !url.contains("exhentai")) + try { + URI uri = URI.create(url); + String scheme = uri.getScheme(); + if ((!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme)) + || !"exhentai.org".equalsIgnoreCase(uri.getHost()) + || uri.getRawUserInfo() != null || uri.getPort() != -1 + || uri.getPath() == null || !uri.getPath().matches("/g/[0-9]+/[A-Za-z0-9]+/?")) + return null; + return scheme.toLowerCase(Locale.ROOT) + "://exhentai.org"; + } catch (IllegalArgumentException invalidUrl) { return null; - return ""; + } } /** @@ -329,16 +343,18 @@ public class GalleryUtil { httpPost.addHeader(header.getKey(), header.getValue()); if(body != null) { + // Keep the caller's form intact; payload is a local encoding selector. + body = new HashMap<>(body); String payload; if((payload = body.remove("payload")).equals(JSON)) { EntityBuilder entityBuilder = EntityBuilder.create(); entityBuilder.setContentType(ContentType.APPLICATION_JSON); entityBuilder.setText(objectMapper.writeValueAsString(body)); httpPost.setEntity(entityBuilder.build()); - } else if(payload.equals(FORM_DATA)) { - MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); - body.forEach(multipartEntityBuilder::addTextBody); - httpPost.setEntity(multipartEntityBuilder.build()); + } else if(payload.equals(FORM_URLENCODED)) { + var fields = body.entrySet().stream() + .map(entry -> new BasicNameValuePair(entry.getKey(), entry.getValue())).toList(); + httpPost.setEntity(new UrlEncodedFormEntity(fields, StandardCharsets.UTF_8)); } } httpResponse = httpClient.execute(httpPost); @@ -348,13 +364,15 @@ public class GalleryUtil { int statusCode = httpResponse.getCode(); StringBuilder stringBuilder = new StringBuilder(); if (statusCode == 200 && responseEntity != null) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent()))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8))) { String str; while ((str = reader.readLine()) != null) stringBuilder.append(str).append("\n"); } } else { - log.warn("{}:{}", url, statusCode); + // Never log gallery tokens, cookies or an arbitrary upstream error body. + throw new IOException("上游 HTTP " + statusCode + " (" + method.toUpperCase(Locale.ROOT) + + " " + URI.create(url).getHost() + ")"); } return stringBuilder.toString(); } diff --git a/src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java b/src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java new file mode 100644 index 0000000..b9654f4 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java @@ -0,0 +1,113 @@ +package com.lion.lionwebsite.Util; + +import com.lion.lionwebsite.Domain.Gallery; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +/** Exercise actual HTTP bytes against loopback, without contacting or queuing upstream. */ +class GallerySubmissionHttpTest { + private HttpServer server; + private String endpoint; + private int responseStatus = 200; + private String responseBody = "

Queued

#12345

"; + private final AtomicInteger requestCount = new AtomicInteger(); + private final AtomicReference received = new AtomicReference<>(); + + record Received(String method, String origin, String referer, String contentType, String body) { } + + @BeforeEach + void startServer() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/submit", exchange -> { + requestCount.incrementAndGet(); + var headers = exchange.getRequestHeaders(); + received.set(new Received(exchange.getRequestMethod(), headers.getFirst("Origin"), + headers.getFirst("Referer"), headers.getFirst("Content-Type"), + new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8))); + byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(responseStatus, bytes.length); + try (var output = exchange.getResponseBody()) { output.write(bytes); } + }); + server.start(); + endpoint = "http://127.0.0.1:" + server.getAddress().getPort() + "/submit"; + } + + @AfterEach + void stopServer() { server.stop(0); } + + private PreparedGallery prepared(String resolution) { + Gallery gallery = new Gallery(); + gallery.setGid(1234567); + var resolutions = new LinkedHashMap(); + resolutions.put(resolution, "12.5MiB"); + gallery.setAvailableResolution(resolutions); + return new PreparedGallery(gallery, endpoint, + GalleryUtil.verifyLink("https://exhentai.org/g/1234567/0123456789/")); + } + + @Test + void originalSubmissionMatchesBrowserFormAndHeaders() throws Exception { + Gallery gallery = GalleryUtil.submit(prepared("Original"), "Original"); + + assertEquals("已提交", gallery.getStatus()); + assertEquals(1, requestCount.get()); + Received request = received.get(); + assertEquals("POST", request.method()); + assertEquals("https://exhentai.org", request.origin()); + assertEquals(endpoint, request.referer()); + assertTrue(request.contentType().startsWith("application/x-www-form-urlencoded")); + assertEquals("hathdl_xres=org", request.body()); + } + + @Test + void resampledSubmissionSendsNumericWidth() throws Exception { + GalleryUtil.submit(prepared("1280x"), "1280x"); + assertEquals("hathdl_xres=1280", received.get().body()); + } + + @ParameterizedTest + @ValueSource(ints = {403, 429, 503}) + void rejectedSubmissionReportsStatusWithoutSecretsOrAutomaticReplay(int status) { + responseStatus = status; + responseBody = "private upstream response"; + PreparedGallery initial = prepared("Original"); + PreparedGallery withToken = new PreparedGallery(initial.gallery(), + endpoint + "?token=private-gallery-token", initial.origin()); + + IOException error = assertThrows(IOException.class, + () -> GalleryUtil.submit(withToken, "Original")); + + assertTrue(error.getMessage().contains("HTTP " + status)); + assertFalse(error.getMessage().contains("private-gallery-token")); + assertFalse(error.getMessage().contains(responseBody)); + assertEquals(1, requestCount.get(), "A rejected download POST must never be replayed"); + } + + @Test + void jsonImageApiRetainsItsEncodingAndDoesNotMutateCaller() throws Exception { + HashMap body = new HashMap<>(); + body.put("payload", "json"); + body.put("method", "imagedispatch"); + + GalleryUtil.requests(endpoint, "post", null, body); + + assertTrue(received.get().contentType().startsWith("application/json")); + assertEquals("imagedispatch", CustomUtil.objectMapper.readTree(received.get().body()).get("method").asText()); + assertFalse(received.get().body().contains("payload")); + assertEquals("json", body.get("payload")); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java b/src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java index 42220cc..5302680 100644 --- a/src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java +++ b/src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java @@ -41,7 +41,15 @@ class GalleryUtilTest { void verifyLinkAcceptsWellFormedGalleryUrl() { String url = "https://exhentai.org/g/1234567/0123456789ab/"; assertNotNull(GalleryUtil.verifyLink(url)); - assertEquals("", GalleryUtil.verifyLink(url), "通过校验时返回空串作为 origin"); + assertEquals("https://exhentai.org", GalleryUtil.verifyLink(url), "返回真实 origin，不能发送空 Origin"); + } + + @Test + void verifyLinkRejectsLookalikeHostsAndMalformedUrls() { + assertNull(GalleryUtil.verifyLink("https://exhentai.org.example.com/g/1234567/0123456789ab/")); + assertNull(GalleryUtil.verifyLink("https://example.com/g/1234567/exhentai0123456789/")); + assertNull(GalleryUtil.verifyLink("https://user@exhentai.org/g/1234567/0123456789ab/")); + assertNull(GalleryUtil.verifyLink("https://exhentai.org/g/1234567/invalid key/")); } // ---------- parseGid ----------