Fix upstream archive submissions to match browser form encoding

src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java

@@ -10,12 +10,13 @@ import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.classic.methods.HttpPost;
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.entity.EntityBuilder;
-import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
+import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
 import org.apache.hc.client5.http.impl.classic.HttpClients;
 import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.apache.hc.core5.util.Timeout;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -28,6 +29,8 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -40,7 +43,7 @@ public class GalleryUtil {
     static String POST = "post";
     static String GET = "get";
 
-    static String FORM_DATA = "formData";
+    static String FORM_URLENCODED = "formUrlEncoded";
 
     static String JSON = "json";
 
@@ -58,6 +61,8 @@ public class GalleryUtil {
 
     /** Reusable HTTP client —不要每次请求新建 */
     private static final CloseableHttpClient httpClient = HttpClients.custom()
+            // An upstream download POST has no idempotency key; never replay it on 429/503.
+            .disableAutomaticRetries()
             .setDefaultRequestConfig(RequestConfig.custom()
                     .setConnectTimeout(Timeout.ofMilliseconds(5_000))
                     .setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000))
@@ -104,8 +109,7 @@ public class GalleryUtil {
         Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
         String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
         HashMap<String, String> headers = new HashMap<>();
-        headers.put("origin", origin);
+        headers.put("referer", url);
-        headers.put("referer", archiveLink);
         Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
         String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
         if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
@@ -132,7 +136,8 @@ public class GalleryUtil {
 
         //提交下载请求
         HashMap<String, String> body = new HashMap<>();
-        body.put("payload", FORM_DATA);
+        // The upstream form has no enctype: match the browser's default encoding.
+        body.put("payload", FORM_URLENCODED);
         body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
         String downloadPage = requests(download_link, POST, headers, body);
         Document downloadDoc = Jsoup.parse(downloadPage);
@@ -292,9 +297,18 @@ public class GalleryUtil {
         if(url.length() < 40)
             return null;
 
-        if(!url.contains("/g/") || !url.contains("exhentai"))
+        try {
+            URI uri = URI.create(url);
+            String scheme = uri.getScheme();
+            if ((!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme))
+                    || !"exhentai.org".equalsIgnoreCase(uri.getHost())
+                    || uri.getRawUserInfo() != null || uri.getPort() != -1
+                    || uri.getPath() == null || !uri.getPath().matches("/g/[0-9]+/[A-Za-z0-9]+/?"))
+                return null;
+            return scheme.toLowerCase(Locale.ROOT) + "://exhentai.org";
+        } catch (IllegalArgumentException invalidUrl) {
             return null;
-        return "";
+        }
     }
 
     /**
@@ -329,16 +343,18 @@ public class GalleryUtil {
                 httpPost.addHeader(header.getKey(), header.getValue());
 
             if(body != null) {
+                // Keep the caller's form intact; payload is a local encoding selector.
+                body = new HashMap<>(body);
                 String payload;
                 if((payload = body.remove("payload")).equals(JSON)) {
                     EntityBuilder entityBuilder = EntityBuilder.create();
                     entityBuilder.setContentType(ContentType.APPLICATION_JSON);
                     entityBuilder.setText(objectMapper.writeValueAsString(body));
                     httpPost.setEntity(entityBuilder.build());
-                } else if(payload.equals(FORM_DATA)) {
+                } else if(payload.equals(FORM_URLENCODED)) {
-                    MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
+                    var fields = body.entrySet().stream()
-                    body.forEach(multipartEntityBuilder::addTextBody);
+                            .map(entry -> new BasicNameValuePair(entry.getKey(), entry.getValue())).toList();
-                    httpPost.setEntity(multipartEntityBuilder.build());
+                    httpPost.setEntity(new UrlEncodedFormEntity(fields, StandardCharsets.UTF_8));
                 }
             }
             httpResponse = httpClient.execute(httpPost);
@@ -348,13 +364,15 @@ public class GalleryUtil {
             int statusCode = httpResponse.getCode();
             StringBuilder stringBuilder = new StringBuilder();
             if (statusCode == 200 && responseEntity != null) {
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent()))) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8))) {
                     String str;
                     while ((str = reader.readLine()) != null)
                         stringBuilder.append(str).append("\n");
                 }
             } else {
-                log.warn("{}:{}", url, statusCode);
+                // Never log gallery tokens, cookies or an arbitrary upstream error body.
+                throw new IOException("上游 HTTP " + statusCode + " (" + method.toUpperCase(Locale.ROOT)
+                        + " " + URI.create(url).getHost() + ")");
             }
             return stringBuilder.toString();
         }

src/test/java/com/lion/lionwebsite/Util/GallerySubmissionHttpTest.java

@@ -0,0 +1,113 @@
+package com.lion.lionwebsite.Util;
+
+import com.lion.lionwebsite.Domain.Gallery;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/** Exercise actual HTTP bytes against loopback, without contacting or queuing upstream. */
+class GallerySubmissionHttpTest {
+    private HttpServer server;
+    private String endpoint;
+    private int responseStatus = 200;
+    private String responseBody = "<div id='db'><p>Queued</p><p><strong>#12345</strong></p></div>";
+    private final AtomicInteger requestCount = new AtomicInteger();
+    private final AtomicReference<Received> received = new AtomicReference<>();
+
+    record Received(String method, String origin, String referer, String contentType, String body) { }
+
+    @BeforeEach
+    void startServer() throws IOException {
+        server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/submit", exchange -> {
+            requestCount.incrementAndGet();
+            var headers = exchange.getRequestHeaders();
+            received.set(new Received(exchange.getRequestMethod(), headers.getFirst("Origin"),
+                    headers.getFirst("Referer"), headers.getFirst("Content-Type"),
+                    new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)));
+            byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8);
+            exchange.sendResponseHeaders(responseStatus, bytes.length);
+            try (var output = exchange.getResponseBody()) { output.write(bytes); }
+        });
+        server.start();
+        endpoint = "http://127.0.0.1:" + server.getAddress().getPort() + "/submit";
+    }
+
+    @AfterEach
+    void stopServer() { server.stop(0); }
+
+    private PreparedGallery prepared(String resolution) {
+        Gallery gallery = new Gallery();
+        gallery.setGid(1234567);
+        var resolutions = new LinkedHashMap<String, String>();
+        resolutions.put(resolution, "12.5MiB");
+        gallery.setAvailableResolution(resolutions);
+        return new PreparedGallery(gallery, endpoint,
+                GalleryUtil.verifyLink("https://exhentai.org/g/1234567/0123456789/"));
+    }
+
+    @Test
+    void originalSubmissionMatchesBrowserFormAndHeaders() throws Exception {
+        Gallery gallery = GalleryUtil.submit(prepared("Original"), "Original");
+
+        assertEquals("已提交", gallery.getStatus());
+        assertEquals(1, requestCount.get());
+        Received request = received.get();
+        assertEquals("POST", request.method());
+        assertEquals("https://exhentai.org", request.origin());
+        assertEquals(endpoint, request.referer());
+        assertTrue(request.contentType().startsWith("application/x-www-form-urlencoded"));
+        assertEquals("hathdl_xres=org", request.body());
+    }
+
+    @Test
+    void resampledSubmissionSendsNumericWidth() throws Exception {
+        GalleryUtil.submit(prepared("1280x"), "1280x");
+        assertEquals("hathdl_xres=1280", received.get().body());
+    }
+
+    @ParameterizedTest
+    @ValueSource(ints = {403, 429, 503})
+    void rejectedSubmissionReportsStatusWithoutSecretsOrAutomaticReplay(int status) {
+        responseStatus = status;
+        responseBody = "private upstream response";
+        PreparedGallery initial = prepared("Original");
+        PreparedGallery withToken = new PreparedGallery(initial.gallery(),
+                endpoint + "?token=private-gallery-token", initial.origin());
+
+        IOException error = assertThrows(IOException.class,
+                () -> GalleryUtil.submit(withToken, "Original"));
+
+        assertTrue(error.getMessage().contains("HTTP " + status));
+        assertFalse(error.getMessage().contains("private-gallery-token"));
+        assertFalse(error.getMessage().contains(responseBody));
+        assertEquals(1, requestCount.get(), "A rejected download POST must never be replayed");
+    }
+
+    @Test
+    void jsonImageApiRetainsItsEncodingAndDoesNotMutateCaller() throws Exception {
+        HashMap<String, String> body = new HashMap<>();
+        body.put("payload", "json");
+        body.put("method", "imagedispatch");
+
+        GalleryUtil.requests(endpoint, "post", null, body);
+
+        assertTrue(received.get().contentType().startsWith("application/json"));
+        assertEquals("imagedispatch", CustomUtil.objectMapper.readTree(received.get().body()).get("method").asText());
+        assertFalse(received.get().body().contains("payload"));
+        assertEquals("json", body.get("payload"));
+    }
+}

src/test/java/com/lion/lionwebsite/Util/GalleryUtilTest.java

@@ -41,7 +41,15 @@ class GalleryUtilTest {
     void verifyLinkAcceptsWellFormedGalleryUrl() {
         String url = "https://exhentai.org/g/1234567/0123456789ab/";
         assertNotNull(GalleryUtil.verifyLink(url));
-        assertEquals("", GalleryUtil.verifyLink(url), "通过校验时返回空串作为 origin");
+        assertEquals("https://exhentai.org", GalleryUtil.verifyLink(url), "返回真实 origin，不能发送空 Origin");
+    }
+
+    @Test
+    void verifyLinkRejectsLookalikeHostsAndMalformedUrls() {
+        assertNull(GalleryUtil.verifyLink("https://exhentai.org.example.com/g/1234567/0123456789ab/"));
+        assertNull(GalleryUtil.verifyLink("https://example.com/g/1234567/exhentai0123456789/"));
+        assertNull(GalleryUtil.verifyLink("https://user@exhentai.org/g/1234567/0123456789ab/"));
+        assertNull(GalleryUtil.verifyLink("https://exhentai.org/g/1234567/invalid key/"));
     }
 
     // ---------- parseGid ----------