lion/lionwebsite-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix upstream archive submissions to match browser form encoding

Browse Source
This commit is contained in:
root
2026-10-22 22:27:59 +08:00
parent a72d99ff0a
commit b9358d9817
3 changed files with 153 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java
+31 -13
View File
@@ -10,12 +10,13 @@ import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.entity.EntityBuilder;
import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -28,6 +29,8 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -40,7 +43,7 @@ public class GalleryUtil {
static String POST = "post";
static String GET = "get";
static String FORM_DATA = "formData";
static String FORM_URLENCODED = "formUrlEncoded";
static String JSON = "json";
@@ -58,6 +61,8 @@ public class GalleryUtil {
/** Reusable HTTP client —不要每次请求新建 */
private static final CloseableHttpClient httpClient = HttpClients.custom()
// An upstream download POST has no idempotency key; never replay it on 429/503.
.disableAutomaticRetries()
.setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(Timeout.ofMilliseconds(5_000))
.setConnectionRequestTimeout(Timeout.ofMilliseconds(5_000))
@@ -104,8 +109,7 @@ public class GalleryUtil {
Gallery gallery = GalleryPageParser.metadata(galleryDoc, url, gid);
String archiveLink = GalleryPageParser.archiveLink(galleryDoc);
HashMap<String, String> headers = new HashMap<>();
headers.put("origin", origin);
headers.put("referer", archiveLink);
headers.put("referer", url);
Document downloadDoc = Jsoup.parse(requests(archiveLink, GET, headers, null), archiveLink);
String download_link = downloadDoc.select("#hathdl_form").attr("abs:action");
if (download_link.isBlank()) throw new IOException("上游页面缺少下载提交表单");
@@ -132,7 +136,8 @@ public class GalleryUtil {
//提交下载请求
HashMap<String, String> body = new HashMap<>();
body.put("payload", FORM_DATA);
// The upstream form has no enctype: match the browser's default encoding.
body.put("payload", FORM_URLENCODED);
body.put("hathdl_xres", targetResolution.replace("x", "").replace("Original", "org"));
String downloadPage = requests(download_link, POST, headers, body);
Document downloadDoc = Jsoup.parse(downloadPage);
@@ -292,9 +297,18 @@ public class GalleryUtil {
if(url.length() < 40)
return null;
if(!url.contains("/g/") || !url.contains("exhentai"))
try {
URI uri = URI.create(url);
String scheme = uri.getScheme();
if ((!"https".equalsIgnoreCase(scheme) && !"http".equalsIgnoreCase(scheme))
|| !"exhentai.org".equalsIgnoreCase(uri.getHost())
|| uri.getRawUserInfo() != null || uri.getPort() != -1
|| uri.getPath() == null || !uri.getPath().matches("/g/[0-9]+/[A-Za-z0-9]+/?"))
return null;
return scheme.toLowerCase(Locale.ROOT) + "://exhentai.org";
} catch (IllegalArgumentException invalidUrl) {
return null;
return "";
}
}
/**
@@ -329,16 +343,18 @@ public class GalleryUtil {
httpPost.addHeader(header.getKey(), header.getValue());
if(body != null) {
// Keep the caller's form intact; payload is a local encoding selector.
body = new HashMap<>(body);
String payload;
if((payload = body.remove("payload")).equals(JSON)) {
EntityBuilder entityBuilder = EntityBuilder.create();
entityBuilder.setContentType(ContentType.APPLICATION_JSON);
entityBuilder.setText(objectMapper.writeValueAsString(body));
httpPost.setEntity(entityBuilder.build());
} else if(payload.equals(FORM_DATA)) {
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
body.forEach(multipartEntityBuilder::addTextBody);
httpPost.setEntity(multipartEntityBuilder.build());
} else if(payload.equals(FORM_URLENCODED)) {
var fields = body.entrySet().stream()
.map(entry -> new BasicNameValuePair(entry.getKey(), entry.getValue())).toList();
httpPost.setEntity(new UrlEncodedFormEntity(fields, StandardCharsets.UTF_8));
}
}
httpResponse = httpClient.execute(httpPost);
@@ -348,13 +364,15 @@ public class GalleryUtil {
int statusCode = httpResponse.getCode();
StringBuilder stringBuilder = new StringBuilder();
if (statusCode == 200 && responseEntity != null) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent()))) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseEntity.getContent(), StandardCharsets.UTF_8))) {
String str;
while ((str = reader.readLine()) != null)
stringBuilder.append(str).append("\n");
}
} else {
log.warn("{}:{}", url, statusCode);
// Never log gallery tokens, cookies or an arbitrary upstream error body.
throw new IOException("上游 HTTP " + statusCode + " (" + method.toUpperCase(Locale.ROOT)
+ " " + URI.create(url).getHost() + ")");
}
return stringBuilder.toString();
}