lion/lionwebsite-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

对获取图片的接口进行并发控制，同时只允许一个;galleryForQuery新增gid字段;gallery新增thumb_link字段

Browse Source
This commit is contained in:
chuzhongzai 2023-12-25 20:46:22 +08:00
parent 5cb2b7005b
commit bb6a247381
10 changed files with 129 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/com/lion/lionwebsite/Controller/GalleryManageController.java
View File

@ -91,30 +91,20 @@ public class GalleryManageController {
return galleryManageService.getWeekUsedAmount();
}
@GetMapping("/onlineImage/{page}")
public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){
galleryManageService.getOnlineImage(gid, page, request, response);
// @GetMapping("/onlineImage/{page}")
// public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){
// galleryManageService.getOnlineImage(gid, page, request, response);
// }
@PostMapping("/cache")
public String cacheImageKeys(String url){
return galleryManageService.cacheImagesKey(url);
}
// @PostMapping("/cache")
// public String cacheImageKeys(String url){
// return galleryManageService.cacheImagesKey(url);
// }
//
// @GetMapping("/onlineImage/{page}")
// public void getCacheImage(String gid, @PathVariable("page") int page, HttpServletRequest request, HttpServletResponse response){
// try {
// log.info("gid:" + gid + " page:" + page);
// galleryManageService.getCachedImage(gid, page, request, response);
// log.info("return gid:" + gid + " page:" + page);
// }catch (Exception e){
// e.printStackTrace();
// log.error(e.getMessage());
// try {
// response.sendError(500);
// }catch (Exception ignore){}
// }
// }
@GetMapping("/onlineImage/{page}")
public void getCacheImage(String gid, @PathVariable("page") int page, HttpServletRequest request, HttpServletResponse response){
galleryManageService.getCachedImage(gid, page, request, response);
}
@PostMapping("/reset")
public String resetUndone(){

3
src/main/java/com/lion/lionwebsite/Domain/GalleryForQuery.java
View File

@ -6,6 +6,9 @@ import lombok.Data;
@Data
public class GalleryForQuery {
@JsonProperty("gid")
String gid;
@JsonProperty("name")
String name;

105
src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java
View File

@ -23,10 +23,7 @@ import org.springframework.stereotype.Service;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static com.lion.lionwebsite.Util.CustomUtil.fourZeroFour;
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
@ -65,8 +62,6 @@ public class GalleryManageService {
PushService pushService;
ExecutorService convertThread;
public GalleryManageService(GalleryMapper galleryMapper, CollectMapper collectMapper, CustomConfigurationMapper configurationMapper, UserMapper userMapper, ShareFileMapper shareFileMapper,
TagMapper tagMapper, PageNameCacheMapper pageNameCacheMapper, RemoteService remoteService, PushService pushService, ImageCacheMapper imageCacheMapper) {
this.galleryMapper = galleryMapper;
@ -79,7 +74,6 @@ public class GalleryManageService {
this.remoteService = remoteService;
this.pushService = pushService;
this.imageCacheMapper = imageCacheMapper;
convertThread = Executors.newFixedThreadPool(2);
}
/**
@ -421,7 +415,6 @@ public class GalleryManageService {
return;
}
//页名缓存
String pageName;
if((pageName = page2pageNameCache.get(gid + String.valueOf(page))) == null) //内存
@ -453,25 +446,22 @@ public class GalleryManageService {
Response response = Response.generateResponse();
String gid = String.valueOf(GalleryUtil.parseGid(url));
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
Gallery gallery;
//已缓存过，直接返回
if(gidToKey != null) {
gallery = galleryMapper.selectGalleryByGid(Integer.parseInt(gid));
return response.success(objectMapper.valueToTree(gallery)).toJSONString();
return response.success(objectMapper.valueToTree(gidToKey)).toJSONString();
}
try {
gidToKey = new GidToKey();
gidToKey.setGid(gid);
gidToKey.setKey(url.split("/")[5].strip());
gallery = GalleryUtil.parse(url, false, null);
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
gidToKey.setPages(imageKeyCaches.size());
imageCacheMapper.insertGidToKey(gidToKey);
for (ImageKeyCache imageKeyCache : imageKeyCaches)
imageCacheMapper.insertImageKeyCache(imageKeyCache);
response.success(objectMapper.valueToTree(gallery));
}catch (IOException | ResolutionNotMatchException e){
response.success(objectMapper.valueToTree(gidToKey));
}catch (IOException e){
log.error(e.getMessage());
response.failure("网络波动或其他异常");
}
@ -479,7 +469,7 @@ public class GalleryManageService {
}
String[] suffixes = {".webp", ".gif"};
public void getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException {
public void getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) {
//检查文件夹是否存在
File folder = new File(cachePath + gid);
if(!folder.isDirectory())
@ -504,50 +494,61 @@ public class GalleryManageService {
return;
}
String imageUrl = null;
synchronized (this) {
//获取该图片key
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
if (imageKeyCache == null) {
response.sendError(404);
return;
}
synchronized (this){
if(response.isCommitted()) {
log.info("连接已关闭: gid=" + gid + " page=" + page);
return;
}
//获取图片地址
for (int i = 0; i < 2; i++) {
imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache);
if (imageUrl != null)
break;
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
}
if (imageUrl == null)
try {
String imageUrl = null;
//获取该图片key
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
if (imageKeyCache == null) {
CustomUtil.fourZeroFour(response);
return;
}
//获取图片地址
for (int i = 0; i < 2; i++) {
imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache);
if (imageUrl != null)
break;
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
}
if (imageUrl == null) {
CustomUtil.fourZeroFour(response);
log.error("获取图片url失败:gid=" + gid + " page=" + page + " imageKey=" + imageKeyCache.getImgkey());
response.sendError(404);
return;
} catch (IOException ignored) {
return;
}
}
//下载图片，转格式并返回
String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
String imagePath = cachePath + gid + "/" + page + suffix;
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
FileDownload.export(request, response, imagePath);
if (!suffix.equals(".gif"))
convertThread.submit(() -> {
ConvertCmd convertCmd = new ConvertCmd(true);
IMOperation operation = new IMOperation();
operation.addImage(imagePath);
operation.format("webp");
operation.addImage(imagePath.replace("jpg", "webp").replace("png", "webp"));
//下载图片，转格式并返回
String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
String imagePath = cachePath + gid + "/" + page + suffix;
try {
convertCmd.run(operation);
new File(imagePath).delete();
}catch (IOException | IM4JavaException | InterruptedException e){
log.error("文件" + imagePath + "转换失败");
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
}catch (Exception e){
log.error("下载图片失败:url" + imageUrl);
e.printStackTrace();
CustomUtil.fourZeroFour(response);
return;
}
});
if (!suffix.equals(".gif")) {
ConvertCmd convertCmd = new ConvertCmd(true);
IMOperation operation = new IMOperation();
operation.addImage(imagePath);
operation.format("webp");
operation.addImage(imagePath.replace(suffix, ".webp"));
try {
convertCmd.run(operation);
new File(imagePath).delete();
imagePath = imagePath.replace(suffix, ".webp");
} catch (IOException | IM4JavaException | InterruptedException e) {
log.error("文件" + imagePath + "转换失败");
}
}
FileDownload.export(request, response, imagePath);
}
}
public String resetUndone(){

5
src/main/java/com/lion/lionwebsite/Service/PublicServiceImpl.java
View File

@ -133,11 +133,6 @@ public class PublicServiceImpl {
outputStream.close();
}catch (IOException | URISyntaxException e){
e.printStackTrace();
try {
response.sendError(503);
}catch (IOException ex){
ex.printStackTrace();
}
}
}

4
src/main/java/com/lion/lionwebsite/Service/QueryService.java
View File

@ -1,6 +1,7 @@
package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Domain.GalleryForQuery;
import com.lion.lionwebsite.Util.GalleryUtil;
import com.lion.lionwebsite.Util.Response;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
@ -46,7 +47,6 @@ public class QueryService {
Document parse = Jsoup.parse(get);
Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr");
ArrayList<GalleryForQuery> galleries = new ArrayList<>();
ListIterator<Element> elementListIterator = elements.listIterator();
try {
@ -67,9 +67,9 @@ public class QueryService {
gallery.setThumbnailUrl(src); //thumbnailSrc
gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime
gallery.setLink(element.child(2).child(0).attr("href")); //link
gallery.setGid(GalleryUtil.parseGid(gallery.getLink()) + ""); //gid
gallery.setName(element.child(2).child(0).child(0).text()); //name
gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page
galleries.add(gallery);
}

33
src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java
View File

@ -71,6 +71,8 @@ public class GalleryUtil {
gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0]));
gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", "")));
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
String thumb_link = galleryDoc.select("#gd1 > div").getFirst().attr("style");
gallery.setThumb_link(thumb_link.substring(thumb_link.indexOf("url(") + 4, thumb_link.lastIndexOf(")")).replace("https://s.exhentai.org", ""));
//查找下载页面链接并初始化参数
String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1];
@ -165,19 +167,26 @@ public class GalleryUtil {
return imageKeyCaches;
}
public static String getMpvKey(String url) throws IOException {
public static synchronized String getMpvKey(String url){
String gid = String.valueOf(parseGid(url));
if(!gid2MpvKey.containsKey(gid))
refreshMpvKey(url);
return gid2MpvKey.get(gid);
}
public static void refreshMpvKey(String url) throws IOException {
public static void refreshMpvKey(String url) {
String temp = url.substring(url.indexOf("/g/") + 3);
String mpvUrl = "https://exhentai.org/mpv/" + temp;
HashMap<String, String> header = new HashMap<>();
header.put("Referer", url);
String content = requests(mpvUrl, "get", header, null);
String content;
try {
content = requests(mpvUrl, "get", header, null);
}catch (Exception e){
e.printStackTrace();
gid2MpvKey.put(parseGid(url) + "", null);
return;
}
Document document = Jsoup.parse(content);
Element script = document.select("body > script").get(1);
String[] scripts = script.html().split("\n");
@ -186,7 +195,7 @@ public class GalleryUtil {
System.out.println("刷新key:" + mpvKey);
}
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) throws IOException {
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) {
String apiUrl = "https://s.exhentai.org/api.php";
HashMap<String, String> header = new HashMap<>();
header.put("Referer", "https://exhentai.org");
@ -197,13 +206,17 @@ public class GalleryUtil {
body.put("method", "imagedispatch");
body.put("page", "" + imageKeyCache.getPage());
body.put("payload", "json");
System.out.println("开始获取imgurl:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
String result = requests(apiUrl, "post", header, body);
System.out.println("获取imgurl成功:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
JsonNode jsonNode = objectMapper.readTree(result);
if (jsonNode.has("error") && jsonNode.get("error").asText().equals("Key mismatch"))
try {
String result = requests(apiUrl, "post", header, body);
JsonNode jsonNode = objectMapper.readTree(result);
if (jsonNode.has("error") && jsonNode.get("error").asText().equals("Key mismatch"))
return null;
return jsonNode.get("i").asText();
}catch (Exception e){
System.out.println("获取imgurl失败:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
e.printStackTrace();
return null;
return jsonNode.get("i").asText();
}
}

2
src/main/resources/static/asserts/index/index.css
View File

File diff suppressed because one or more lines are too long

38
src/main/resources/static/asserts/index/index.js
View File

File diff suppressed because one or more lines are too long

2
src/main/resources/static/asserts/mobile/index.css
View File

File diff suppressed because one or more lines are too long

30
src/main/resources/static/asserts/mobile/index.js
View File

File diff suppressed because one or more lines are too long