对获取图片的接口进行并发控制,同时只允许一个;galleryForQuery新增gid字段;gallery新增thumb_link字段
This commit is contained in:
parent
5cb2b7005b
commit
bb6a247381
@ -91,30 +91,20 @@ public class GalleryManageController {
|
|||||||
return galleryManageService.getWeekUsedAmount();
|
return galleryManageService.getWeekUsedAmount();
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/onlineImage/{page}")
|
// @GetMapping("/onlineImage/{page}")
|
||||||
public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){
|
// public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){
|
||||||
galleryManageService.getOnlineImage(gid, page, request, response);
|
// galleryManageService.getOnlineImage(gid, page, request, response);
|
||||||
|
// }
|
||||||
|
|
||||||
|
@PostMapping("/cache")
|
||||||
|
public String cacheImageKeys(String url){
|
||||||
|
return galleryManageService.cacheImagesKey(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
// @PostMapping("/cache")
|
@GetMapping("/onlineImage/{page}")
|
||||||
// public String cacheImageKeys(String url){
|
public void getCacheImage(String gid, @PathVariable("page") int page, HttpServletRequest request, HttpServletResponse response){
|
||||||
// return galleryManageService.cacheImagesKey(url);
|
galleryManageService.getCachedImage(gid, page, request, response);
|
||||||
// }
|
}
|
||||||
//
|
|
||||||
// @GetMapping("/onlineImage/{page}")
|
|
||||||
// public void getCacheImage(String gid, @PathVariable("page") int page, HttpServletRequest request, HttpServletResponse response){
|
|
||||||
// try {
|
|
||||||
// log.info("gid:" + gid + " page:" + page);
|
|
||||||
// galleryManageService.getCachedImage(gid, page, request, response);
|
|
||||||
// log.info("return gid:" + gid + " page:" + page);
|
|
||||||
// }catch (Exception e){
|
|
||||||
// e.printStackTrace();
|
|
||||||
// log.error(e.getMessage());
|
|
||||||
// try {
|
|
||||||
// response.sendError(500);
|
|
||||||
// }catch (Exception ignore){}
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
@PostMapping("/reset")
|
@PostMapping("/reset")
|
||||||
public String resetUndone(){
|
public String resetUndone(){
|
||||||
|
|||||||
@ -6,6 +6,9 @@ import lombok.Data;
|
|||||||
@Data
|
@Data
|
||||||
public class GalleryForQuery {
|
public class GalleryForQuery {
|
||||||
|
|
||||||
|
@JsonProperty("gid")
|
||||||
|
String gid;
|
||||||
|
|
||||||
@JsonProperty("name")
|
@JsonProperty("name")
|
||||||
String name;
|
String name;
|
||||||
|
|
||||||
|
|||||||
@ -23,10 +23,7 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
|
|
||||||
import static com.lion.lionwebsite.Util.CustomUtil.fourZeroFour;
|
import static com.lion.lionwebsite.Util.CustomUtil.fourZeroFour;
|
||||||
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
|
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
|
||||||
@ -65,8 +62,6 @@ public class GalleryManageService {
|
|||||||
|
|
||||||
PushService pushService;
|
PushService pushService;
|
||||||
|
|
||||||
ExecutorService convertThread;
|
|
||||||
|
|
||||||
public GalleryManageService(GalleryMapper galleryMapper, CollectMapper collectMapper, CustomConfigurationMapper configurationMapper, UserMapper userMapper, ShareFileMapper shareFileMapper,
|
public GalleryManageService(GalleryMapper galleryMapper, CollectMapper collectMapper, CustomConfigurationMapper configurationMapper, UserMapper userMapper, ShareFileMapper shareFileMapper,
|
||||||
TagMapper tagMapper, PageNameCacheMapper pageNameCacheMapper, RemoteService remoteService, PushService pushService, ImageCacheMapper imageCacheMapper) {
|
TagMapper tagMapper, PageNameCacheMapper pageNameCacheMapper, RemoteService remoteService, PushService pushService, ImageCacheMapper imageCacheMapper) {
|
||||||
this.galleryMapper = galleryMapper;
|
this.galleryMapper = galleryMapper;
|
||||||
@ -79,7 +74,6 @@ public class GalleryManageService {
|
|||||||
this.remoteService = remoteService;
|
this.remoteService = remoteService;
|
||||||
this.pushService = pushService;
|
this.pushService = pushService;
|
||||||
this.imageCacheMapper = imageCacheMapper;
|
this.imageCacheMapper = imageCacheMapper;
|
||||||
convertThread = Executors.newFixedThreadPool(2);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -421,7 +415,6 @@ public class GalleryManageService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
//页名缓存
|
//页名缓存
|
||||||
String pageName;
|
String pageName;
|
||||||
if((pageName = page2pageNameCache.get(gid + String.valueOf(page))) == null) //内存
|
if((pageName = page2pageNameCache.get(gid + String.valueOf(page))) == null) //内存
|
||||||
@ -453,25 +446,22 @@ public class GalleryManageService {
|
|||||||
Response response = Response.generateResponse();
|
Response response = Response.generateResponse();
|
||||||
String gid = String.valueOf(GalleryUtil.parseGid(url));
|
String gid = String.valueOf(GalleryUtil.parseGid(url));
|
||||||
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
|
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
|
||||||
Gallery gallery;
|
|
||||||
//已缓存过,直接返回
|
//已缓存过,直接返回
|
||||||
if(gidToKey != null) {
|
if(gidToKey != null) {
|
||||||
gallery = galleryMapper.selectGalleryByGid(Integer.parseInt(gid));
|
return response.success(objectMapper.valueToTree(gidToKey)).toJSONString();
|
||||||
return response.success(objectMapper.valueToTree(gallery)).toJSONString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
gidToKey = new GidToKey();
|
gidToKey = new GidToKey();
|
||||||
gidToKey.setGid(gid);
|
gidToKey.setGid(gid);
|
||||||
gidToKey.setKey(url.split("/")[5].strip());
|
gidToKey.setKey(url.split("/")[5].strip());
|
||||||
gallery = GalleryUtil.parse(url, false, null);
|
|
||||||
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
|
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
|
||||||
gidToKey.setPages(imageKeyCaches.size());
|
gidToKey.setPages(imageKeyCaches.size());
|
||||||
imageCacheMapper.insertGidToKey(gidToKey);
|
imageCacheMapper.insertGidToKey(gidToKey);
|
||||||
for (ImageKeyCache imageKeyCache : imageKeyCaches)
|
for (ImageKeyCache imageKeyCache : imageKeyCaches)
|
||||||
imageCacheMapper.insertImageKeyCache(imageKeyCache);
|
imageCacheMapper.insertImageKeyCache(imageKeyCache);
|
||||||
response.success(objectMapper.valueToTree(gallery));
|
response.success(objectMapper.valueToTree(gidToKey));
|
||||||
}catch (IOException | ResolutionNotMatchException e){
|
}catch (IOException e){
|
||||||
log.error(e.getMessage());
|
log.error(e.getMessage());
|
||||||
response.failure("网络波动或其他异常");
|
response.failure("网络波动或其他异常");
|
||||||
}
|
}
|
||||||
@ -479,7 +469,7 @@ public class GalleryManageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
String[] suffixes = {".webp", ".gif"};
|
String[] suffixes = {".webp", ".gif"};
|
||||||
public void getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException {
|
public void getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) {
|
||||||
//检查文件夹是否存在
|
//检查文件夹是否存在
|
||||||
File folder = new File(cachePath + gid);
|
File folder = new File(cachePath + gid);
|
||||||
if(!folder.isDirectory())
|
if(!folder.isDirectory())
|
||||||
@ -504,12 +494,17 @@ public class GalleryManageService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
synchronized (this){
|
||||||
|
if(response.isCommitted()) {
|
||||||
|
log.info("连接已关闭: gid=" + gid + " page=" + page);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
String imageUrl = null;
|
String imageUrl = null;
|
||||||
synchronized (this) {
|
|
||||||
//获取该图片key
|
//获取该图片key
|
||||||
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
|
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
|
||||||
if (imageKeyCache == null) {
|
if (imageKeyCache == null) {
|
||||||
response.sendError(404);
|
CustomUtil.fourZeroFour(response);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -520,34 +515,40 @@ public class GalleryManageService {
|
|||||||
break;
|
break;
|
||||||
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
|
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
|
||||||
}
|
}
|
||||||
if (imageUrl == null)
|
|
||||||
try {
|
if (imageUrl == null) {
|
||||||
|
CustomUtil.fourZeroFour(response);
|
||||||
log.error("获取图片url失败:gid=" + gid + " page=" + page + " imageKey=" + imageKeyCache.getImgkey());
|
log.error("获取图片url失败:gid=" + gid + " page=" + page + " imageKey=" + imageKeyCache.getImgkey());
|
||||||
response.sendError(404);
|
|
||||||
return;
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
//下载图片,转格式并返回
|
//下载图片,转格式并返回
|
||||||
String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
|
String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
|
||||||
String imagePath = cachePath + gid + "/" + page + suffix;
|
String imagePath = cachePath + gid + "/" + page + suffix;
|
||||||
|
try {
|
||||||
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
|
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
|
||||||
FileDownload.export(request, response, imagePath);
|
}catch (Exception e){
|
||||||
if (!suffix.equals(".gif"))
|
log.error("下载图片失败:url" + imageUrl);
|
||||||
convertThread.submit(() -> {
|
e.printStackTrace();
|
||||||
|
CustomUtil.fourZeroFour(response);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!suffix.equals(".gif")) {
|
||||||
ConvertCmd convertCmd = new ConvertCmd(true);
|
ConvertCmd convertCmd = new ConvertCmd(true);
|
||||||
IMOperation operation = new IMOperation();
|
IMOperation operation = new IMOperation();
|
||||||
operation.addImage(imagePath);
|
operation.addImage(imagePath);
|
||||||
operation.format("webp");
|
operation.format("webp");
|
||||||
operation.addImage(imagePath.replace("jpg", "webp").replace("png", "webp"));
|
operation.addImage(imagePath.replace(suffix, ".webp"));
|
||||||
try {
|
try {
|
||||||
convertCmd.run(operation);
|
convertCmd.run(operation);
|
||||||
new File(imagePath).delete();
|
new File(imagePath).delete();
|
||||||
}catch (IOException | IM4JavaException | InterruptedException e){
|
imagePath = imagePath.replace(suffix, ".webp");
|
||||||
|
} catch (IOException | IM4JavaException | InterruptedException e) {
|
||||||
log.error("文件" + imagePath + "转换失败");
|
log.error("文件" + imagePath + "转换失败");
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
FileDownload.export(request, response, imagePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String resetUndone(){
|
public String resetUndone(){
|
||||||
|
|||||||
@ -133,11 +133,6 @@ public class PublicServiceImpl {
|
|||||||
outputStream.close();
|
outputStream.close();
|
||||||
}catch (IOException | URISyntaxException e){
|
}catch (IOException | URISyntaxException e){
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
try {
|
|
||||||
response.sendError(503);
|
|
||||||
}catch (IOException ex){
|
|
||||||
ex.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package com.lion.lionwebsite.Service;
|
package com.lion.lionwebsite.Service;
|
||||||
|
|
||||||
import com.lion.lionwebsite.Domain.GalleryForQuery;
|
import com.lion.lionwebsite.Domain.GalleryForQuery;
|
||||||
|
import com.lion.lionwebsite.Util.GalleryUtil;
|
||||||
import com.lion.lionwebsite.Util.Response;
|
import com.lion.lionwebsite.Util.Response;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
@ -46,7 +47,6 @@ public class QueryService {
|
|||||||
Document parse = Jsoup.parse(get);
|
Document parse = Jsoup.parse(get);
|
||||||
Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr");
|
Elements elements = parse.select("body > div.ido > div:nth-child(2) > table > tbody > tr");
|
||||||
|
|
||||||
|
|
||||||
ArrayList<GalleryForQuery> galleries = new ArrayList<>();
|
ArrayList<GalleryForQuery> galleries = new ArrayList<>();
|
||||||
ListIterator<Element> elementListIterator = elements.listIterator();
|
ListIterator<Element> elementListIterator = elements.listIterator();
|
||||||
try {
|
try {
|
||||||
@ -67,9 +67,9 @@ public class QueryService {
|
|||||||
gallery.setThumbnailUrl(src); //thumbnailSrc
|
gallery.setThumbnailUrl(src); //thumbnailSrc
|
||||||
gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime
|
gallery.setUploadTime(element.child(1).select("div > div [onclick]").get(1).text()); //uploadTime
|
||||||
gallery.setLink(element.child(2).child(0).attr("href")); //link
|
gallery.setLink(element.child(2).child(0).attr("href")); //link
|
||||||
|
gallery.setGid(GalleryUtil.parseGid(gallery.getLink()) + ""); //gid
|
||||||
gallery.setName(element.child(2).child(0).child(0).text()); //name
|
gallery.setName(element.child(2).child(0).child(0).text()); //name
|
||||||
gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page
|
gallery.setPage(Integer.parseInt(element.child(3).child(1).text().split(" ")[0])); //page
|
||||||
|
|
||||||
galleries.add(gallery);
|
galleries.add(gallery);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -71,6 +71,8 @@ public class GalleryUtil {
|
|||||||
gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0]));
|
gallery.setPages(Integer.parseInt(galleryDoc.select("#gdd > table > tbody > tr:nth-child(6) > td.gdt2").text().split(" ")[0]));
|
||||||
gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", "")));
|
gallery.setFileSize(CustomUtil.stringToFileSize(galleryDoc.select("#gdd > table > tbody > tr:nth-child(5) > td.gdt2").text().replace(" ", "")));
|
||||||
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
|
gallery.setDisplayFileSize(CustomUtil.fileSizeToString(gallery.getFileSize()));
|
||||||
|
String thumb_link = galleryDoc.select("#gd1 > div").getFirst().attr("style");
|
||||||
|
gallery.setThumb_link(thumb_link.substring(thumb_link.indexOf("url(") + 4, thumb_link.lastIndexOf(")")).replace("https://s.exhentai.org", ""));
|
||||||
|
|
||||||
//查找下载页面链接并初始化参数
|
//查找下载页面链接并初始化参数
|
||||||
String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1];
|
String download_link = galleryDoc.select("#gd5 > p:nth-child(2) > a").attr("onclick").split("'")[1];
|
||||||
@ -165,19 +167,26 @@ public class GalleryUtil {
|
|||||||
return imageKeyCaches;
|
return imageKeyCaches;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getMpvKey(String url) throws IOException {
|
public static synchronized String getMpvKey(String url){
|
||||||
String gid = String.valueOf(parseGid(url));
|
String gid = String.valueOf(parseGid(url));
|
||||||
if(!gid2MpvKey.containsKey(gid))
|
if(!gid2MpvKey.containsKey(gid))
|
||||||
refreshMpvKey(url);
|
refreshMpvKey(url);
|
||||||
return gid2MpvKey.get(gid);
|
return gid2MpvKey.get(gid);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void refreshMpvKey(String url) throws IOException {
|
public static void refreshMpvKey(String url) {
|
||||||
String temp = url.substring(url.indexOf("/g/") + 3);
|
String temp = url.substring(url.indexOf("/g/") + 3);
|
||||||
String mpvUrl = "https://exhentai.org/mpv/" + temp;
|
String mpvUrl = "https://exhentai.org/mpv/" + temp;
|
||||||
HashMap<String, String> header = new HashMap<>();
|
HashMap<String, String> header = new HashMap<>();
|
||||||
header.put("Referer", url);
|
header.put("Referer", url);
|
||||||
String content = requests(mpvUrl, "get", header, null);
|
String content;
|
||||||
|
try {
|
||||||
|
content = requests(mpvUrl, "get", header, null);
|
||||||
|
}catch (Exception e){
|
||||||
|
e.printStackTrace();
|
||||||
|
gid2MpvKey.put(parseGid(url) + "", null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
Document document = Jsoup.parse(content);
|
Document document = Jsoup.parse(content);
|
||||||
Element script = document.select("body > script").get(1);
|
Element script = document.select("body > script").get(1);
|
||||||
String[] scripts = script.html().split("\n");
|
String[] scripts = script.html().split("\n");
|
||||||
@ -186,7 +195,7 @@ public class GalleryUtil {
|
|||||||
System.out.println("刷新key:" + mpvKey);
|
System.out.println("刷新key:" + mpvKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) throws IOException {
|
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) {
|
||||||
String apiUrl = "https://s.exhentai.org/api.php";
|
String apiUrl = "https://s.exhentai.org/api.php";
|
||||||
HashMap<String, String> header = new HashMap<>();
|
HashMap<String, String> header = new HashMap<>();
|
||||||
header.put("Referer", "https://exhentai.org");
|
header.put("Referer", "https://exhentai.org");
|
||||||
@ -197,13 +206,17 @@ public class GalleryUtil {
|
|||||||
body.put("method", "imagedispatch");
|
body.put("method", "imagedispatch");
|
||||||
body.put("page", "" + imageKeyCache.getPage());
|
body.put("page", "" + imageKeyCache.getPage());
|
||||||
body.put("payload", "json");
|
body.put("payload", "json");
|
||||||
System.out.println("开始获取imgurl:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
|
try {
|
||||||
String result = requests(apiUrl, "post", header, body);
|
String result = requests(apiUrl, "post", header, body);
|
||||||
System.out.println("获取imgurl成功:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
|
|
||||||
JsonNode jsonNode = objectMapper.readTree(result);
|
JsonNode jsonNode = objectMapper.readTree(result);
|
||||||
if (jsonNode.has("error") && jsonNode.get("error").asText().equals("Key mismatch"))
|
if (jsonNode.has("error") && jsonNode.get("error").asText().equals("Key mismatch"))
|
||||||
return null;
|
return null;
|
||||||
return jsonNode.get("i").asText();
|
return jsonNode.get("i").asText();
|
||||||
|
}catch (Exception e){
|
||||||
|
System.out.println("获取imgurl失败:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user