lion/lionwebsite-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

上线动态获取图片后端,使用mpvKey以及imgKey请求图片链接;

Browse Source
This commit is contained in:
chuzhongzai 2023-12-24 21:04:53 +08:00
parent 1db8cdf264
commit 5cb2b7005b
8 changed files with 261 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/lion/lionwebsite/Configuration/CustomBean.java
View File

@ -14,7 +14,7 @@ import org.springframework.context.annotation.Configuration;
import com.pengrad.telegrambot.model.User; import com.pengrad.telegrambot.model.User;
@Configuration @Configuration
@RegisterReflectionForBinding(classes = {CustomConfiguration.class, @RegisterReflectionForBinding(classes = {CustomConfiguration.class, GidToKey.class, ImageKeyCache.class,
GalleryForQuery.class, Gallery.class, GalleryTask.class, HikariConfig.class, GalleryForQuery.class, Gallery.class, GalleryTask.class, HikariConfig.class,
PageNameCache.class, ShareFile.class, Tag.class, TagMark.class, User.class, PageNameCache.class, ShareFile.class, Tag.class, TagMark.class, User.class,
SendResponse.class, Message.class, com.pengrad.telegrambot.model.User.class, SendResponse.class, Message.class, com.pengrad.telegrambot.model.User.class,

27
src/main/java/com/lion/lionwebsite/Controller/GalleryManageController.java
View File

@ -91,19 +91,30 @@ public class GalleryManageController {
return galleryManageService.getWeekUsedAmount(); return galleryManageService.getWeekUsedAmount();
} }
@GetMapping("/onlineImage/{page}") @GetMapping("/onlineImage/{page}")
public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){ public void getOnlineImage(Integer gid, @PathVariable("page") Short page, HttpServletRequest request, HttpServletResponse response){
galleryManageService.getOnlineImage(gid, page, request, response); galleryManageService.getOnlineImage(gid, page, request, response);
} }
// @PostMapping("/cache")
@PostMapping("/share") // public String cacheImageKeys(String url){
public String shareGallery(Integer gid, Integer userId, Integer expireHour){ // return galleryManageService.cacheImagesKey(url);
if(userId.equals(3)) // }
return galleryManageService.shareGallery(gid, expireHour); //
return Response._failure("非法访问"); // @GetMapping("/onlineImage/{page}")
} // public void getCacheImage(String gid, @PathVariable("page") int page, HttpServletRequest request, HttpServletResponse response){
// try {
// log.info("gid:" + gid + " page:" + page);
// galleryManageService.getCachedImage(gid, page, request, response);
// log.info("return gid:" + gid + " page:" + page);
// }catch (Exception e){
// e.printStackTrace();
// log.error(e.getMessage());
// try {
// response.sendError(500);
// }catch (Exception ignore){}
// }
// }
@PostMapping("/reset") @PostMapping("/reset")
public String resetUndone(){ public String resetUndone(){

24
src/main/java/com/lion/lionwebsite/Dao/cache/ImageCacheMapper.java vendored Normal file
View File

@ -0,0 +1,24 @@
package com.lion.lionwebsite.Dao.cache;
import com.lion.lionwebsite.Domain.GidToKey;
import com.lion.lionwebsite.Domain.ImageKeyCache;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
@Mapper
public interface ImageCacheMapper {
@Insert("insert into ImageKeyCache values (#{gid}, #{imgkey}, #{page})")
void insertImageKeyCache(ImageKeyCache imageKeyCache);
@Insert("insert into gidToKey values (#{gid}, #{key}, #{pages})")
void insertGidToKey(GidToKey gidToKey);
@Select("select * from ImageKeyCache where gid=#{gid} and page=#{page}")
ImageKeyCache selectImageKeyCacheByGidAndPage(@Param("gid") String gid, @Param("page") int page);
@Select("select * from gidToKey where gid=#{gid}")
GidToKey selectKeyByGid(String gid);
}

14
src/main/java/com/lion/lionwebsite/Domain/GidToKey.java Normal file
View File

@ -0,0 +1,14 @@
package com.lion.lionwebsite.Domain;
import lombok.Data;
@Data
public class GidToKey {
String gid;
String key;
int pages;
public String toUrl(){
return "https://exhentai.org/g/" + gid + "/" + key;
}
}

10
src/main/java/com/lion/lionwebsite/Domain/ImageKeyCache.java Normal file
View File

@ -0,0 +1,10 @@
package com.lion.lionwebsite.Domain;
import lombok.Data;
@Data
public class ImageKeyCache {
String gid;
int page;
String imgkey;
}

209
src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java
View File

@ -1,7 +1,7 @@
package com.lion.lionwebsite.Service; package com.lion.lionwebsite.Service;
import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.RandomUtil; import com.lion.lionwebsite.Dao.cache.ImageCacheMapper;
import com.lion.lionwebsite.Dao.cache.PageNameCacheMapper; import com.lion.lionwebsite.Dao.cache.PageNameCacheMapper;
import com.lion.lionwebsite.Dao.normal.*; import com.lion.lionwebsite.Dao.normal.*;
import com.lion.lionwebsite.Domain.*; import com.lion.lionwebsite.Domain.*;
@ -16,23 +16,28 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.im4java.core.ConvertCmd;
import org.im4java.core.IM4JavaException;
import org.im4java.core.IMOperation;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.*; import java.io.*;
import java.time.LocalDateTime; import java.net.URI;
import java.time.ZoneId; import java.net.URISyntaxException;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static com.lion.lionwebsite.Util.CustomUtil.dateTimeFormatter;
import static com.lion.lionwebsite.Util.CustomUtil.fourZeroFour; import static com.lion.lionwebsite.Util.CustomUtil.fourZeroFour;
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
import static com.lion.lionwebsite.Util.GalleryUtil.*;
@Service @Service
@Data @Data
@Slf4j @Slf4j
public class GalleryManageService { public class GalleryManageService {
String TargetPath = "/root/gallery/"; String TargetPath = "/root/gallery/";
String cachePath = "/root/galleryCache/";
int cacheSize = 100;
Map<Integer, String> gid2name_cache = new HashMap<>(); Map<Integer, String> gid2name_cache = new HashMap<>();
@ -54,12 +59,16 @@ public class GalleryManageService {
PageNameCacheMapper pageNameCacheMapper; PageNameCacheMapper pageNameCacheMapper;
ImageCacheMapper imageCacheMapper;
RemoteService remoteService; RemoteService remoteService;
PushService pushService; PushService pushService;
ExecutorService convertThread;
public GalleryManageService(GalleryMapper galleryMapper, CollectMapper collectMapper, CustomConfigurationMapper configurationMapper, UserMapper userMapper, ShareFileMapper shareFileMapper, public GalleryManageService(GalleryMapper galleryMapper, CollectMapper collectMapper, CustomConfigurationMapper configurationMapper, UserMapper userMapper, ShareFileMapper shareFileMapper,
TagMapper tagMapper, PageNameCacheMapper pageNameCacheMapper, RemoteService remoteService, PushService pushService) { TagMapper tagMapper, PageNameCacheMapper pageNameCacheMapper, RemoteService remoteService, PushService pushService, ImageCacheMapper imageCacheMapper) {
this.galleryMapper = galleryMapper; this.galleryMapper = galleryMapper;
this.collectMapper = collectMapper; this.collectMapper = collectMapper;
this.configurationMapper = configurationMapper; this.configurationMapper = configurationMapper;
@ -69,6 +78,8 @@ public class GalleryManageService {
this.pageNameCacheMapper = pageNameCacheMapper; this.pageNameCacheMapper = pageNameCacheMapper;
this.remoteService = remoteService; this.remoteService = remoteService;
this.pushService = pushService; this.pushService = pushService;
this.imageCacheMapper = imageCacheMapper;
convertThread = Executors.newFixedThreadPool(2);
} }
/** /**
@ -123,12 +134,11 @@ public class GalleryManageService {
} }
} }
} catch (ResolutionNotMatchException e) { } catch (ResolutionNotMatchException e) {
e.printStackTrace();
response.failure("提交失败，分辨率不存在"); response.failure("提交失败，分辨率不存在");
pushService.taskCreateReport(user.getUsername(), link, response); pushService.taskCreateReport(user.getUsername(), link, response);
return response.toJSONString(); return response.toJSONString();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); log.error(e.getMessage());
response.failure("IO错误，可能是网络波动"); response.failure("IO错误，可能是网络波动");
pushService.taskCreateReport(user.getUsername(), link, response); pushService.taskCreateReport(user.getUsername(), link, response);
return response.toJSONString(); return response.toJSONString();
@ -320,54 +330,6 @@ public class GalleryManageService {
return response.toJSONString(); return response.toJSONString();
} }
// /**
// * 获取本子图片的名字
// * @param gid gid
// * @return 名字数组
// */
// public String selectOnlineFilename(Integer gid){
// Response response = Response.generateResponse();
// String path;
//
// if((path = gid2name_cache.get(gid)) == null) {
// Gallery gallery = galleryMapper.selectGalleryByGid(gid);
//
// if (gallery == null) {
// response.failure("本子不存在");
// return response.toJSONString();
// }
//
// path = TargetPath + gallery.getName();
// gid2name_cache.put(gid, path);
// if(gid2name_cache.size() > cacheSize)
// gid2name_cache.remove(gid2name_cache.keySet().iterator().next());
// }
//
// File file = new File(path);
// if(!file.isDirectory()){
// response.failure("本子文件已被删除");
// return response.toJSONString();
// }
//
// File[] files = file.listFiles(pathname -> pathname.getName().endsWith(".webp") && !pathname.getName().startsWith("thumbnail"));
// if(files == null || files.length == 0){
// response.failure("本子文件丢失");
// return response.toJSONString();
// }
//
// ArrayList<String> images = new ArrayList<>();
// for(File image: files){
// images.add(image.getName().replace(".webp", ""));
// }
// //字符串长度相等时比较字典顺序，不相等时比较长度
// images.sort((s1, s2) ->
// s1.length() == s2.length() ? s1.compareTo(s2): s1.length() - s2.length()
// );
//
// response.success(new ObjectMapper().valueToTree(images).toString());
// return response.toJSONString();
// }
/** /**
* 删除本子以及对应的文件（如果存在的话） * 删除本子以及对应的文件（如果存在的话）
* *
@ -384,7 +346,7 @@ public class GalleryManageService {
} }
ArrayList<Integer> collector = collectMapper.selectCollectorByGid(gallery.getGid()); ArrayList<Integer> collector = collectMapper.selectCollectorByGid(gallery.getGid());
if (!(collector.isEmpty() || collector.size() == 1 && collector.get(0).equals(user.getId()) //判断收藏 if (!(collector.isEmpty() || collector.size() == 1 && collector.getFirst().equals(user.getId()) //判断收藏
&& gallery.getDownloader() == user.getId())) //判断下载 && gallery.getDownloader() == user.getId())) //判断下载
response.failure("删除失败，该本子已被别人收藏或你不是下载人"); response.failure("删除失败，该本子已被别人收藏或你不是下载人");
else { else {
@ -432,7 +394,7 @@ public class GalleryManageService {
/** /**
* 获取在线图片 * 获取在线图片
* @param gid gid * @param gid gid
* @param page 文件名 * @param page 文件名
* @param request 请求对象 * @param request 请求对象
* @param response 响应对象 * @param response 响应对象
*/ */
@ -487,42 +449,105 @@ public class GalleryManageService {
FileDownload.export(request, response, file.getAbsolutePath()); FileDownload.export(request, response, file.getAbsolutePath());
} }
public String cacheImagesKey(String url) {
public String shareGallery(Integer gid, Integer expireHour){
Response response = Response.generateResponse(); Response response = Response.generateResponse();
Gallery gallery = galleryMapper.selectGalleryByGid(gid); String gid = String.valueOf(GalleryUtil.parseGid(url));
Map<String, String> jsonObject = new HashMap<>(); GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
if(gallery == null){ Gallery gallery;
response.failure("本子不存在"); //已缓存过，直接返回
return response.toJSONString(); if(gidToKey != null) {
gallery = galleryMapper.selectGalleryByGid(Integer.parseInt(gid));
return response.success(objectMapper.valueToTree(gallery)).toJSONString();
} }
File file = new File(TargetPath + gallery.getName() + ".zip"); try {
gidToKey = new GidToKey();
gidToKey.setGid(gid);
gidToKey.setKey(url.split("/")[5].strip());
gallery = GalleryUtil.parse(url, false, null);
ArrayList<ImageKeyCache> imageKeyCaches = GalleryUtil.parseImageKeys(url);
gidToKey.setPages(imageKeyCaches.size());
imageCacheMapper.insertGidToKey(gidToKey);
for (ImageKeyCache imageKeyCache : imageKeyCaches)
imageCacheMapper.insertImageKeyCache(imageKeyCache);
response.success(objectMapper.valueToTree(gallery));
}catch (IOException | ResolutionNotMatchException e){
log.error(e.getMessage());
response.failure("网络波动或其他异常");
}
return response.toJSONString();
}
if(!file.isFile()){ String[] suffixes = {".webp", ".gif"};
response.failure("本子文件不存在"); public void getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException {
return response.toJSONString(); //检查文件夹是否存在
File folder = new File(cachePath + gid);
if(!folder.isDirectory())
folder.mkdirs();
//检查对应图片是否存在，存在则直接返回
for (String suffix : suffixes) {
if(new File(cachePath + gid + "/" + page + suffix).exists()){
FileDownload.export(request, response, cachePath + gid + "/" + page + suffix);
return;
}
} }
ShareFile shareFile = shareFileMapper.selectShareFileByFilePath(file.getAbsolutePath()); //检查该本子缓存是否存在
if(shareFile != null){ GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
jsonObject.put("shareCode", shareFile.getShareCode()); if(gidToKey == null)
jsonObject.put("expireTime", dateTimeFormatter.format(LocalDateTime.ofInstant(shareFile.getExpireTime().toInstant(), ZoneId.systemDefault()))); try {
response.success(new ObjectMapper().valueToTree(jsonObject).toString()); log.error("未缓存gid:" + gid);
return response.toJSONString(); response.sendError(404);
return;
}catch (IOException ignored){
return;
}
String imageUrl = null;
synchronized (this) {
//获取该图片key
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
if (imageKeyCache == null) {
response.sendError(404);
return;
}
//获取图片地址
for (int i = 0; i < 2; i++) {
imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache);
if (imageUrl != null)
break;
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
}
if (imageUrl == null)
try {
log.error("获取图片url失败:gid=" + gid + " page=" + page + " imageKey=" + imageKeyCache.getImgkey());
response.sendError(404);
return;
} catch (IOException ignored) {
return;
}
} }
//下载图片，转格式并返回
String ShareCode = RandomUtil.randomString(8); String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
Calendar expireTime = Calendar.getInstance(); String imagePath = cachePath + gid + "/" + page + suffix;
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
expireTime.add(Calendar.HOUR, expireHour); FileDownload.export(request, response, imagePath);
shareFileMapper.insertShareFile(ShareCode, file.getAbsolutePath(), expireTime.getTime()); if (!suffix.equals(".gif"))
jsonObject.put("shareCode", ShareCode); convertThread.submit(() -> {
jsonObject.put("expireTime", dateTimeFormatter.format(expireTime.getTime().toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime())); ConvertCmd convertCmd = new ConvertCmd(true);
response.success(new ObjectMapper().valueToTree(jsonObject).toString()); IMOperation operation = new IMOperation();
operation.addImage(imagePath);
return response.toJSONString(); operation.format("webp");
operation.addImage(imagePath.replace("jpg", "webp").replace("png", "webp"));
try {
convertCmd.run(operation);
new File(imagePath).delete();
}catch (IOException | IM4JavaException | InterruptedException e){
log.error("文件" + imagePath + "转换失败");
}
});
} }
public String resetUndone(){ public String resetUndone(){
@ -545,12 +570,4 @@ public class GalleryManageService {
return response.toJSONString(); return response.toJSONString();
} }
public static Integer parseGid(String link){
try {
return Integer.parseInt(link.split("/g/")[1].split("/")[0]);
}catch (IndexOutOfBoundsException e){
return null;
}
}
} }

86
src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java
View File

@ -1,6 +1,8 @@
package com.lion.lionwebsite.Util; package com.lion.lionwebsite.Util;
import com.fasterxml.jackson.databind.JsonNode;
import com.lion.lionwebsite.Domain.Gallery; import com.lion.lionwebsite.Domain.Gallery;
import com.lion.lionwebsite.Domain.ImageKeyCache;
import com.lion.lionwebsite.Exception.ResolutionNotMatchException; import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.EntityBuilder;
@ -20,6 +22,9 @@ import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import static com.lion.lionwebsite.Util.CustomUtil.objectMapper;
public class GalleryUtil { public class GalleryUtil {
@ -30,11 +35,7 @@ public class GalleryUtil {
static String JSON = "json"; static String JSON = "json";
public static ArrayList<Gallery> galleriesForDownload; static HashMap<String, String> gid2MpvKey = new HashMap<>();
static {
galleriesForDownload = new ArrayList<>();
}
/** /**
@ -140,6 +141,71 @@ public class GalleryUtil {
return gallery; return gallery;
} }
public static ArrayList<ImageKeyCache> parseImageKeys(String url) throws IOException {
String temp = url.substring(url.indexOf("/g/") + 3);
String mpvUrl = "https://exhentai.org/mpv/" + temp;
String gid = url.split("/")[4];
HashMap<String, String> header = new HashMap<>();
header.put("Referer", url);
String content = requests(mpvUrl, "get", header, null);
Document document = Jsoup.parse(content);
Element script = document.select("body > script").get(1);
String[] scripts = script.html().split("\n");
ArrayList<ImageKeyCache> imageKeyCaches = new ArrayList<>();
AtomicInteger page = new AtomicInteger(1);
gid2MpvKey.put(gid, scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", ""));
JsonNode nodes = objectMapper.readValue(scripts[2].replace("var imagelist = ", ""), JsonNode.class);
nodes.forEach((n) -> {
ImageKeyCache imageKeyCache = new ImageKeyCache();
imageKeyCache.setGid(gid);
imageKeyCache.setImgkey(n.get("k").asText());
imageKeyCache.setPage(page.getAndIncrement());
imageKeyCaches.add(imageKeyCache);
});
return imageKeyCaches;
}
public static String getMpvKey(String url) throws IOException {
String gid = String.valueOf(parseGid(url));
if(!gid2MpvKey.containsKey(gid))
refreshMpvKey(url);
return gid2MpvKey.get(gid);
}
public static void refreshMpvKey(String url) throws IOException {
String temp = url.substring(url.indexOf("/g/") + 3);
String mpvUrl = "https://exhentai.org/mpv/" + temp;
HashMap<String, String> header = new HashMap<>();
header.put("Referer", url);
String content = requests(mpvUrl, "get", header, null);
Document document = Jsoup.parse(content);
Element script = document.select("body > script").get(1);
String[] scripts = script.html().split("\n");
String mpvKey = scripts[1].split("=")[1].replace(";", "").replace("\"", "").replace(" ", "");
gid2MpvKey.put(parseGid(url) + "", mpvKey);
System.out.println("刷新key:" + mpvKey);
}
public static String getImageUrl(String mpvKey, ImageKeyCache imageKeyCache) throws IOException {
String apiUrl = "https://s.exhentai.org/api.php";
HashMap<String, String> header = new HashMap<>();
header.put("Referer", "https://exhentai.org");
HashMap<String, String> body = new HashMap<>();
body.put("gid", imageKeyCache.getGid());
body.put("mpvkey", mpvKey);
body.put("imgkey", imageKeyCache.getImgkey());
body.put("method", "imagedispatch");
body.put("page", "" + imageKeyCache.getPage());
body.put("payload", "json");
System.out.println("开始获取imgurl:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
String result = requests(apiUrl, "post", header, body);
System.out.println("获取imgurl成功:" + imageKeyCache.getGid() + ":" + imageKeyCache.getPage());
JsonNode jsonNode = objectMapper.readTree(result);
if (jsonNode.has("error") && jsonNode.get("error").asText().equals("Key mismatch"))
return null;
return jsonNode.get("i").asText();
}
/** /**
* 验证链接 * 验证链接
@ -192,7 +258,7 @@ public class GalleryUtil {
if((payload = body.remove("payload")).equals(JSON)) { if((payload = body.remove("payload")).equals(JSON)) {
EntityBuilder entityBuilder = EntityBuilder.create(); EntityBuilder entityBuilder = EntityBuilder.create();
entityBuilder.setContentType(ContentType.APPLICATION_JSON); entityBuilder.setContentType(ContentType.APPLICATION_JSON);
entityBuilder.setText(CustomUtil.objectMapper.writeValueAsString(body)); entityBuilder.setText(objectMapper.writeValueAsString(body));
httpPost.setEntity(entityBuilder.build()); httpPost.setEntity(entityBuilder.build());
} else if(payload.equals(FORM_DATA)) { } else if(payload.equals(FORM_DATA)) {
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
@ -221,4 +287,12 @@ public class GalleryUtil {
return stringBuilder.toString(); return stringBuilder.toString();
} }
public static Integer parseGid(String link){
try {
return Integer.parseInt(link.split("/g/")[1].split("/")[0]);
}catch (IndexOutOfBoundsException e){
return null;
}
}
} }

BIN
src/main/resources/cache.db
View File

Binary file not shown.