lion/lionwebsite-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

原子发布图片缓存并关闭下载和打包文件流

Browse Source
This commit is contained in:
root
2026-09-08 09:30:23 +08:00
parent b15eeaf45e
commit 60facae9b5
8 changed files with 302 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java
+31 -66
View File
@@ -7,6 +7,8 @@ import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
import com.lion.lionwebsite.Error.ErrorCode;
import com.lion.lionwebsite.Util.CustomUtil;
import com.lion.lionwebsite.Util.FileDownload;
import com.lion.lionwebsite.Util.ImageFileCache;
import java.nio.file.Path;
import com.lion.lionwebsite.Util.GalleryUtil;
import com.lion.lionwebsite.Util.Response;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -346,75 +348,38 @@ public class GalleryManageService {
return response.toJSONString();
}
String[] suffixes = {".avif", ".gif"};
public Callable<?> getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) {
//检查文件夹是否存在
File folder = new File(cachePath + gid);
if(!folder.isDirectory())
folder.mkdirs();
//检查对应图片是否存在，存在则直接返回
for (String suffix : suffixes) {
if(new File(cachePath + gid + "/" + page + suffix).exists()){
FileDownload.export(request, response, cachePath + gid + "/" + page + suffix);
return null;
}
Path directory = Path.of(cachePath, gid);
String name = String.valueOf(page);
Path cached = ImageFileCache.find(directory, name);
if (cached != null) {
FileDownload.export(request, response, cached.toString());
return null;
}
//检查该图片缓存是否存在
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
if(gidToKey == null)
try {
log.error("未缓存gid:{}", gid);
response.sendError(404);
return null;
}catch (IOException e){
log.warn("sendError 404 failed", e);
return null;
}
return () -> {
if(response.isCommitted()) {
log.info("连接已关闭: gid={} page={}", gid, page);
return null;
}
String imageUrl = null;
//获取该图片key
ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
if (imageKeyCache == null) {
CustomUtil.fourZeroFour(response);
return null;
}
//获取图片地址
for (int i = 0; i < 2; i++) {
imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache);
if (imageUrl != null)
break;
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
}
if (imageUrl == null) {
CustomUtil.fourZeroFour(response);
log.error("获取图片url失败:gid={} page={} imageKey={}", gid, page, imageKeyCache.getImgkey());
return null;
}
//下载图片，转格式并返回
String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
String imagePath = cachePath + gid + "/" + page + suffix;
try {
new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
}catch (Exception e){
log.error("下载图片失败:url{}", imageUrl, e);
CustomUtil.fourZeroFour(response);
return null;
}
if (!suffix.equals(".gif"))
imagePath = GalleryUtil.convertImg(imagePath, suffix);
FileDownload.export(request, response, imagePath);
return null;
if (response.isCommitted()) return null;
try {
Path image = ImageFileCache.get(directory, name, () -> {
GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
ImageKeyCache imageKey = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
if (gidToKey == null || imageKey == null)
throw new IOException("图片索引不存在");
for (int attempt = 0; attempt < 2; attempt++) {
String url = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKey);
if (url != null) return url;
GalleryUtil.refreshMpvKey(gidToKey.toUrl());
}
throw new IOException("无法获取图片地址");
});
FileDownload.export(request, response, image.toString());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if (!response.isCommitted()) response.sendError(503);
} catch (Exception e) {
log.warn("获取在线图片失败: gid={} page={} errorType={}", gid, page, e.getClass().getSimpleName());
if (!response.isCommitted()) response.sendError(404);
}
return null;
};
}
src/main/java/com/lion/lionwebsite/Service/PersonalService.java
+39 -26
View File
@@ -27,6 +27,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
@@ -303,39 +304,51 @@ public class PersonalService{
}
compressThreadPool.submit(() -> {
try(OutputStream bos = new BufferedOutputStream(Files.newOutputStream(Paths.get(finalPath + ".tar***undone")));
TarArchiveOutputStream aos = new TarArchiveOutputStream(bos)) {
aos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); //解除文件名长度限制
Path dirPath = Paths.get(finalPath);
Files.walkFileTree(dirPath, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
TarArchiveEntry entry = new TarArchiveEntry(dir.toFile(), dirPath.relativize(dir).toString());
aos.putArchiveEntry(entry);
aos.closeArchiveEntry();
return super.preVisitDirectory(dir, attrs);
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
TarArchiveEntry entry = new TarArchiveEntry(file.toFile(), dirPath.relativize(file).toString());
aos.putArchiveEntry(entry);
IOUtils.copy(Files.newInputStream(file.toFile().toPath()), aos);
aos.closeArchiveEntry();
return super.visitFile(file, attrs);
}
});
File targetFile = new File(finalPath + ".tar***undone");
log.info("打包成功，重命名:" + targetFile.renameTo(new File(finalPath + ".tar")));
}catch (IOException e){
Path temporary = Paths.get(finalPath + ".tar***undone");
try {
writeTar(Paths.get(finalPath), temporary);
// writeTar closes the archive (including its trailer) before publication.
try {
Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.REPLACE_EXISTING);
}
log.info("打包成功: {}.tar", finalPath);
} catch (IOException e) {
log.error("打包失败", e);
log.info("打包失败，删除文件结果:" + new File(finalPath + ".tar***undone").delete());
} finally {
try { Files.deleteIfExists(temporary); }
catch (IOException e) { log.warn("清理打包临时文件失败", e); }
}
});
response.success("加入队列成功");
return response.toJSONString();
}
static void writeTar(Path directory, Path output) throws IOException {
try (OutputStream stream = new BufferedOutputStream(Files.newOutputStream(output));
TarArchiveOutputStream archive = new TarArchiveOutputStream(stream)) {
archive.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) throws IOException {
if (!path.equals(directory)) {
archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString()));
archive.closeArchiveEntry();
}
return FileVisitResult.CONTINUE;
}
@Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString()));
try (InputStream input = Files.newInputStream(path)) {
IOUtils.copy(input, archive);
}
archive.closeArchiveEntry();
return FileVisitResult.CONTINUE;
}
});
}
}
/**
* 删除文件
* @param path 目标路径
src/main/java/com/lion/lionwebsite/Service/QueryService.java
+12 -16
View File
@@ -3,6 +3,8 @@ package com.lion.lionwebsite.Service;
import com.lion.lionwebsite.Domain.GalleryForQuery;
import com.lion.lionwebsite.Util.FileDownload;
import com.lion.lionwebsite.Util.GalleryUtil;
import com.lion.lionwebsite.Util.ImageFileCache;
import java.nio.file.Path;
import com.lion.lionwebsite.Util.Response;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.ServletOutputStream;
@@ -103,22 +105,16 @@ public class QueryService {
String fileName = path.substring(path.lastIndexOf("/") + 1);
String suffix = fileName.substring(fileName.lastIndexOf("."));
fileName = fileName.substring(0, fileName.lastIndexOf("."));
File image = new File(CachePath, fileName + ".avif");
if(image.isFile()){
FileDownload.export(request, response, image.getAbsolutePath());
return;
}
path = "https://ehgt.org/" + path;
try(ServletOutputStream outputStream = response.getOutputStream()){
new URI(path).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(CachePath + fileName + suffix));
GalleryUtil.convertImg(CachePath + fileName + suffix, suffix);
FileInputStream inputStream = new FileInputStream(image.getAbsoluteFile()); //如果放到括号里，会导致图片未创建时创建文件流失败报错
outputStream.write(inputStream.readAllBytes());
inputStream.close();
}catch (IOException | URISyntaxException e){
log.error("获取缩略图失败", e);
String sourceUrl = "https://ehgt.org/" + path;
try {
Path image = ImageFileCache.get(Path.of(CachePath), fileName, () -> sourceUrl);
FileDownload.export(request, response, image.toString());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
response.setStatus(503);
} catch (Exception e) {
log.warn("获取缩略图失败: errorType={}", e.getClass().getSimpleName());
if (!response.isCommitted()) response.setStatus(404);
}
}
}
src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java
+36 -8
View File
@@ -5,6 +5,9 @@ import com.lion.lionwebsite.Domain.Gallery;
import com.lion.lionwebsite.Domain.ImageKeyCache;
import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import java.nio.file.*;
import java.util.concurrent.TimeUnit;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
@@ -44,7 +47,9 @@ public class GalleryUtil {
static ConcurrentHashMap<String, String> gid2MpvKey = new ConcurrentHashMap<>();
/** Reusable HTTP client —不要每次请求新建 */
private static final CloseableHttpClient httpClient = HttpClients.createDefault();
private static final CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(5_000)
.setConnectionRequestTimeout(5_000).setSocketTimeout(15_000).build()).build();
/** E-Hentai Cookie, injected from application.yaml via CustomBean */
private static String ehentaiCookie = "";
@@ -240,15 +245,38 @@ public class GalleryUtil {
}
public static String convertImg(String imagePath, String suffix){
Runtime rt = Runtime.getRuntime();
Path source = Path.of(imagePath);
Path target = source.resolveSibling(source.getFileName().toString().replaceFirst("\\Q" + suffix + "\\E$", ".avif"));
if (source.equals(target)) return imagePath;
Path temporary = null;
Process process = null;
try {
Process exec = rt.exec(new String[]{"convert", imagePath, imagePath.replace(suffix, ".avif")});
exec.waitFor();
boolean ignored = new File(imagePath).delete();
return imagePath.replace(suffix, ".avif");
} catch (IOException| InterruptedException e) {
log.error("文件{}转换失败", imagePath, e);
temporary = Files.createTempFile(target.toAbsolutePath().getParent(), ".convert-", ".avif");
process = new ProcessBuilder("convert", source.toString(), temporary.toString())
.redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start();
if (!process.waitFor(60, TimeUnit.SECONDS))
throw new IOException("图片转换超时");
if (process.exitValue() != 0 || Files.size(temporary) == 0)
throw new IOException("图片转换失败");
try {
Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);
}
Files.deleteIfExists(source);
return target.toString();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
} catch (IOException e) {
log.warn("文件{}转换失败", imagePath, e);
return null;
} finally {
if (process != null && process.isAlive()) process.destroyForcibly();
if (temporary != null) {
try { Files.deleteIfExists(temporary); }
catch (IOException e) { log.warn("清理图片转换临时文件失败", e); }
}
}
}
src/main/java/com/lion/lionwebsite/Util/ImageFileCache.java
+67
View File
@@ -0,0 +1,67 @@
package com.lion.lionwebsite.Util;
import java.io.*;
import java.net.URI;
import java.net.URLConnection;
import java.nio.file.*;
import java.util.Locale;
import java.util.concurrent.Callable;
/** Downloads into private temporary files and publishes only completed images. */
public final class ImageFileCache {
private static final SingleFlight<Path, Path> downloads = new SingleFlight<>();
private ImageFileCache() { }
public static Path find(Path directory, String name) {
for (String suffix : new String[]{".avif", ".gif"}) {
Path path = directory.resolve(name + suffix);
if (Files.isRegularFile(path) && path.toFile().length() > 0)
return path;
}
return null;
}
public static Path get(Path directory, String name, Callable<String> sourceUrl) throws Exception {
Path key = directory.resolve(name).toAbsolutePath().normalize();
return downloads.run(key, () -> {
Path cached = find(directory, name);
if (cached != null) return cached;
Files.createDirectories(directory);
String url = sourceUrl.call();
if (url == null) throw new IOException("图片地址不存在");
URI source = new URI(url);
boolean gif = source.getPath().toLowerCase(Locale.ROOT).endsWith(".gif");
String suffix = gif ? ".gif" : ".img";
Path temporary = Files.createTempFile(directory, ".download-", suffix);
Path converted = null;
try {
URLConnection connection = source.toURL().openConnection();
connection.setConnectTimeout(5_000);
connection.setReadTimeout(15_000);
try (InputStream input = connection.getInputStream();
OutputStream output = Files.newOutputStream(temporary)) {
input.transferTo(output);
}
if (Files.size(temporary) == 0) throw new IOException("图片内容为空");
if (gif) {
converted = temporary;
} else {
String result = GalleryUtil.convertImg(temporary.toString(), suffix);
if (result == null) throw new IOException("图片转换失败");
converted = Path.of(result);
}
Path target = directory.resolve(name + (gif ? ".gif" : ".avif"));
try {
Files.move(converted, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(converted, target, StandardCopyOption.REPLACE_EXISTING);
}
return target;
} finally {
Files.deleteIfExists(temporary);
if (converted != null && !converted.equals(temporary)) Files.deleteIfExists(converted);
}
});
}
}
src/main/java/com/lion/lionwebsite/Util/SingleFlight.java
+30
View File
@@ -0,0 +1,30 @@
package com.lion.lionwebsite.Util;
import java.util.concurrent.*;
/** Concurrent callers for the same key share one in-flight operation, including its failure. */
public final class SingleFlight<K, V> {
private final ConcurrentHashMap<K, CompletableFuture<V>> running = new ConcurrentHashMap<>();
public V run(K key, Callable<V> operation) throws Exception {
CompletableFuture<V> mine = new CompletableFuture<>();
CompletableFuture<V> existing = running.putIfAbsent(key, mine);
if (existing != null) {
try { return existing.get(); }
catch (ExecutionException e) {
if (e.getCause() instanceof Exception cause) throw cause;
throw new IllegalStateException(e.getCause());
}
}
try {
V result = operation.call();
mine.complete(result);
return result;
} catch (Exception | Error e) {
mine.completeExceptionally(e);
throw e;
} finally {
running.remove(key, mine);
}
}
}
src/test/java/com/lion/lionwebsite/Service/PersonalArchiveTest.java
+27
View File
@@ -0,0 +1,27 @@
package com.lion.lionwebsite.Service;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.*;
import java.util.HashMap;
import static org.junit.jupiter.api.Assertions.*;
class PersonalArchiveTest {
@Test void archiveContainsCompleteFilesAndCanBeOpenedImmediately(@TempDir Path root) throws Exception {
Path source = Files.createDirectory(root.resolve("source"));
Files.createDirectory(source.resolve("nested"));
for (int i = 0; i < 100; i++) Files.writeString(source.resolve("nested/" + i + ".txt"), "content-" + i);
Path archive = root.resolve("result.tar");
PersonalService.writeTar(source, archive);
var contents = new HashMap<String, String>();
try (var input = new TarArchiveInputStream(Files.newInputStream(archive))) {
org.apache.commons.compress.archivers.tar.TarArchiveEntry entry;
while ((entry = input.getNextTarEntry()) != null) {
if (!entry.isDirectory()) contents.put(entry.getName(), new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8));
}
}
assertEquals(100, contents.size());
for (int i = 0; i < 100; i++) assertEquals("content-" + i, contents.get("nested/" + i + ".txt"));
}
}
src/test/java/com/lion/lionwebsite/Util/ImageFileCacheTest.java
+60
View File
@@ -0,0 +1,60 @@
package com.lion.lionwebsite.Util;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.time.Duration;
import static org.junit.jupiter.api.Assertions.*;
class ImageFileCacheTest {
@Test void publishesClosedGifAndReusesIt(@TempDir Path root) throws Exception {
Path source = root.resolve("source.gif");
byte[] content = "GIF89a test image".getBytes(java.nio.charset.StandardCharsets.UTF_8);
Files.write(source, content);
Path cache = root.resolve("cache");
Path result = ImageFileCache.get(cache, "1", () -> source.toUri().toString());
assertArrayEquals(content, Files.readAllBytes(result));
assertEquals(result, ImageFileCache.get(cache, "1", () -> { throw new AssertionError("cache must avoid source access"); }));
try (var files = Files.list(cache)) { assertEquals(1, files.count()); }
}
@Test void failedDownloadLeavesNoPublishedOrTemporaryFile(@TempDir Path root) throws Exception {
Path cache = root.resolve("cache");
assertThrows(Exception.class, () -> ImageFileCache.get(cache, "1", () -> root.resolve("missing.gif").toUri().toString()));
assertNull(ImageFileCache.find(cache, "1"));
try (var files = Files.list(cache)) { assertEquals(0, files.count()); }
}
@Test void concurrentCallersShareTheSameOperation() throws Exception {
SingleFlight<String, String> flight = new SingleFlight<>();
CountDownLatch entered = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
AtomicInteger calls = new AtomicInteger();
AtomicReference<Thread> followerThread = new AtomicReference<>();
ExecutorService workers = Executors.newFixedThreadPool(2);
try {
Future<String> leader = workers.submit(() -> flight.run("image", () -> {
calls.incrementAndGet(); entered.countDown();
if (!release.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("test timed out");
return "completed";
}));
assertTrue(entered.await(2, TimeUnit.SECONDS));
Future<String> follower = workers.submit(() -> {
followerThread.set(Thread.currentThread());
return flight.run("image", () -> { calls.incrementAndGet(); return "duplicate"; });
});
assertTimeoutPreemptively(Duration.ofSeconds(2), () -> {
while (followerThread.get() == null || followerThread.get().getState() != Thread.State.WAITING) {
assertFalse(follower.isDone(), "follower must wait for the first operation");
Thread.sleep(1);
}
});
release.countDown();
assertEquals("completed", leader.get(2, TimeUnit.SECONDS));
assertEquals("completed", follower.get(2, TimeUnit.SECONDS));
assertEquals(1, calls.get());
} finally { release.countDown(); workers.shutdownNow(); }
}
}