原子发布图片缓存并关闭下载和打包文件流

src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java

@@ -7,6 +7,8 @@ import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
 import com.lion.lionwebsite.Error.ErrorCode;
 import com.lion.lionwebsite.Util.CustomUtil;
 import com.lion.lionwebsite.Util.FileDownload;
+import com.lion.lionwebsite.Util.ImageFileCache;
+import java.nio.file.Path;
 import com.lion.lionwebsite.Util.GalleryUtil;
 import com.lion.lionwebsite.Util.Response;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -346,74 +348,37 @@ public class GalleryManageService {
         return  response.toJSONString();
     }
 
-    String[] suffixes = {".avif", ".gif"};
     public Callable<?> getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) {
-        //检查文件夹是否存在
+        Path directory = Path.of(cachePath, gid);
-        File folder = new File(cachePath + gid);
+        String name = String.valueOf(page);
-        if(!folder.isDirectory())
+        Path cached = ImageFileCache.find(directory, name);
-            folder.mkdirs();
+        if (cached != null) {
-
+            FileDownload.export(request, response, cached.toString());
-        //检查对应图片是否存在，存在则直接返回
-        for (String suffix : suffixes) {
-            if(new File(cachePath + gid + "/" + page + suffix).exists()){
-                FileDownload.export(request, response, cachePath + gid + "/" + page + suffix);
             return null;
         }
-        }
-
-        //检查该图片缓存是否存在
-        GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
-        if(gidToKey == null)
-            try {
-                log.error("未缓存gid:{}", gid);
-                response.sendError(404);
-                return null;
-            }catch (IOException e){
-                log.warn("sendError 404 failed", e);
-                return null;
-            }
-
         return () -> {
-                if(response.isCommitted()) {
+            if (response.isCommitted()) return null;
-                    log.info("连接已关闭: gid={} page={}", gid, page);
+            try {
-                    return null;
+                Path image = ImageFileCache.get(directory, name, () -> {
-                }
+                    GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid);
-
+                    ImageKeyCache imageKey = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
-                String imageUrl = null;
+                    if (gidToKey == null || imageKey == null)
-                //获取该图片key
+                        throw new IOException("图片索引不存在");
-                ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page);
+                    for (int attempt = 0; attempt < 2; attempt++) {
-                if (imageKeyCache == null) {
+                        String url = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKey);
-                    CustomUtil.fourZeroFour(response);
+                        if (url != null) return url;
-                    return null;
-                }
-
-                //获取图片地址
-                for (int i = 0; i < 2; i++) {
-                    imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache);
-                    if (imageUrl != null)
-                        break;
                         GalleryUtil.refreshMpvKey(gidToKey.toUrl());
                     }
-
+                    throw new IOException("无法获取图片地址");
-                if (imageUrl == null) {
+                });
-                    CustomUtil.fourZeroFour(response);
+                FileDownload.export(request, response, image.toString());
-                    log.error("获取图片url失败:gid={} page={} imageKey={}", gid, page, imageKeyCache.getImgkey());
+            } catch (InterruptedException e) {
-                    return null;
+                Thread.currentThread().interrupt();
+                if (!response.isCommitted()) response.sendError(503);
+            } catch (Exception e) {
+                log.warn("获取在线图片失败: gid={} page={} errorType={}", gid, page, e.getClass().getSimpleName());
+                if (!response.isCommitted()) response.sendError(404);
             }
-
-                //下载图片，转格式并返回
-                String suffix = imageUrl.substring(imageUrl.lastIndexOf("."));
-                String imagePath = cachePath + gid + "/" + page + suffix;
-                try {
-                    new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath));
-                }catch (Exception e){
-                    log.error("下载图片失败:url{}", imageUrl, e);
-                    CustomUtil.fourZeroFour(response);
-                    return null;
-                }
-                if (!suffix.equals(".gif"))
-                    imagePath = GalleryUtil.convertImg(imagePath, suffix);
-                FileDownload.export(request, response, imagePath);
             return null;
         };
     }

src/main/java/com/lion/lionwebsite/Service/PersonalService.java

@@ -27,6 +27,7 @@ import org.springframework.web.multipart.MultipartFile;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
@@ -303,39 +304,51 @@ public class PersonalService{
         }
 
         compressThreadPool.submit(() -> {
-            try(OutputStream bos = new BufferedOutputStream(Files.newOutputStream(Paths.get(finalPath + ".tar***undone")));
+            Path temporary = Paths.get(finalPath + ".tar***undone");
-                TarArchiveOutputStream aos = new TarArchiveOutputStream(bos)) {
+            try {
-                aos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);   //解除文件名长度限制
+                writeTar(Paths.get(finalPath), temporary);
-                Path dirPath = Paths.get(finalPath);
+                // writeTar closes the archive (including its trailer) before publication.
-                Files.walkFileTree(dirPath, new SimpleFileVisitor<>() {
+                try {
-                    @Override
+                    Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
-                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                } catch (AtomicMoveNotSupportedException e) {
-                        TarArchiveEntry entry = new TarArchiveEntry(dir.toFile(), dirPath.relativize(dir).toString());
+                    Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.REPLACE_EXISTING);
-                        aos.putArchiveEntry(entry);
-                        aos.closeArchiveEntry();
-                        return super.preVisitDirectory(dir, attrs);
                 }
-
+                log.info("打包成功: {}.tar", finalPath);
-                    @Override
+            } catch (IOException e) {
-                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                        TarArchiveEntry entry = new TarArchiveEntry(file.toFile(), dirPath.relativize(file).toString());
-                        aos.putArchiveEntry(entry);
-                        IOUtils.copy(Files.newInputStream(file.toFile().toPath()), aos);
-                        aos.closeArchiveEntry();
-                        return super.visitFile(file, attrs);
-                    }
-                });
-                File targetFile = new File(finalPath + ".tar***undone");
-                log.info("打包成功，重命名:" + targetFile.renameTo(new File(finalPath + ".tar")));
-            }catch (IOException e){
                 log.error("打包失败", e);
-                log.info("打包失败，删除文件结果:" + new File(finalPath + ".tar***undone").delete());
+            } finally {
+                try { Files.deleteIfExists(temporary); }
+                catch (IOException e) { log.warn("清理打包临时文件失败", e); }
             }
         });
         response.success("加入队列成功");
         return response.toJSONString();
     }
 
+    static void writeTar(Path directory, Path output) throws IOException {
+        try (OutputStream stream = new BufferedOutputStream(Files.newOutputStream(output));
+             TarArchiveOutputStream archive = new TarArchiveOutputStream(stream)) {
+            archive.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+            Files.walkFileTree(directory, new SimpleFileVisitor<>() {
+                @Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) throws IOException {
+                    if (!path.equals(directory)) {
+                        archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString()));
+                        archive.closeArchiveEntry();
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+                @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+                    archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString()));
+                    try (InputStream input = Files.newInputStream(path)) {
+                        IOUtils.copy(input, archive);
+                    }
+                    archive.closeArchiveEntry();
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        }
+    }
+
     /**
      * 删除文件
      * @param path  目标路径

src/main/java/com/lion/lionwebsite/Service/QueryService.java

@@ -3,6 +3,8 @@ package com.lion.lionwebsite.Service;
 import com.lion.lionwebsite.Domain.GalleryForQuery;
 import com.lion.lionwebsite.Util.FileDownload;
 import com.lion.lionwebsite.Util.GalleryUtil;
+import com.lion.lionwebsite.Util.ImageFileCache;
+import java.nio.file.Path;
 import com.lion.lionwebsite.Util.Response;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.servlet.ServletOutputStream;
@@ -103,22 +105,16 @@ public class QueryService {
         String fileName = path.substring(path.lastIndexOf("/") + 1);
         String suffix = fileName.substring(fileName.lastIndexOf("."));
         fileName = fileName.substring(0, fileName.lastIndexOf("."));
-        File image = new File(CachePath, fileName + ".avif");
+        String sourceUrl = "https://ehgt.org/" + path;
-
+        try {
-        if(image.isFile()){
+            Path image = ImageFileCache.get(Path.of(CachePath), fileName, () -> sourceUrl);
-            FileDownload.export(request, response, image.getAbsolutePath());
+            FileDownload.export(request, response, image.toString());
-            return;
+        } catch (InterruptedException e) {
-        }
+            Thread.currentThread().interrupt();
-
+            response.setStatus(503);
-        path = "https://ehgt.org/" + path;
+        } catch (Exception e) {
-        try(ServletOutputStream outputStream = response.getOutputStream()){
+            log.warn("获取缩略图失败: errorType={}", e.getClass().getSimpleName());
-            new URI(path).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(CachePath + fileName + suffix));
+            if (!response.isCommitted()) response.setStatus(404);
-            GalleryUtil.convertImg(CachePath + fileName + suffix, suffix);
-            FileInputStream inputStream = new FileInputStream(image.getAbsoluteFile());    //如果放到括号里，会导致图片未创建时创建文件流失败报错
-            outputStream.write(inputStream.readAllBytes());
-            inputStream.close();
-        }catch (IOException | URISyntaxException e){
-            log.error("获取缩略图失败", e);
         }
     }
 }

src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java

@@ -5,6 +5,9 @@ import com.lion.lionwebsite.Domain.Gallery;
 import com.lion.lionwebsite.Domain.ImageKeyCache;
 import com.lion.lionwebsite.Exception.ResolutionNotMatchException;
 import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import java.nio.file.*;
+import java.util.concurrent.TimeUnit;
 import org.apache.http.client.entity.EntityBuilder;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -44,7 +47,9 @@ public class GalleryUtil {
     static ConcurrentHashMap<String, String> gid2MpvKey = new ConcurrentHashMap<>();
 
     /** Reusable HTTP client —不要每次请求新建 */
-    private static final CloseableHttpClient httpClient = HttpClients.createDefault();
+    private static final CloseableHttpClient httpClient = HttpClients.custom()
+            .setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(5_000)
+                    .setConnectionRequestTimeout(5_000).setSocketTimeout(15_000).build()).build();
 
     /** E-Hentai Cookie, injected from application.yaml via CustomBean */
     private static String ehentaiCookie = "";
@@ -240,15 +245,38 @@ public class GalleryUtil {
     }
 
     public static String convertImg(String imagePath, String suffix){
-        Runtime rt = Runtime.getRuntime();
+        Path source = Path.of(imagePath);
+        Path target = source.resolveSibling(source.getFileName().toString().replaceFirst("\\Q" + suffix + "\\E$", ".avif"));
+        if (source.equals(target)) return imagePath;
+        Path temporary = null;
+        Process process = null;
         try {
-            Process exec = rt.exec(new String[]{"convert", imagePath, imagePath.replace(suffix, ".avif")});
+            temporary = Files.createTempFile(target.toAbsolutePath().getParent(), ".convert-", ".avif");
-            exec.waitFor();
+            process = new ProcessBuilder("convert", source.toString(), temporary.toString())
-            boolean ignored = new File(imagePath).delete();
+                    .redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start();
-            return imagePath.replace(suffix, ".avif");
+            if (!process.waitFor(60, TimeUnit.SECONDS))
-        } catch (IOException| InterruptedException e) {
+                throw new IOException("图片转换超时");
-            log.error("文件{}转换失败", imagePath, e);
+            if (process.exitValue() != 0 || Files.size(temporary) == 0)
+                throw new IOException("图片转换失败");
+            try {
+                Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+            } catch (AtomicMoveNotSupportedException e) {
+                Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);
+            }
+            Files.deleteIfExists(source);
+            return target.toString();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             return null;
+        } catch (IOException e) {
+            log.warn("文件{}转换失败", imagePath, e);
+            return null;
+        } finally {
+            if (process != null && process.isAlive()) process.destroyForcibly();
+            if (temporary != null) {
+                try { Files.deleteIfExists(temporary); }
+                catch (IOException e) { log.warn("清理图片转换临时文件失败", e); }
+            }
         }
     }
 

src/main/java/com/lion/lionwebsite/Util/ImageFileCache.java

@@ -0,0 +1,67 @@
+package com.lion.lionwebsite.Util;
+
+import java.io.*;
+import java.net.URI;
+import java.net.URLConnection;
+import java.nio.file.*;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+
+/** Downloads into private temporary files and publishes only completed images. */
+public final class ImageFileCache {
+    private static final SingleFlight<Path, Path> downloads = new SingleFlight<>();
+
+    private ImageFileCache() { }
+
+    public static Path find(Path directory, String name) {
+        for (String suffix : new String[]{".avif", ".gif"}) {
+            Path path = directory.resolve(name + suffix);
+            if (Files.isRegularFile(path) && path.toFile().length() > 0)
+                return path;
+        }
+        return null;
+    }
+
+    public static Path get(Path directory, String name, Callable<String> sourceUrl) throws Exception {
+        Path key = directory.resolve(name).toAbsolutePath().normalize();
+        return downloads.run(key, () -> {
+            Path cached = find(directory, name);
+            if (cached != null) return cached;
+            Files.createDirectories(directory);
+            String url = sourceUrl.call();
+            if (url == null) throw new IOException("图片地址不存在");
+            URI source = new URI(url);
+            boolean gif = source.getPath().toLowerCase(Locale.ROOT).endsWith(".gif");
+            String suffix = gif ? ".gif" : ".img";
+            Path temporary = Files.createTempFile(directory, ".download-", suffix);
+            Path converted = null;
+            try {
+                URLConnection connection = source.toURL().openConnection();
+                connection.setConnectTimeout(5_000);
+                connection.setReadTimeout(15_000);
+                try (InputStream input = connection.getInputStream();
+                     OutputStream output = Files.newOutputStream(temporary)) {
+                    input.transferTo(output);
+                }
+                if (Files.size(temporary) == 0) throw new IOException("图片内容为空");
+                if (gif) {
+                    converted = temporary;
+                } else {
+                    String result = GalleryUtil.convertImg(temporary.toString(), suffix);
+                    if (result == null) throw new IOException("图片转换失败");
+                    converted = Path.of(result);
+                }
+                Path target = directory.resolve(name + (gif ? ".gif" : ".avif"));
+                try {
+                    Files.move(converted, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+                } catch (AtomicMoveNotSupportedException e) {
+                    Files.move(converted, target, StandardCopyOption.REPLACE_EXISTING);
+                }
+                return target;
+            } finally {
+                Files.deleteIfExists(temporary);
+                if (converted != null && !converted.equals(temporary)) Files.deleteIfExists(converted);
+            }
+        });
+    }
+}

src/main/java/com/lion/lionwebsite/Util/SingleFlight.java

@@ -0,0 +1,30 @@
+package com.lion.lionwebsite.Util;
+
+import java.util.concurrent.*;
+
+/** Concurrent callers for the same key share one in-flight operation, including its failure. */
+public final class SingleFlight<K, V> {
+    private final ConcurrentHashMap<K, CompletableFuture<V>> running = new ConcurrentHashMap<>();
+
+    public V run(K key, Callable<V> operation) throws Exception {
+        CompletableFuture<V> mine = new CompletableFuture<>();
+        CompletableFuture<V> existing = running.putIfAbsent(key, mine);
+        if (existing != null) {
+            try { return existing.get(); }
+            catch (ExecutionException e) {
+                if (e.getCause() instanceof Exception cause) throw cause;
+                throw new IllegalStateException(e.getCause());
+            }
+        }
+        try {
+            V result = operation.call();
+            mine.complete(result);
+            return result;
+        } catch (Exception | Error e) {
+            mine.completeExceptionally(e);
+            throw e;
+        } finally {
+            running.remove(key, mine);
+        }
+    }
+}

src/test/java/com/lion/lionwebsite/Service/PersonalArchiveTest.java

@@ -0,0 +1,27 @@
+package com.lion.lionwebsite.Service;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import java.nio.file.*;
+import java.util.HashMap;
+import static org.junit.jupiter.api.Assertions.*;
+
+class PersonalArchiveTest {
+    @Test void archiveContainsCompleteFilesAndCanBeOpenedImmediately(@TempDir Path root) throws Exception {
+        Path source = Files.createDirectory(root.resolve("source"));
+        Files.createDirectory(source.resolve("nested"));
+        for (int i = 0; i < 100; i++) Files.writeString(source.resolve("nested/" + i + ".txt"), "content-" + i);
+        Path archive = root.resolve("result.tar");
+        PersonalService.writeTar(source, archive);
+        var contents = new HashMap<String, String>();
+        try (var input = new TarArchiveInputStream(Files.newInputStream(archive))) {
+            org.apache.commons.compress.archivers.tar.TarArchiveEntry entry;
+            while ((entry = input.getNextTarEntry()) != null) {
+                if (!entry.isDirectory()) contents.put(entry.getName(), new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8));
+            }
+        }
+        assertEquals(100, contents.size());
+        for (int i = 0; i < 100; i++) assertEquals("content-" + i, contents.get("nested/" + i + ".txt"));
+    }
+}

src/test/java/com/lion/lionwebsite/Util/ImageFileCacheTest.java

@@ -0,0 +1,60 @@
+package com.lion.lionwebsite.Util;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import java.nio.file.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.time.Duration;
+import static org.junit.jupiter.api.Assertions.*;
+
+class ImageFileCacheTest {
+    @Test void publishesClosedGifAndReusesIt(@TempDir Path root) throws Exception {
+        Path source = root.resolve("source.gif");
+        byte[] content = "GIF89a test image".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        Files.write(source, content);
+        Path cache = root.resolve("cache");
+        Path result = ImageFileCache.get(cache, "1", () -> source.toUri().toString());
+        assertArrayEquals(content, Files.readAllBytes(result));
+        assertEquals(result, ImageFileCache.get(cache, "1", () -> { throw new AssertionError("cache must avoid source access"); }));
+        try (var files = Files.list(cache)) { assertEquals(1, files.count()); }
+    }
+
+    @Test void failedDownloadLeavesNoPublishedOrTemporaryFile(@TempDir Path root) throws Exception {
+        Path cache = root.resolve("cache");
+        assertThrows(Exception.class, () -> ImageFileCache.get(cache, "1", () -> root.resolve("missing.gif").toUri().toString()));
+        assertNull(ImageFileCache.find(cache, "1"));
+        try (var files = Files.list(cache)) { assertEquals(0, files.count()); }
+    }
+
+    @Test void concurrentCallersShareTheSameOperation() throws Exception {
+        SingleFlight<String, String> flight = new SingleFlight<>();
+        CountDownLatch entered = new CountDownLatch(1);
+        CountDownLatch release = new CountDownLatch(1);
+        AtomicInteger calls = new AtomicInteger();
+        AtomicReference<Thread> followerThread = new AtomicReference<>();
+        ExecutorService workers = Executors.newFixedThreadPool(2);
+        try {
+            Future<String> leader = workers.submit(() -> flight.run("image", () -> {
+                calls.incrementAndGet(); entered.countDown();
+                if (!release.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("test timed out");
+                return "completed";
+            }));
+            assertTrue(entered.await(2, TimeUnit.SECONDS));
+            Future<String> follower = workers.submit(() -> {
+                followerThread.set(Thread.currentThread());
+                return flight.run("image", () -> { calls.incrementAndGet(); return "duplicate"; });
+            });
+            assertTimeoutPreemptively(Duration.ofSeconds(2), () -> {
+                while (followerThread.get() == null || followerThread.get().getState() != Thread.State.WAITING) {
+                    assertFalse(follower.isDone(), "follower must wait for the first operation");
+                    Thread.sleep(1);
+                }
+            });
+            release.countDown();
+            assertEquals("completed", leader.get(2, TimeUnit.SECONDS));
+            assertEquals("completed", follower.get(2, TimeUnit.SECONDS));
+            assertEquals(1, calls.get());
+        } finally { release.countDown(); workers.shutdownNow(); }
+    }
+}