diff --git a/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java b/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java index 06e39f0..02ce371 100644 --- a/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java +++ b/src/main/java/com/lion/lionwebsite/Service/GalleryManageService.java @@ -7,6 +7,8 @@ import com.lion.lionwebsite.Exception.ResolutionNotMatchException; import com.lion.lionwebsite.Error.ErrorCode; import com.lion.lionwebsite.Util.CustomUtil; import com.lion.lionwebsite.Util.FileDownload; +import com.lion.lionwebsite.Util.ImageFileCache; +import java.nio.file.Path; import com.lion.lionwebsite.Util.GalleryUtil; import com.lion.lionwebsite.Util.Response; import com.fasterxml.jackson.databind.ObjectMapper; @@ -346,75 +348,38 @@ public class GalleryManageService { return response.toJSONString(); } - String[] suffixes = {".avif", ".gif"}; public Callable getCachedImage(String gid, Integer page, HttpServletRequest request, HttpServletResponse response) { - //检查文件夹是否存在 - File folder = new File(cachePath + gid); - if(!folder.isDirectory()) - folder.mkdirs(); - - //检查对应图片是否存在,存在则直接返回 - for (String suffix : suffixes) { - if(new File(cachePath + gid + "/" + page + suffix).exists()){ - FileDownload.export(request, response, cachePath + gid + "/" + page + suffix); - return null; - } + Path directory = Path.of(cachePath, gid); + String name = String.valueOf(page); + Path cached = ImageFileCache.find(directory, name); + if (cached != null) { + FileDownload.export(request, response, cached.toString()); + return null; } - - //检查该图片缓存是否存在 - GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid); - if(gidToKey == null) - try { - log.error("未缓存gid:{}", gid); - response.sendError(404); - return null; - }catch (IOException e){ - log.warn("sendError 404 failed", e); - return null; - } - return () -> { - if(response.isCommitted()) { - log.info("连接已关闭: gid={} page={}", gid, page); - return null; - } - - String imageUrl = null; - //获取该图片key - ImageKeyCache imageKeyCache = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page); - if (imageKeyCache == null) { - CustomUtil.fourZeroFour(response); - return null; - } - - //获取图片地址 - for (int i = 0; i < 2; i++) { - imageUrl = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKeyCache); - if (imageUrl != null) - break; - GalleryUtil.refreshMpvKey(gidToKey.toUrl()); - } - - if (imageUrl == null) { - CustomUtil.fourZeroFour(response); - log.error("获取图片url失败:gid={} page={} imageKey={}", gid, page, imageKeyCache.getImgkey()); - return null; - } - - //下载图片,转格式并返回 - String suffix = imageUrl.substring(imageUrl.lastIndexOf(".")); - String imagePath = cachePath + gid + "/" + page + suffix; - try { - new URI(imageUrl).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(imagePath)); - }catch (Exception e){ - log.error("下载图片失败:url{}", imageUrl, e); - CustomUtil.fourZeroFour(response); - return null; - } - if (!suffix.equals(".gif")) - imagePath = GalleryUtil.convertImg(imagePath, suffix); - FileDownload.export(request, response, imagePath); - return null; + if (response.isCommitted()) return null; + try { + Path image = ImageFileCache.get(directory, name, () -> { + GidToKey gidToKey = imageCacheMapper.selectKeyByGid(gid); + ImageKeyCache imageKey = imageCacheMapper.selectImageKeyCacheByGidAndPage(gid, page); + if (gidToKey == null || imageKey == null) + throw new IOException("图片索引不存在"); + for (int attempt = 0; attempt < 2; attempt++) { + String url = GalleryUtil.getImageUrl(getMpvKey(gidToKey.toUrl()), imageKey); + if (url != null) return url; + GalleryUtil.refreshMpvKey(gidToKey.toUrl()); + } + throw new IOException("无法获取图片地址"); + }); + FileDownload.export(request, response, image.toString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (!response.isCommitted()) response.sendError(503); + } catch (Exception e) { + log.warn("获取在线图片失败: gid={} page={} errorType={}", gid, page, e.getClass().getSimpleName()); + if (!response.isCommitted()) response.sendError(404); + } + return null; }; } diff --git a/src/main/java/com/lion/lionwebsite/Service/PersonalService.java b/src/main/java/com/lion/lionwebsite/Service/PersonalService.java index f5bf81a..e8217a2 100644 --- a/src/main/java/com/lion/lionwebsite/Service/PersonalService.java +++ b/src/main/java/com/lion/lionwebsite/Service/PersonalService.java @@ -27,6 +27,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -303,39 +304,51 @@ public class PersonalService{ } compressThreadPool.submit(() -> { - try(OutputStream bos = new BufferedOutputStream(Files.newOutputStream(Paths.get(finalPath + ".tar***undone"))); - TarArchiveOutputStream aos = new TarArchiveOutputStream(bos)) { - aos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); //解除文件名长度限制 - Path dirPath = Paths.get(finalPath); - Files.walkFileTree(dirPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - TarArchiveEntry entry = new TarArchiveEntry(dir.toFile(), dirPath.relativize(dir).toString()); - aos.putArchiveEntry(entry); - aos.closeArchiveEntry(); - return super.preVisitDirectory(dir, attrs); - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - TarArchiveEntry entry = new TarArchiveEntry(file.toFile(), dirPath.relativize(file).toString()); - aos.putArchiveEntry(entry); - IOUtils.copy(Files.newInputStream(file.toFile().toPath()), aos); - aos.closeArchiveEntry(); - return super.visitFile(file, attrs); - } - }); - File targetFile = new File(finalPath + ".tar***undone"); - log.info("打包成功,重命名:" + targetFile.renameTo(new File(finalPath + ".tar"))); - }catch (IOException e){ + Path temporary = Paths.get(finalPath + ".tar***undone"); + try { + writeTar(Paths.get(finalPath), temporary); + // writeTar closes the archive (including its trailer) before publication. + try { + Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporary, Paths.get(finalPath + ".tar"), StandardCopyOption.REPLACE_EXISTING); + } + log.info("打包成功: {}.tar", finalPath); + } catch (IOException e) { log.error("打包失败", e); - log.info("打包失败,删除文件结果:" + new File(finalPath + ".tar***undone").delete()); + } finally { + try { Files.deleteIfExists(temporary); } + catch (IOException e) { log.warn("清理打包临时文件失败", e); } } }); response.success("加入队列成功"); return response.toJSONString(); } + static void writeTar(Path directory, Path output) throws IOException { + try (OutputStream stream = new BufferedOutputStream(Files.newOutputStream(output)); + TarArchiveOutputStream archive = new TarArchiveOutputStream(stream)) { + archive.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) throws IOException { + if (!path.equals(directory)) { + archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString())); + archive.closeArchiveEntry(); + } + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + archive.putArchiveEntry(new TarArchiveEntry(path.toFile(), directory.relativize(path).toString())); + try (InputStream input = Files.newInputStream(path)) { + IOUtils.copy(input, archive); + } + archive.closeArchiveEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + } + /** * 删除文件 * @param path 目标路径 diff --git a/src/main/java/com/lion/lionwebsite/Service/QueryService.java b/src/main/java/com/lion/lionwebsite/Service/QueryService.java index 71efd8c..c02ae6f 100644 --- a/src/main/java/com/lion/lionwebsite/Service/QueryService.java +++ b/src/main/java/com/lion/lionwebsite/Service/QueryService.java @@ -3,6 +3,8 @@ package com.lion.lionwebsite.Service; import com.lion.lionwebsite.Domain.GalleryForQuery; import com.lion.lionwebsite.Util.FileDownload; import com.lion.lionwebsite.Util.GalleryUtil; +import com.lion.lionwebsite.Util.ImageFileCache; +import java.nio.file.Path; import com.lion.lionwebsite.Util.Response; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.servlet.ServletOutputStream; @@ -103,22 +105,16 @@ public class QueryService { String fileName = path.substring(path.lastIndexOf("/") + 1); String suffix = fileName.substring(fileName.lastIndexOf(".")); fileName = fileName.substring(0, fileName.lastIndexOf(".")); - File image = new File(CachePath, fileName + ".avif"); - - if(image.isFile()){ - FileDownload.export(request, response, image.getAbsolutePath()); - return; - } - - path = "https://ehgt.org/" + path; - try(ServletOutputStream outputStream = response.getOutputStream()){ - new URI(path).toURL().openConnection().getInputStream().transferTo(new FileOutputStream(CachePath + fileName + suffix)); - GalleryUtil.convertImg(CachePath + fileName + suffix, suffix); - FileInputStream inputStream = new FileInputStream(image.getAbsoluteFile()); //如果放到括号里,会导致图片未创建时创建文件流失败报错 - outputStream.write(inputStream.readAllBytes()); - inputStream.close(); - }catch (IOException | URISyntaxException e){ - log.error("获取缩略图失败", e); + String sourceUrl = "https://ehgt.org/" + path; + try { + Path image = ImageFileCache.get(Path.of(CachePath), fileName, () -> sourceUrl); + FileDownload.export(request, response, image.toString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + response.setStatus(503); + } catch (Exception e) { + log.warn("获取缩略图失败: errorType={}", e.getClass().getSimpleName()); + if (!response.isCommitted()) response.setStatus(404); } } } diff --git a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java index f114ad1..1ac7fef 100644 --- a/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java +++ b/src/main/java/com/lion/lionwebsite/Util/GalleryUtil.java @@ -5,6 +5,9 @@ import com.lion.lionwebsite.Domain.Gallery; import com.lion.lionwebsite.Domain.ImageKeyCache; import com.lion.lionwebsite.Exception.ResolutionNotMatchException; import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import java.nio.file.*; +import java.util.concurrent.TimeUnit; import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -44,7 +47,9 @@ public class GalleryUtil { static ConcurrentHashMap gid2MpvKey = new ConcurrentHashMap<>(); /** Reusable HTTP client —不要每次请求新建 */ - private static final CloseableHttpClient httpClient = HttpClients.createDefault(); + private static final CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(5_000) + .setConnectionRequestTimeout(5_000).setSocketTimeout(15_000).build()).build(); /** E-Hentai Cookie, injected from application.yaml via CustomBean */ private static String ehentaiCookie = ""; @@ -240,15 +245,38 @@ public class GalleryUtil { } public static String convertImg(String imagePath, String suffix){ - Runtime rt = Runtime.getRuntime(); + Path source = Path.of(imagePath); + Path target = source.resolveSibling(source.getFileName().toString().replaceFirst("\\Q" + suffix + "\\E$", ".avif")); + if (source.equals(target)) return imagePath; + Path temporary = null; + Process process = null; try { - Process exec = rt.exec(new String[]{"convert", imagePath, imagePath.replace(suffix, ".avif")}); - exec.waitFor(); - boolean ignored = new File(imagePath).delete(); - return imagePath.replace(suffix, ".avif"); - } catch (IOException| InterruptedException e) { - log.error("文件{}转换失败", imagePath, e); + temporary = Files.createTempFile(target.toAbsolutePath().getParent(), ".convert-", ".avif"); + process = new ProcessBuilder("convert", source.toString(), temporary.toString()) + .redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start(); + if (!process.waitFor(60, TimeUnit.SECONDS)) + throw new IOException("图片转换超时"); + if (process.exitValue() != 0 || Files.size(temporary) == 0) + throw new IOException("图片转换失败"); + try { + Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + } + Files.deleteIfExists(source); + return target.toString(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return null; + } catch (IOException e) { + log.warn("文件{}转换失败", imagePath, e); + return null; + } finally { + if (process != null && process.isAlive()) process.destroyForcibly(); + if (temporary != null) { + try { Files.deleteIfExists(temporary); } + catch (IOException e) { log.warn("清理图片转换临时文件失败", e); } + } } } diff --git a/src/main/java/com/lion/lionwebsite/Util/ImageFileCache.java b/src/main/java/com/lion/lionwebsite/Util/ImageFileCache.java new file mode 100644 index 0000000..9bdb2e9 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/ImageFileCache.java @@ -0,0 +1,67 @@ +package com.lion.lionwebsite.Util; + +import java.io.*; +import java.net.URI; +import java.net.URLConnection; +import java.nio.file.*; +import java.util.Locale; +import java.util.concurrent.Callable; + +/** Downloads into private temporary files and publishes only completed images. */ +public final class ImageFileCache { + private static final SingleFlight downloads = new SingleFlight<>(); + + private ImageFileCache() { } + + public static Path find(Path directory, String name) { + for (String suffix : new String[]{".avif", ".gif"}) { + Path path = directory.resolve(name + suffix); + if (Files.isRegularFile(path) && path.toFile().length() > 0) + return path; + } + return null; + } + + public static Path get(Path directory, String name, Callable sourceUrl) throws Exception { + Path key = directory.resolve(name).toAbsolutePath().normalize(); + return downloads.run(key, () -> { + Path cached = find(directory, name); + if (cached != null) return cached; + Files.createDirectories(directory); + String url = sourceUrl.call(); + if (url == null) throw new IOException("图片地址不存在"); + URI source = new URI(url); + boolean gif = source.getPath().toLowerCase(Locale.ROOT).endsWith(".gif"); + String suffix = gif ? ".gif" : ".img"; + Path temporary = Files.createTempFile(directory, ".download-", suffix); + Path converted = null; + try { + URLConnection connection = source.toURL().openConnection(); + connection.setConnectTimeout(5_000); + connection.setReadTimeout(15_000); + try (InputStream input = connection.getInputStream(); + OutputStream output = Files.newOutputStream(temporary)) { + input.transferTo(output); + } + if (Files.size(temporary) == 0) throw new IOException("图片内容为空"); + if (gif) { + converted = temporary; + } else { + String result = GalleryUtil.convertImg(temporary.toString(), suffix); + if (result == null) throw new IOException("图片转换失败"); + converted = Path.of(result); + } + Path target = directory.resolve(name + (gif ? ".gif" : ".avif")); + try { + Files.move(converted, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(converted, target, StandardCopyOption.REPLACE_EXISTING); + } + return target; + } finally { + Files.deleteIfExists(temporary); + if (converted != null && !converted.equals(temporary)) Files.deleteIfExists(converted); + } + }); + } +} diff --git a/src/main/java/com/lion/lionwebsite/Util/SingleFlight.java b/src/main/java/com/lion/lionwebsite/Util/SingleFlight.java new file mode 100644 index 0000000..03fd5c2 --- /dev/null +++ b/src/main/java/com/lion/lionwebsite/Util/SingleFlight.java @@ -0,0 +1,30 @@ +package com.lion.lionwebsite.Util; + +import java.util.concurrent.*; + +/** Concurrent callers for the same key share one in-flight operation, including its failure. */ +public final class SingleFlight { + private final ConcurrentHashMap> running = new ConcurrentHashMap<>(); + + public V run(K key, Callable operation) throws Exception { + CompletableFuture mine = new CompletableFuture<>(); + CompletableFuture existing = running.putIfAbsent(key, mine); + if (existing != null) { + try { return existing.get(); } + catch (ExecutionException e) { + if (e.getCause() instanceof Exception cause) throw cause; + throw new IllegalStateException(e.getCause()); + } + } + try { + V result = operation.call(); + mine.complete(result); + return result; + } catch (Exception | Error e) { + mine.completeExceptionally(e); + throw e; + } finally { + running.remove(key, mine); + } + } +} diff --git a/src/test/java/com/lion/lionwebsite/Service/PersonalArchiveTest.java b/src/test/java/com/lion/lionwebsite/Service/PersonalArchiveTest.java new file mode 100644 index 0000000..6ec9153 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Service/PersonalArchiveTest.java @@ -0,0 +1,27 @@ +package com.lion.lionwebsite.Service; + +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.nio.file.*; +import java.util.HashMap; +import static org.junit.jupiter.api.Assertions.*; + +class PersonalArchiveTest { + @Test void archiveContainsCompleteFilesAndCanBeOpenedImmediately(@TempDir Path root) throws Exception { + Path source = Files.createDirectory(root.resolve("source")); + Files.createDirectory(source.resolve("nested")); + for (int i = 0; i < 100; i++) Files.writeString(source.resolve("nested/" + i + ".txt"), "content-" + i); + Path archive = root.resolve("result.tar"); + PersonalService.writeTar(source, archive); + var contents = new HashMap(); + try (var input = new TarArchiveInputStream(Files.newInputStream(archive))) { + org.apache.commons.compress.archivers.tar.TarArchiveEntry entry; + while ((entry = input.getNextTarEntry()) != null) { + if (!entry.isDirectory()) contents.put(entry.getName(), new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + } + assertEquals(100, contents.size()); + for (int i = 0; i < 100; i++) assertEquals("content-" + i, contents.get("nested/" + i + ".txt")); + } +} diff --git a/src/test/java/com/lion/lionwebsite/Util/ImageFileCacheTest.java b/src/test/java/com/lion/lionwebsite/Util/ImageFileCacheTest.java new file mode 100644 index 0000000..31cdd99 --- /dev/null +++ b/src/test/java/com/lion/lionwebsite/Util/ImageFileCacheTest.java @@ -0,0 +1,60 @@ +package com.lion.lionwebsite.Util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.nio.file.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.time.Duration; +import static org.junit.jupiter.api.Assertions.*; + +class ImageFileCacheTest { + @Test void publishesClosedGifAndReusesIt(@TempDir Path root) throws Exception { + Path source = root.resolve("source.gif"); + byte[] content = "GIF89a test image".getBytes(java.nio.charset.StandardCharsets.UTF_8); + Files.write(source, content); + Path cache = root.resolve("cache"); + Path result = ImageFileCache.get(cache, "1", () -> source.toUri().toString()); + assertArrayEquals(content, Files.readAllBytes(result)); + assertEquals(result, ImageFileCache.get(cache, "1", () -> { throw new AssertionError("cache must avoid source access"); })); + try (var files = Files.list(cache)) { assertEquals(1, files.count()); } + } + + @Test void failedDownloadLeavesNoPublishedOrTemporaryFile(@TempDir Path root) throws Exception { + Path cache = root.resolve("cache"); + assertThrows(Exception.class, () -> ImageFileCache.get(cache, "1", () -> root.resolve("missing.gif").toUri().toString())); + assertNull(ImageFileCache.find(cache, "1")); + try (var files = Files.list(cache)) { assertEquals(0, files.count()); } + } + + @Test void concurrentCallersShareTheSameOperation() throws Exception { + SingleFlight flight = new SingleFlight<>(); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger calls = new AtomicInteger(); + AtomicReference followerThread = new AtomicReference<>(); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future leader = workers.submit(() -> flight.run("image", () -> { + calls.incrementAndGet(); entered.countDown(); + if (!release.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("test timed out"); + return "completed"; + })); + assertTrue(entered.await(2, TimeUnit.SECONDS)); + Future follower = workers.submit(() -> { + followerThread.set(Thread.currentThread()); + return flight.run("image", () -> { calls.incrementAndGet(); return "duplicate"; }); + }); + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> { + while (followerThread.get() == null || followerThread.get().getState() != Thread.State.WAITING) { + assertFalse(follower.isDone(), "follower must wait for the first operation"); + Thread.sleep(1); + } + }); + release.countDown(); + assertEquals("completed", leader.get(2, TimeUnit.SECONDS)); + assertEquals("completed", follower.get(2, TimeUnit.SECONDS)); + assertEquals(1, calls.get()); + } finally { release.countDown(); workers.shutdownNow(); } + } +}