commit cc6a19cf40fdc0c148bab3b0bb489c8e10245110 Author: chuzhongzai Date: Sun Jul 30 17:51:41 2023 +0800 第一个版本 diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..82dbec8 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..ed567a5 --- /dev/null +++ b/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + org.lion + storageNode + 1.0 + + + 17 + 17 + UTF-8 + + + + + io.netty + netty-all + 4.1.86.Final + + + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 + + + + org.projectlombok + lombok + 1.18.28 + + + + org.im4java + im4java + 1.4.0 + + + + log4j + log4j + 1.2.17 + + + + cn.hutool + hutool-all + 5.8.16 + + + + org.apache.commons + commons-compress + 1.21 + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.4.2 + + + package + + single + + + + + + jar-with-dependencies + + + + lion.Main + + + + + + + + \ No newline at end of file diff --git a/src/main/java/lion/Domain/GalleryTask.java b/src/main/java/lion/Domain/GalleryTask.java new file mode 100644 index 0000000..3c59457 --- /dev/null +++ b/src/main/java/lion/Domain/GalleryTask.java @@ -0,0 +1,50 @@ +package lion.Domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +public class GalleryTask { + + public static byte DOWNLOADING = 1; + + public static byte DOWNLOAD_COMPLETE = 2; + + public static byte DOWNLOAD_QUEUED = 3; + + public static byte COMPRESS_COMPLETE = 4; + + + public static byte DOWNLOAD_ALL = 3; + public static byte DOWNLOAD_PREVIEW = 2; + public static byte DOWNLOAD_SOURCE = 1; + + + + @JsonInclude(JsonInclude.Include.NON_NULL) + private String name; + + private int gid; + + private int pages; + + private byte status; + + private int proceeding; + + private byte type; + + @JsonIgnore + private String path; + + @JsonIgnore + public boolean is_download_complete(){ + return status == DOWNLOAD_COMPLETE; + } + + @JsonIgnore + public boolean is_compress_complete(){ + return status == COMPRESS_COMPLETE; + } +} diff --git a/src/main/java/lion/ErrorCode/ErrorCode.java b/src/main/java/lion/ErrorCode/ErrorCode.java new file mode 100644 index 0000000..26f432c --- /dev/null +++ b/src/main/java/lion/ErrorCode/ErrorCode.java @@ -0,0 +1,9 @@ +package lion.ErrorCode; + +public class ErrorCode { + public static final byte IO_ERROR = 1; + public static final byte FILE_NOT_FOUND = 2; + + public static final byte COMPRESS_ERROR = 3; + +} diff --git a/src/main/java/lion/Main.java b/src/main/java/lion/Main.java new file mode 100644 index 0000000..da07d45 --- /dev/null +++ b/src/main/java/lion/Main.java @@ -0,0 +1,28 @@ +package lion; + +import lion.ErrorCode.ErrorCode; +import lombok.extern.log4j.Log4j; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + + +@Log4j +public class Main { + + public static void main(String[] args) { + + new Thread(() -> MultiThreadedHTTPServer.main(null)).start(); + + new storageNode(); + } + + + + +} diff --git a/src/main/java/lion/Message/AbstractMessage.java b/src/main/java/lion/Message/AbstractMessage.java new file mode 100644 index 0000000..75725c1 --- /dev/null +++ b/src/main/java/lion/Message/AbstractMessage.java @@ -0,0 +1,24 @@ +package lion.Message; + +public class AbstractMessage { + + public static final byte DOWNLOAD_POST_MESSAGE = 1; + + public static final byte DOWNLOAD_STATUS_MESSAGE = 2; + + public static final byte DELETE_GALLERY_MESSAGE = 3; + + public static final byte RESPONSE_MESSAGE = 0; + + public static final byte UPDATE_GALLERY_MESSAGE = 4; + + public static final byte GALLERY_PAGE_QUERY_MESSAGE = 5; + + public static final byte IDENTITY_MESSAGE = 6; + + public static final byte GALLERY_REQUEST_MESSAGE = 101; + + public byte messageType; + + public int messageId; +} diff --git a/src/main/java/lion/Message/DeleteGalleryMessage.java b/src/main/java/lion/Message/DeleteGalleryMessage.java new file mode 100644 index 0000000..af1acd1 --- /dev/null +++ b/src/main/java/lion/Message/DeleteGalleryMessage.java @@ -0,0 +1,20 @@ +package lion.Message; + +import lombok.Data; + +@Data +public class DeleteGalleryMessage extends AbstractMessage{ + { + messageType = DELETE_GALLERY_MESSAGE; + } + + public static final byte DELETE_ALL = 3; + public static final byte DELETE_PREVIEW = 2; + public static final byte DELETE_SOURCE = 1; + + + + byte deleteType; + + String galleryName; +} diff --git a/src/main/java/lion/Message/DownloadPostMessage.java b/src/main/java/lion/Message/DownloadPostMessage.java new file mode 100644 index 0000000..da1f30d --- /dev/null +++ b/src/main/java/lion/Message/DownloadPostMessage.java @@ -0,0 +1,13 @@ +package lion.Message; + +import lion.Domain.GalleryTask; +import lombok.Data; + +@Data +public class DownloadPostMessage extends AbstractMessage{ + { + messageType = DOWNLOAD_POST_MESSAGE; + } + + GalleryTask galleryTask; +} diff --git a/src/main/java/lion/Message/DownloadStatusMessage.java b/src/main/java/lion/Message/DownloadStatusMessage.java new file mode 100644 index 0000000..32fab14 --- /dev/null +++ b/src/main/java/lion/Message/DownloadStatusMessage.java @@ -0,0 +1,13 @@ +package lion.Message; + +import lion.Domain.GalleryTask; +import lombok.Data; + +@Data +public class DownloadStatusMessage extends AbstractMessage{ + GalleryTask[] galleryTasks; + + { + messageType = DOWNLOAD_STATUS_MESSAGE; + } +} diff --git a/src/main/java/lion/Message/GalleryPageQueryMessage.java b/src/main/java/lion/Message/GalleryPageQueryMessage.java new file mode 100644 index 0000000..1f86d72 --- /dev/null +++ b/src/main/java/lion/Message/GalleryPageQueryMessage.java @@ -0,0 +1,21 @@ +package lion.Message; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +public class GalleryPageQueryMessage extends AbstractMessage{ + { + messageType = GALLERY_PAGE_QUERY_MESSAGE; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + String name; + + int page; + + @JsonInclude(JsonInclude.Include.NON_NULL) + String pageName; + + byte result; +} diff --git a/src/main/java/lion/Message/GalleryRequestMessage.java b/src/main/java/lion/Message/GalleryRequestMessage.java new file mode 100644 index 0000000..22085dd --- /dev/null +++ b/src/main/java/lion/Message/GalleryRequestMessage.java @@ -0,0 +1,25 @@ +package lion.Message; + +import lombok.Data; + +@Data +//请求预览/压缩包 +public class GalleryRequestMessage extends AbstractMessage{ + + public static final byte SOURCE = 1; + public static final byte PREVIEW = 2; + + public static final byte COMPRESS_SOURCE = 3; + + { + messageType = GALLERY_REQUEST_MESSAGE; + } + + String galleryName; + + byte type; + + short page; + + short port; +} diff --git a/src/main/java/lion/Message/IdentityMessage.java b/src/main/java/lion/Message/IdentityMessage.java new file mode 100644 index 0000000..ad97153 --- /dev/null +++ b/src/main/java/lion/Message/IdentityMessage.java @@ -0,0 +1,10 @@ +package lion.Message; + +import lombok.Data; + +@Data +public class IdentityMessage extends AbstractMessage{ + { + messageType = IDENTITY_MESSAGE; + } +} diff --git a/src/main/java/lion/Message/MessageCodec.java b/src/main/java/lion/Message/MessageCodec.java new file mode 100644 index 0000000..5849dca --- /dev/null +++ b/src/main/java/lion/Message/MessageCodec.java @@ -0,0 +1,58 @@ +package lion.Message; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageCodec; +import lombok.extern.log4j.Log4j; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Log4j +public class MessageCodec extends ByteToMessageCodec { + + ObjectMapper objectMapper; + + public MessageCodec(){ + objectMapper = new ObjectMapper(); + } + + @Override + protected void encode(ChannelHandlerContext channelHandlerContext, AbstractMessage abstractMessage, ByteBuf byteBuf) { + byteBuf.writeByte(abstractMessage.messageType); + + byte[] bytes = objectMapper.valueToTree(abstractMessage).toString().getBytes(StandardCharsets.UTF_8); + byteBuf.writeInt(bytes.length); + byteBuf.writeBytes(bytes); + } + + @Override + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { + byte messageType = byteBuf.readByte(); + int length = byteBuf.readInt(); + System.out.println(length); + byte[] bytes = new byte[length]; + byteBuf.readBytes(bytes); + final String metadata = new String(bytes, StandardCharsets.UTF_8); + + AbstractMessage abstractMessage = switch (messageType){ + case AbstractMessage.DOWNLOAD_POST_MESSAGE -> objectMapper.readValue(metadata, DownloadPostMessage.class); + case AbstractMessage.DOWNLOAD_STATUS_MESSAGE -> objectMapper.readValue(metadata, DownloadStatusMessage.class); + case AbstractMessage.GALLERY_REQUEST_MESSAGE -> objectMapper.readValue(metadata, GalleryRequestMessage.class); + case AbstractMessage.RESPONSE_MESSAGE -> objectMapper.readValue(metadata, ResponseMessage.class); + case AbstractMessage.UPDATE_GALLERY_MESSAGE -> objectMapper.readValue(metadata, UpdateGalleryMessage.class); + case AbstractMessage.DELETE_GALLERY_MESSAGE -> objectMapper.readValue(metadata, DeleteGalleryMessage.class); + case AbstractMessage.GALLERY_PAGE_QUERY_MESSAGE -> objectMapper.readValue(metadata, GalleryPageQueryMessage.class); + case AbstractMessage.IDENTITY_MESSAGE -> objectMapper.readValue(metadata, IdentityMessage.class); + default -> null; + }; + + if (abstractMessage == null){ + log.error("decode error, messageType: " + messageType + "ip:" + channelHandlerContext.channel().remoteAddress().toString()); + return; + } + list.add(abstractMessage); + } +} diff --git a/src/main/java/lion/Message/ResponseMessage.java b/src/main/java/lion/Message/ResponseMessage.java new file mode 100644 index 0000000..89430c7 --- /dev/null +++ b/src/main/java/lion/Message/ResponseMessage.java @@ -0,0 +1,21 @@ +package lion.Message; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class ResponseMessage extends AbstractMessage{ + { + messageType = RESPONSE_MESSAGE; + } + + static final byte SUCCESS = 0; + + byte result; + + public ResponseMessage(int messageId, byte result){ + this.messageId = messageId; + this.result = result; + } +} diff --git a/src/main/java/lion/Message/UpdateGalleryMessage.java b/src/main/java/lion/Message/UpdateGalleryMessage.java new file mode 100644 index 0000000..3daf96a --- /dev/null +++ b/src/main/java/lion/Message/UpdateGalleryMessage.java @@ -0,0 +1,13 @@ +package lion.Message; + +import lion.Domain.GalleryTask; +import lombok.Data; + +@Data +public class UpdateGalleryMessage extends AbstractMessage{ + { + messageType = UPDATE_GALLERY_MESSAGE; + } + + GalleryTask galleryTask; +} diff --git a/src/main/java/lion/Message/lombok.config b/src/main/java/lion/Message/lombok.config new file mode 100644 index 0000000..8e37527 --- /dev/null +++ b/src/main/java/lion/Message/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling=true +lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/src/main/java/lion/MultiThreadedHTTPServer.java b/src/main/java/lion/MultiThreadedHTTPServer.java new file mode 100644 index 0000000..782faba --- /dev/null +++ b/src/main/java/lion/MultiThreadedHTTPServer.java @@ -0,0 +1,180 @@ +package lion; + +import java.io.*; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class MultiThreadedHTTPServer { + private static final int PORT = 8888; + private static final int BUFFER_SIZE = 1024; + + public static void main(String[] args) { + ExecutorService threadPool = Executors.newCachedThreadPool(); + try(ServerSocket serverSocket = new ServerSocket(PORT)) { + System.out.println("Server listening on port " + PORT); + while (true) { + Socket clientSocket = serverSocket.accept(); + System.out.println("Client connected: " + clientSocket.getInetAddress().getHostAddress()); + // 线程池处理下载请求 + threadPool.submit(() -> handleClientRequest(clientSocket)); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static void handleClientRequest(Socket clientSocket) { + try { + BufferedReader requestReader = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); + String requestLine = requestReader.readLine(); + + // Parse the request line to get the method and path + String[] requestParts = requestLine.split(" "); + String method = requestParts[0]; + Map paramMap = parseRequestLine(requestParts[1]);//path + System.out.println(Arrays.toString(requestParts)); + + // Only handle GET requests + if (method.equals("GET")) { + // Set the file path for download + File file; + if(paramMap.get("AuthCode") != null) + if(paramMap.get("AuthCode").equals("alone")){ + String path = URLDecoder.decode(requestParts[1].split("\\?")[0], StandardCharsets.UTF_8); + file = new File(path); + } + else { + String filePath = "/root/gallery/gallery"; + String path = URLDecoder.decode(requestParts[1].split("\\?")[0], StandardCharsets.UTF_8); + if(!path.contains(".")){ + file = new File("/root/abc"); + }else { + String name = path.split("\\.")[0]; + filePath += (name + "/" + name + ".zip"); + file = new File(filePath); + } + } + else{ + sendErrorResponse(clientSocket, "403 Forbidden"); + return; + } + System.out.println(file.getAbsolutePath()); + // Check if the file exists and is readable + if (file.exists() && file.isFile() && file.canRead()) { + // Get the file length + long fileLength = file.length(); + + // Get the range information for resuming download + long startByte = 0; + long endByte = fileLength - 1; + String rangeHeader = getRequestHeader(requestReader, "Range"); + if (rangeHeader != null && rangeHeader.startsWith("bytes=")) { + String[] rangeValues = rangeHeader.substring(6).split("-"); + startByte = Long.parseLong(rangeValues[0]); + if (rangeValues.length > 1 && !rangeValues[1].isEmpty()) { + endByte = Long.parseLong(rangeValues[1]); + } + } + + // Send the HTTP response headers + OutputStream responseStream = clientSocket.getOutputStream(); + PrintWriter responseWriter = new PrintWriter(responseStream, true); + responseWriter.println("HTTP/1.1 206 Partial Content"); + responseWriter.println("Content-Type: application/octet-stream"); + responseWriter.println("Accept-Ranges: bytes"); + responseWriter.println("Content-Length: " + (endByte - startByte + 1)); + responseWriter.println("Content-Range: bytes " + startByte + "-" + endByte + "/" + fileLength); + responseWriter.println(); + + // Send the file content + try (RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r")) { + randomAccessFile.seek(startByte); + byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + long bytesRemaining = endByte - startByte + 1; + while (bytesRemaining > 0 && (bytesRead = randomAccessFile.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + responseStream.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + + // Close the response output stream + responseStream.close(); + } else { + // File not found or not readable, send 404 response + sendErrorResponse(clientSocket, "404 Not Found"); + } + } else { + // Non-GET requests, send 501 response + sendErrorResponse(clientSocket, "501 Not Implemented"); + } + + // Close the request reader and client socket + requestReader.close(); + clientSocket.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static String getRequestHeader(BufferedReader requestReader, String headerName) throws IOException { + String line; + while ((line = requestReader.readLine()) != null) { + if (line.trim().isEmpty()) { + break; + } + + if (line.startsWith(headerName + ":")) { + return line.substring(headerName.length() + 1).trim(); + } + } + return null; + } + + public static Map parseRequestLine(String requestLine) { + Map queryParams = new HashMap<>(); + + if(requestLine == null) + return null; + + if (requestLine.contains("?")) { + String[] requestParts = requestLine.split("\\?"); + String path = requestParts[0]; + queryParams.put("path", path); + + // Extract the query string after the '?' character + String queryString = requestParts[1]; + String[] paramPairs = queryString.split("&"); + + // Split the query string into individual parameter key-value pairs + for (String paramPair : paramPairs) { + String[] keyValue = paramPair.split("="); + if (keyValue.length == 2) { + String key = URLDecoder.decode(keyValue[0], StandardCharsets.UTF_8); + String value = URLDecoder.decode(keyValue[1], StandardCharsets.UTF_8); + queryParams.put(key, value); + } + } + }else{ + queryParams.put("path", requestLine); + } + return queryParams; + } + + private static void sendErrorResponse(Socket clientSocket, String statusCode) throws IOException { + OutputStream responseStream = clientSocket.getOutputStream(); + PrintWriter responseWriter = new PrintWriter(responseStream, true); + responseWriter.println("HTTP/1.1 " + statusCode); + responseWriter.println("Content-Type: text/html"); + responseWriter.println(); + responseWriter.println("

" + statusCode + "

"); + responseStream.close(); + } +} \ No newline at end of file diff --git a/src/main/java/lion/Service/DeleteService.java b/src/main/java/lion/Service/DeleteService.java new file mode 100644 index 0000000..170a6ca --- /dev/null +++ b/src/main/java/lion/Service/DeleteService.java @@ -0,0 +1,45 @@ +package lion.Service; + +import lion.ErrorCode.ErrorCode; +import cn.hutool.core.io.FileUtil; + +import java.io.File; + +public class DeleteService { + public static byte deleteAll(String path){ + File file = new File(path); + if(!file.isDirectory()) + return ErrorCode.FILE_NOT_FOUND; + + if (FileUtil.del(path)) { + return 0; + }else{ + return ErrorCode.IO_ERROR; + } + } + + public static byte deletePreview(String path){ + File directory = new File(path); + File[] files = directory.listFiles((dir, name) -> !name.endsWith("zip")); + + if(files == null) + return ErrorCode.FILE_NOT_FOUND; + for (File file : files) + if(!FileUtil.del(file)) + return ErrorCode.IO_ERROR; + + return 0; + } + + public static byte deleteSource(String path){ + File directory = new File(path); + File[] files = directory.listFiles((dir, name) -> name.endsWith("zip")); + + if(files == null) + return ErrorCode.FILE_NOT_FOUND; + if(!FileUtil.del(files[0])) + return ErrorCode.IO_ERROR; + + return 0; + } +} diff --git a/src/main/java/lion/Service/DeliveryService.java b/src/main/java/lion/Service/DeliveryService.java new file mode 100644 index 0000000..1344e4b --- /dev/null +++ b/src/main/java/lion/Service/DeliveryService.java @@ -0,0 +1,100 @@ +package lion.Service; +import lion.ErrorCode.ErrorCode; +import lion.Message.GalleryPageQueryMessage; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; + +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class DeliveryService { + + static String storagePath = "/root/gallery/gallery/"; + //缓存排序后的页数 + static LinkedHashMap> pageCache; + + static ExecutorService singleThreadPool = Executors.newSingleThreadExecutor(); + + + static { + pageCache = new LinkedHashMap<>(); + } + + public static byte deliveryPreview(String name, short page, short port){ + if(!pageCache.containsKey(name)){ + byte result; + if((result = pageCache(name)) != 0) + return result; + } + + //0页为缩略图 + if(page == 0){ + return delivery(new File(storagePath, name + "/thumbnail.webp"), port); + }else{ + return delivery(new File(storagePath, name + "/" + pageCache.get(name).get(page)), port); + } + } + + public static byte delivery(File file, short port){ + if(!file.exists()) + return ErrorCode.FILE_NOT_FOUND; + + singleThreadPool.submit(() -> { + try(SocketChannel socketChannel = SocketChannel.open(new InetSocketAddress("194.36.27.28", port)); + FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + ByteBuffer buffer = ByteBuffer.allocate(1024); + while (fileChannel.read(buffer)!=-1){ + buffer.flip(); + socketChannel.write(buffer); + buffer.clear(); + } + socketChannel.shutdownOutput(); + return 0; + }catch (IOException e){ + e.printStackTrace(); + return ErrorCode.IO_ERROR; + } + }); + return 0; + } + + public static byte pageQuery(GalleryPageQueryMessage gpqm){ + if(!pageCache.containsKey(gpqm.getName())){ + byte result; + if((result = pageCache(gpqm.getName())) != 0) + return result; + } + ArrayList pages = pageCache.get(gpqm.getName()); + if(pages.size() <= gpqm.getPage()) + return ErrorCode.FILE_NOT_FOUND; + gpqm.setPageName(pageCache.get(gpqm.getName()).get(gpqm.getPage())); + return 0; + } + + public static byte pageCache(String name){ + File directory = new File(storagePath, name); + if(!directory.isDirectory()) + return ErrorCode.FILE_NOT_FOUND; + ArrayList pageList = new ArrayList<>(); + File[] files = directory.listFiles(((dir, name1) -> name1.contains(".webp") && !name1.equals("thumbnail.webp"))); + + if(files == null) + return ErrorCode.FILE_NOT_FOUND; + ArrayList fileArrayList = new ArrayList<>(Arrays.asList(files)); + fileArrayList.sort(Comparator.naturalOrder()); + pageList.add("thumbnail.webp"); + fileArrayList.forEach((f) -> pageList.add(f.getName())); + pageCache.put(name, pageList); + return 0; + } +} diff --git a/src/main/java/lion/Service/DownloadCheckService.java b/src/main/java/lion/Service/DownloadCheckService.java new file mode 100644 index 0000000..4e5cf39 --- /dev/null +++ b/src/main/java/lion/Service/DownloadCheckService.java @@ -0,0 +1,132 @@ +package lion.Service; + +import lion.Domain.GalleryTask; +import lion.ErrorCode.ErrorCode; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.ZipUtil; +import lombok.extern.log4j.Log4j; +import org.im4java.core.ConvertCmd; +import org.im4java.core.IM4JavaException; +import org.im4java.core.IMOperation; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + + +@Log4j +public class DownloadCheckService { + ArrayList queue; + + String downloadPath = "/root/gallery/hentai/download/"; + + String storagePath = "/root/gallery/gallery/"; + + public DownloadCheckService(ArrayList queue){ + this.queue = queue; + } + + public boolean downloadCheck(){ + if(queue.size() == 0) + return false; + log.info("下载检查:" + Arrays.toString(queue.toArray())); + + File downloadDirectory = new File(downloadPath); + File[] fileArray = downloadDirectory.listFiles(); + + if(fileArray == null || fileArray.length == 0) + return false; + + ArrayList files = new ArrayList<>(Arrays.asList(fileArray)); + + //扫描进度 + for(File file: files) + for(GalleryTask galleryTask: queue){ + if(!file.getName().contains(String.valueOf(galleryTask.getGid()))) + continue; + + galleryTask.setName(file.getName()); + + File[] pages = file.listFiles((dir, name) -> !name.equals("galleryinfo.txt")); + if (pages == null || pages.length == 0) + continue; + galleryTask.setProceeding(pages.length); + + if (new File(file.getPath(), "galleryinfo.txt").exists()) { + galleryTask.setStatus(GalleryTask.DOWNLOAD_COMPLETE); + galleryTask.setPath(file.getPath()); + } + } + + + //转格式 + ConvertCmd convertCmd = new ConvertCmd(true); + for(GalleryTask galleryTask: queue){ + //跳过未完成 + if (!galleryTask.is_download_complete()) + continue; + + File[] images = new File(galleryTask.getPath()).listFiles((dir, name) -> name.endsWith(".jpg") || name.endsWith(".png")); + if(images == null){ + galleryTask.setStatus(ErrorCode.COMPRESS_ERROR); + continue; + } + //长度相同比较字典序，否则比较长度 + images = Arrays.stream(images).sorted((f1, f2) -> { + if(f1.getName().length() == f2.getName().length()) + return f1.compareTo(f2); + else + return f1.getName().length() - f2.getName().length(); + }).toArray(File[]::new); + + //创建文件夹 + File file = new File(storagePath + galleryTask.getName()); + if(file.isDirectory() || file.mkdirs()){ + log.info(galleryTask.getName() + "文件夹创建成功"); + }else{ + log.error(galleryTask.getName() + "文件夹创建失败"); + continue; + } + + //thumbnail + IMOperation operation = new IMOperation(); + operation.addImage(images[0].getAbsolutePath()); + operation.resize(500, 500); + operation.format("webp"); + operation.addImage(storagePath + galleryTask.getName() + "/thumbnail.webp"); + try{ + log.info("文件" + images[0].getName() + "，转换为thumbnail.webp"); + convertCmd.run(operation); + }catch (IOException | IM4JavaException | InterruptedException e){ + log.error("创建" + galleryTask.getName() + "缩略图失败"); + galleryTask.setStatus(ErrorCode.COMPRESS_ERROR); + continue; + } + + if(galleryTask.getType() == GalleryTask.DOWNLOAD_ALL || galleryTask.getType() == GalleryTask.DOWNLOAD_PREVIEW) + for (File image : images) { + log.info("文件" + image.getName() + "，转换为webp"); + operation = new IMOperation(); + operation.addImage(image.getAbsolutePath()); + operation.format("webp"); + operation.addImage(storagePath + galleryTask.getName() + "/" + image.getName().replace(".png", ".webp").replace(".jpg", ".webp")); + + try { + convertCmd.run(operation); + } catch (IOException | InterruptedException | IM4JavaException e) { + log.error("文件" + image.getName() + "转换失败"); + galleryTask.setStatus(ErrorCode.COMPRESS_ERROR); + break; + } + } + + if(galleryTask.getType() == GalleryTask.DOWNLOAD_ALL || galleryTask.getType() == GalleryTask.DOWNLOAD_SOURCE) + ZipUtil.zip(galleryTask.getPath(), storagePath + galleryTask.getName() + "/" + galleryTask.getName() + ".zip"); + FileUtil.del(galleryTask.getPath()); + } + + return true; + } + +} diff --git a/src/main/java/lion/storageNode.java b/src/main/java/lion/storageNode.java new file mode 100644 index 0000000..25d1466 --- /dev/null +++ b/src/main/java/lion/storageNode.java @@ -0,0 +1,158 @@ +package lion; + +import lion.Domain.GalleryTask; +import lion.Message.*; +import lion.Service.DeleteService; +import lion.Service.DeliveryService; +import lion.Service.DownloadCheckService; +import cn.hutool.core.io.FileUtil; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.logging.LoggingHandler; +import lombok.extern.log4j.Log4j; + +import java.util.ArrayList; +import java.util.ListIterator; +import java.util.concurrent.*; +import java.util.concurrent.locks.ReentrantLock; + +@Log4j +public class storageNode { + + ChannelFuture channelFuture; + + Channel server; + + DownloadCheckService downloadCheckService; + + ArrayList queue; + + ScheduledExecutorService checkThreadPool; + + + ReentrantLock lock; + + public static String storagePath = "/root/gallery/gallery/"; + + public storageNode(){ + queue = new ArrayList<>(0); + lock = new ReentrantLock(); + channelFuture = new ServerBootstrap() + .channel(NioServerSocketChannel.class) + .group(new NioEventLoopGroup()) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(NioSocketChannel channel) { + channel.pipeline().addLast(new LengthFieldBasedFrameDecoder(100000000, 1, 4)); + channel.pipeline().addLast(new MessageCodec()); + channel.pipeline().addLast(new LoggingHandler()); + channel.pipeline().addLast(new MyChannelInboundHandlerAdapter(queue)); + + } + }).option(ChannelOption.SO_BACKLOG, 128) + .childOption(ChannelOption.SO_KEEPALIVE, true) + .bind(26321); + log.info("listen port:8080"); + downloadCheckService = new DownloadCheckService(queue); + checkThreadPool = Executors.newScheduledThreadPool(1); + checkThreadPool.scheduleAtFixedRate(this::mainThread, 5, 5, TimeUnit.SECONDS); + } + + public void mainThread(){ + //检查 + if(!downloadCheckService.downloadCheck()) + return; + //发送 + DownloadStatusMessage downloadStatusMessage = new DownloadStatusMessage(); + downloadStatusMessage.setGalleryTasks(queue.toArray(GalleryTask[]::new)); + server.writeAndFlush(downloadStatusMessage); + + ListIterator listIterator = queue.listIterator(); + lock.lock(); + while (listIterator.hasNext()){ + GalleryTask galleryTask = listIterator.next(); + if(galleryTask.is_download_complete()) + listIterator.remove(); + } + lock.unlock(); + log.info("任务状态发送完成"); + } + + class MyChannelInboundHandlerAdapter extends ChannelInboundHandlerAdapter{ + ArrayList queue; + + public MyChannelInboundHandlerAdapter(ArrayList queue) { + super(); + this.queue = queue; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + System.out.println(msg); + + if(msg instanceof IdentityMessage) { + server = ctx.channel(); + log.info("server 上线"); + //提交下载 + }else if(msg instanceof DownloadPostMessage dpm){ + lock.lock(); + queue.add(dpm.getGalleryTask()); + lock.unlock(); + ctx.writeAndFlush(new ResponseMessage(dpm.messageId, (byte) 0)); + + //删除本子 全部/预览/源文件 + }else if(msg instanceof DeleteGalleryMessage deleteGalleryMessage){ + byte result = switch (deleteGalleryMessage.getDeleteType()){ + case DeleteGalleryMessage.DELETE_ALL -> + DeleteService.deleteAll(storagePath + deleteGalleryMessage.getGalleryName()); + case DeleteGalleryMessage.DELETE_PREVIEW -> + DeleteService.deletePreview(storagePath + deleteGalleryMessage.getGalleryName()); + case DeleteGalleryMessage.DELETE_SOURCE -> + DeleteService.deleteSource(storagePath + deleteGalleryMessage.getGalleryName()); + default -> -1; + }; + ResponseMessage responseMessage = new ResponseMessage(deleteGalleryMessage.messageId, result); + ctx.writeAndFlush(responseMessage); + + //请求预览 + }else if(msg instanceof GalleryRequestMessage grm){ + byte result = DeliveryService.deliveryPreview(grm.getGalleryName(), grm.getPage(), grm.getPort()); + + ResponseMessage responseMessage = new ResponseMessage(); + responseMessage.messageId = grm.messageId; + responseMessage.setResult(result); + ctx.writeAndFlush(responseMessage); + + //更新本子 删除原有文件，然后加入到队列 + }else if(msg instanceof UpdateGalleryMessage ugm){ + FileUtil.del(storagePath + ugm.getGalleryTask().getName()); + lock.lock(); + queue.add(ugm.getGalleryTask()); + lock.unlock(); + ctx.writeAndFlush(new ResponseMessage(ugm.messageId, (byte) 0)); + + }else if(msg instanceof GalleryPageQueryMessage gpqm){ + byte result = DeliveryService.pageQuery(gpqm); + gpqm.setResult(result); + ctx.writeAndFlush(gpqm); + } + + //修复预览 + + //重新生成压缩包 + } + + @Override + public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { + if(ctx.channel().equals(server)) { + log.info("server 下线"); + server = null; + } + } + } +} +