Compare commits

..

10 Commits

29 changed files with 448 additions and 469 deletions

View File

@ -7,8 +7,13 @@
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
<option name="workspaceImportForciblyTurnedOn" value="true" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="17" project-jdk-type="JavaSDK">
<component name="PWA">
<option name="enabled" value="true" />
<option name="wasEnabledAtLeastOnce" value="true" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="17" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

80
pom.xml
View File

@ -9,8 +9,8 @@
<version>1.0</version>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
@ -18,7 +18,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.86.Final</version>
<version>4.1.101.Final</version>
</dependency>
<dependency>
@ -30,25 +30,27 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.28</version>
<version>1.18.30</version>
</dependency>
<dependency>
<groupId>org.im4java</groupId>
<artifactId>im4java</artifactId>
<version>1.4.0</version>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.7</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.18</version>
<version>5.8.20</version>
</dependency>
<dependency>
@ -60,27 +62,45 @@
<build>
<plugins>
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-assembly-plugin</artifactId>-->
<!-- <version>3.4.2</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <phase>package</phase>-->
<!-- <goals>-->
<!-- <goal>single</goal>-->
<!-- </goals>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- <configuration>-->
<!-- <descriptorRefs>-->
<!-- <descriptorRef>jar-with-dependencies</descriptorRef>-->
<!-- </descriptorRefs>-->
<!-- <archive>-->
<!-- <manifest>-->
<!-- <mainClass>lion.Main</mainClass>-->
<!-- </manifest>-->
<!-- </archive>-->
<!-- </configuration>-->
<!-- </plugin>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.4.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<groupId>org.graalvm.buildtools</groupId>
<artifactId>native-maven-plugin</artifactId>
<version>0.9.28</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>lion.Main</mainClass>
</manifest>
</archive>
<mainClass>lion.Main</mainClass>
<imageName>storageNode</imageName>
<buildArgs>
<arg>-H:+ReportExceptionStackTraces</arg>
<arg>--gc=G1</arg>
<arg>--enable-url-protocols=https</arg>
<arg>-H:ReflectionConfigurationFiles=src/main/resources/reflect-config.json</arg>
</buildArgs>
<metadataRepository>
<enabled>true</enabled>
</metadataRepository>
</configuration>
</plugin>
</plugins>

View File

@ -0,0 +1,15 @@
package lion;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Data;
import java.util.concurrent.atomic.AtomicInteger;
@Data
public class CustomUtil {
public static AtomicInteger counter = new AtomicInteger();
public static ObjectMapper objectMapper = new ObjectMapper();
}

View File

@ -6,35 +6,20 @@ import lombok.Data;
@Data
public class GalleryTask {
public static byte DOWNLOADING = 1;
public static byte DOWNLOAD_COMPLETE = 2;
public static byte DOWNLOAD_QUEUED = 3;
public static byte COMPRESSING = 3;
public static byte COMPRESS_COMPLETE = 4;
public static byte DOWNLOAD_ALL = 3;
public static byte DOWNLOAD_PREVIEW = 2;
public static byte DOWNLOAD_SOURCE = 1;
@JsonInclude(JsonInclude.Include.NON_NULL)
private String name;
private int gid;
private int pages;
private byte status;
private int proceeding;
private byte type;
@JsonIgnore
private String path;
@ -47,4 +32,9 @@ public class GalleryTask {
public boolean is_compress_complete(){
return status == COMPRESS_COMPLETE;
}
@JsonIgnore
public boolean is_compressing(){
return status == COMPRESSING;
}
}

View File

@ -3,7 +3,4 @@ package lion.ErrorCode;
public class ErrorCode {
public static final byte IO_ERROR = 1;
public static final byte FILE_NOT_FOUND = 2;
public static final byte COMPRESS_ERROR = 3;
}

View File

@ -0,0 +1,26 @@
package lion.Extranel;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.security.Key;
public class AESUtils {
private static final String ALGORITHM = "AES";
private static final String TRANSFORMATION = "AES/ECB/PKCS5Padding";
private static final byte[] keyBytes = "ThisIsA128BitKey".getBytes();
public static byte[] encrypt(byte[] data) throws Exception {
Key key = new SecretKeySpec(keyBytes, ALGORITHM);
Cipher cipher = Cipher.getInstance(TRANSFORMATION);
cipher.init(Cipher.ENCRYPT_MODE, key);
return cipher.doFinal(data);
}
public static byte[] decrypt(byte[] encryptedData) throws Exception {
Key key = new SecretKeySpec(keyBytes, ALGORITHM);
Cipher cipher = Cipher.getInstance(TRANSFORMATION);
cipher.init(Cipher.DECRYPT_MODE, key);
return cipher.doFinal(encryptedData);
}
}

View File

@ -0,0 +1,65 @@
package lion.Extranel;
import com.fasterxml.jackson.databind.ObjectMapper;
import lion.CustomUtil;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Slf4j
public class Server {
ExecutorService thread_pool;
ObjectMapper objectMapper;
public static void main(String[] args) {
new Server();
}
public Server(){
log.info("开始监听:55555");
objectMapper = CustomUtil.objectMapper;
thread_pool = Executors.newFixedThreadPool(2);
thread_pool.submit(() -> {
try(ServerSocket server = new ServerSocket(55555)){
server.setSoTimeout(1000000);
while (true){
Socket socket = server.accept();
thread_pool.submit(() -> handleSocket(socket));
}
}catch (IOException e){
log.error(e.getMessage());
}
});
}
public void handleSocket(Socket socket){
try{
BufferedInputStream inputStream = new BufferedInputStream(socket.getInputStream());
Thread.sleep(500);
byte[] buf = new byte[inputStream.available()];
inputStream.read(buf);
byte[] bytes = AESUtils.decrypt(buf);
HashMap<String, String> map = (HashMap<String, String>) objectMapper.readValue(bytes, HashMap.class);
URL url = new URI(map.get("path")).toURL();
log.info("处理反代 ip: {},路径: {}", socket.getInetAddress().getHostAddress(), map.get("path"));
bytes = url.openConnection().getInputStream().readAllBytes();
socket.getOutputStream().write(AESUtils.encrypt(bytes));
socket.getOutputStream().flush();
socket.getOutputStream().close();
} catch (Exception e) {
e.printStackTrace();
}
}
}

View File

@ -1,28 +1,20 @@
package lion;
import lion.ErrorCode.ErrorCode;
import lombok.extern.log4j.Log4j;
import io.netty.bootstrap.Bootstrap;
import lion.Extranel.Server;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
@Log4j
@Slf4j
public class Main {
public static void main(String[] args) {
boot();
new Thread(() -> MultiThreadedHTTPServer.main(null)).start();
new Thread(() -> Server.main(null)).start();
new storageNode();
}
public static void boot(){
new Bootstrap();
}
}

View File

@ -10,16 +10,10 @@ public class AbstractMessage {
public static final byte RESPONSE_MESSAGE = 0;
public static final byte UPDATE_GALLERY_MESSAGE = 4;
public static final byte GALLERY_PAGE_QUERY_MESSAGE = 5;
public static final byte IDENTITY_MESSAGE = 6;
public static final byte MAINTAIN_MESSAGE = 7;
public static final byte GALLERY_REQUEST_MESSAGE = 101;
public byte messageType;
public int messageId;

View File

@ -1,20 +0,0 @@
package lion.Message;
import lombok.Data;
@Data
public class DeleteGalleryMessage extends AbstractMessage{
{
messageType = DELETE_GALLERY_MESSAGE;
}
public static final byte DELETE_ALL = 3;
public static final byte DELETE_PREVIEW = 2;
public static final byte DELETE_SOURCE = 1;
byte deleteType;
String galleryName;
}

View File

@ -1,21 +0,0 @@
package lion.Message;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;
@Data
public class GalleryPageQueryMessage extends AbstractMessage{
{
messageType = GALLERY_PAGE_QUERY_MESSAGE;
}
@JsonInclude(JsonInclude.Include.NON_NULL)
String name;
int page;
@JsonInclude(JsonInclude.Include.NON_NULL)
String pageName;
byte result;
}

View File

@ -1,25 +0,0 @@
package lion.Message;
import lombok.Data;
@Data
//请求预览/压缩包
public class GalleryRequestMessage extends AbstractMessage{
public static final byte SOURCE = 1;
public static final byte PREVIEW = 2;
public static final byte COMPRESS_SOURCE = 3;
{
messageType = GALLERY_REQUEST_MESSAGE;
}
String galleryName;
byte type;
short page;
short port;
}

View File

@ -1,10 +0,0 @@
package lion.Message;
import lombok.Data;
@Data
public class IdentityMessage extends AbstractMessage{
{
messageType = IDENTITY_MESSAGE;
}
}

View File

@ -0,0 +1,12 @@
package lion.Message.Main;
import lion.Message.AbstractMessage;
import lombok.Data;
@Data
public class DeleteGalleryMessage extends AbstractMessage {
{
messageType = DELETE_GALLERY_MESSAGE;
}
String galleryName;
}

View File

@ -1,10 +1,11 @@
package lion.Message;
package lion.Message.Main;
import lion.Domain.GalleryTask;
import lion.Message.AbstractMessage;
import lombok.Data;
@Data
public class DownloadPostMessage extends AbstractMessage{
public class DownloadPostMessage extends AbstractMessage {
{
messageType = DOWNLOAD_POST_MESSAGE;
}

View File

@ -1,10 +1,11 @@
package lion.Message;
package lion.Message.Main;
import lion.Domain.GalleryTask;
import lion.Message.AbstractMessage;
import lombok.Data;
@Data
public class DownloadStatusMessage extends AbstractMessage{
public class DownloadStatusMessage extends AbstractMessage {
GalleryTask[] galleryTasks;
{

View File

@ -0,0 +1,12 @@
package lion.Message.Main;
import lion.Message.AbstractMessage;
import lombok.Data;
@Data
public class IdentityMessage extends AbstractMessage {
{
messageType = IDENTITY_MESSAGE;
}
String identity;
}

View File

@ -0,0 +1,9 @@
package lion.Message.Main;
import lion.Message.AbstractMessage;
public class MaintainMessage extends AbstractMessage {
{
messageType = MAINTAIN_MESSAGE;
}
}

View File

@ -1,11 +1,12 @@
package lion.Message;
package lion.Message.Main;
import lion.Message.AbstractMessage;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class ResponseMessage extends AbstractMessage{
public class ResponseMessage extends AbstractMessage {
{
messageType = RESPONSE_MESSAGE;
}

View File

@ -1,7 +0,0 @@
package lion.Message;
public class MaintainMessage extends AbstractMessage{
{
messageType = MAINTAIN_MESSAGE;
}
}

View File

@ -5,18 +5,20 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageCodec;
import lombok.extern.log4j.Log4j;
import lion.CustomUtil;
import lion.Message.Main.*;
import lombok.extern.slf4j.Slf4j;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Log4j
@Slf4j
public class MessageCodec extends ByteToMessageCodec<AbstractMessage> {
ObjectMapper objectMapper;
public MessageCodec(){
objectMapper = new ObjectMapper();
objectMapper = CustomUtil.objectMapper;
}
@Override
@ -32,7 +34,6 @@ public class MessageCodec extends ByteToMessageCodec<AbstractMessage> {
protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list) throws Exception {
byte messageType = byteBuf.readByte();
int length = byteBuf.readInt();
System.out.println(length);
byte[] bytes = new byte[length];
byteBuf.readBytes(bytes);
final String metadata = new String(bytes, StandardCharsets.UTF_8);
@ -40,11 +41,8 @@ public class MessageCodec extends ByteToMessageCodec<AbstractMessage> {
AbstractMessage abstractMessage = switch (messageType){
case AbstractMessage.DOWNLOAD_POST_MESSAGE -> objectMapper.readValue(metadata, DownloadPostMessage.class);
case AbstractMessage.DOWNLOAD_STATUS_MESSAGE -> objectMapper.readValue(metadata, DownloadStatusMessage.class);
case AbstractMessage.GALLERY_REQUEST_MESSAGE -> objectMapper.readValue(metadata, GalleryRequestMessage.class);
case AbstractMessage.RESPONSE_MESSAGE -> objectMapper.readValue(metadata, ResponseMessage.class);
case AbstractMessage.UPDATE_GALLERY_MESSAGE -> objectMapper.readValue(metadata, UpdateGalleryMessage.class);
case AbstractMessage.DELETE_GALLERY_MESSAGE -> objectMapper.readValue(metadata, DeleteGalleryMessage.class);
case AbstractMessage.GALLERY_PAGE_QUERY_MESSAGE -> objectMapper.readValue(metadata, GalleryPageQueryMessage.class);
case AbstractMessage.IDENTITY_MESSAGE -> objectMapper.readValue(metadata, IdentityMessage.class);
case AbstractMessage.MAINTAIN_MESSAGE -> objectMapper.readValue(metadata, MaintainMessage.class);
default -> null;

View File

@ -1,13 +0,0 @@
package lion.Message;
import lion.Domain.GalleryTask;
import lombok.Data;
@Data
public class UpdateGalleryMessage extends AbstractMessage{
{
messageType = UPDATE_GALLERY_MESSAGE;
}
GalleryTask galleryTask;
}

View File

@ -18,8 +18,7 @@ public class MultiThreadedHTTPServer {
String real_ip;
try{
real_ip = InetAddress.getByName("lionwebsite.xyz").getHostAddress();
}
catch (UnknownHostException ignored){
} catch (UnknownHostException ignored){
real_ip = "207.60.50.74";
}
try(ServerSocket serverSocket = new ServerSocket(PORT)) {
@ -69,9 +68,29 @@ public class MultiThreadedHTTPServer {
file = new File("/root/abc");
}else {
String name = path.substring(0, path.lastIndexOf('.'));
System.out.println(name);
filePath += (name + "/" + name + ".zip");
file = new File(filePath);
name = filePath + name + "/" + name + ".zip";
file = new File(name);
//该文件不存在
if(!file.isFile()){
String gid = paramMap.get("gid");
//文件不存在的情况下gid也不存在直接404
if(gid == null)
file = new File("/root/abc");
//gid存在的情况下尝试查找对应的文件
else {
File[] galleryDirectories = (new File(filePath)).listFiles();
assert galleryDirectories != null;
for (File galleryDirectory : galleryDirectories)
if (galleryDirectory.getName().contains(gid)) {
file = new File(galleryDirectory.getAbsolutePath(), galleryDirectory.getName() + ".zip");
break;
}
}
}
}
}
else{
@ -87,7 +106,7 @@ public class MultiThreadedHTTPServer {
// Get the range information for resuming download
long startByte = 0;
long endByte = fileLength - 1;
String rangeHeader = getRequestHeader(requestReader, "Range");
String rangeHeader = getRequestHeader(requestReader);
if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
String[] rangeValues = rangeHeader.substring(6).split("-");
startByte = Long.parseLong(rangeValues[0]);
@ -139,15 +158,15 @@ public class MultiThreadedHTTPServer {
}
}
private static String getRequestHeader(BufferedReader requestReader, String headerName) throws IOException {
private static String getRequestHeader(BufferedReader requestReader) throws IOException {
String line;
while ((line = requestReader.readLine()) != null) {
if (line.trim().isEmpty()) {
break;
}
if (line.startsWith(headerName + ":")) {
return line.substring(headerName.length() + 1).trim();
if (line.startsWith("Range" + ":")) {
return line.substring("Range".length() + 1).trim();
}
}
return null;

View File

@ -11,35 +11,9 @@ public class DeleteService {
if(!file.isDirectory())
return ErrorCode.FILE_NOT_FOUND;
if (FileUtil.del(path)) {
if (FileUtil.del(path))
return 0;
}else{
else
return ErrorCode.IO_ERROR;
}
}
public static byte deletePreview(String path){
File directory = new File(path);
File[] files = directory.listFiles((dir, name) -> !name.endsWith("zip"));
if(files == null)
return ErrorCode.FILE_NOT_FOUND;
for (File file : files)
if(!FileUtil.del(file))
return ErrorCode.IO_ERROR;
return 0;
}
public static byte deleteSource(String path){
File directory = new File(path);
File[] files = directory.listFiles((dir, name) -> name.endsWith("zip"));
if(files == null)
return ErrorCode.FILE_NOT_FOUND;
if(!FileUtil.del(files[0]))
return ErrorCode.IO_ERROR;
return 0;
}
}

View File

@ -1,102 +0,0 @@
package lion.Service;
import lion.ErrorCode.ErrorCode;
import lion.Message.GalleryPageQueryMessage;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class DeliveryService {
static String storagePath = "/root/gallery/gallery/";
//缓存排序后的页数
static LinkedHashMap<String, ArrayList<String>> pageCache;
static ExecutorService singleThreadPool = Executors.newSingleThreadExecutor();
static {
pageCache = new LinkedHashMap<>();
}
public static byte deliveryPreview(String name, short page, short port){
if(!pageCache.containsKey(name)){
byte result;
if((result = pageCache(name)) != 0)
return result;
}
//0页为缩略图
if(page == 0){
return delivery(new File(storagePath, name + "/thumbnail.webp"), port);
}else{
return delivery(new File(storagePath, name + "/" + pageCache.get(name).get(page)), port);
}
}
public static byte delivery(File file, short port){
if(!file.exists())
return ErrorCode.FILE_NOT_FOUND;
singleThreadPool.submit(() -> {
try(SocketChannel socketChannel = SocketChannel.open(new InetSocketAddress("lionwebsite.xyz", port));
FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
ByteBuffer buffer = ByteBuffer.allocate(1024);
while (fileChannel.read(buffer)!=-1){
buffer.flip();
socketChannel.write(buffer);
buffer.clear();
}
socketChannel.shutdownOutput();
return 0;
}catch (IOException e){
e.printStackTrace();
return ErrorCode.IO_ERROR;
}
});
return 0;
}
public static byte pageQuery(GalleryPageQueryMessage gpqm){
if(!pageCache.containsKey(gpqm.getName())){
byte result;
if((result = pageCache(gpqm.getName())) != 0)
return result;
}
ArrayList<String> pages = pageCache.get(gpqm.getName());
if(pages.size() <= gpqm.getPage())
return ErrorCode.FILE_NOT_FOUND;
gpqm.setPageName(pageCache.get(gpqm.getName()).get(gpqm.getPage()));
return 0;
}
public static byte pageCache(String name){
File directory = new File(storagePath, name);
if(!directory.isDirectory()) {
System.out.printf("文件夹%s没找到\n", directory.getAbsolutePath());
return ErrorCode.FILE_NOT_FOUND;
}
ArrayList<String> pageList = new ArrayList<>();
File[] files = directory.listFiles(((dir, name1) -> !name1.equals("galleryinfo.txt") && !name1.equals("thumbnail.webp") && !name1.endsWith(".zip")));
if(files == null)
return ErrorCode.FILE_NOT_FOUND;
ArrayList<File> fileArrayList = new ArrayList<>(Arrays.asList(files));
fileArrayList.sort(Comparator.naturalOrder());
pageList.add("thumbnail.webp");
fileArrayList.forEach((f) -> pageList.add(f.getName()));
pageCache.put(name, pageList);
return 0;
}
}

View File

@ -1,26 +1,25 @@
package lion.Service;
import io.netty.channel.Channel;
import io.netty.channel.DefaultEventLoop;
import io.netty.channel.EventLoop;
import io.netty.util.concurrent.Promise;
import lion.Domain.GalleryTask;
import lion.ErrorCode.ErrorCode;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ZipUtil;
import lombok.extern.log4j.Log4j;
import org.im4java.core.ConvertCmd;
import org.im4java.core.IM4JavaException;
import org.im4java.core.IMOperation;
import lion.Message.AbstractMessage;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.io.*;
import java.util.*;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
@Log4j
@Slf4j
@Data
public class DownloadCheckService {
ArrayList<GalleryTask> queue;
@ -30,13 +29,21 @@ public class DownloadCheckService {
ScheduledThreadPoolExecutor convert_thread;
ArrayList<GalleryTask> convert_queue;
ArrayList<GalleryTask> compress_queue;
public DownloadCheckService(ArrayList<GalleryTask> queue){
Channel node;
HashMap<Integer, Promise<AbstractMessage>> promises;
EventLoop eventLoop;
public DownloadCheckService(ArrayList<GalleryTask> queue, HashMap<Integer, Promise<AbstractMessage>> promises){
this.queue = queue;
convert_queue = new ArrayList<>(0);
this.promises = promises;
eventLoop = new DefaultEventLoop();
compress_queue = new ArrayList<>(0);
convert_thread = new ScheduledThreadPoolExecutor(1);
convert_thread.scheduleAtFixedRate(this::convert, 0, 5, TimeUnit.SECONDS);
convert_thread.scheduleAtFixedRate(this::compress, 0, 5, TimeUnit.SECONDS);
}
public boolean downloadCheck(){
@ -52,23 +59,35 @@ public class DownloadCheckService {
ArrayList<File> files = new ArrayList<>(Arrays.asList(fileArray));
boolean result = false;
//扫描进度
Iterator<File> fileIterator = files.iterator();
for(GalleryTask galleryTask: queue){
//跳过已经下载完成或者压缩完成的任务
if(galleryTask.is_compress_complete() || galleryTask.is_compressing()) {
result = true;
continue;
}
while(fileIterator.hasNext()){
File file = fileIterator.next();
if(!file.getName().contains(String.valueOf(galleryTask.getGid())))
continue;
galleryTask.setStatus(GalleryTask.DOWNLOADING);
fileIterator.remove();
galleryTask.setName(file.getName());
if(galleryTask.getName() == null || !galleryTask.getName().equals(file.getName()))
galleryTask.setName(file.getName());
File[] pages = file.listFiles((dir, name) -> !name.equals("galleryinfo.txt"));
if (pages == null || pages.length == 0)
continue;
if(galleryTask.getProceeding() != pages.length)
result = true;
galleryTask.setProceeding(pages.length);
if (new File(file.getPath(), "galleryinfo.txt").exists()) {
result = true;
galleryTask.setStatus(GalleryTask.DOWNLOAD_COMPLETE);
galleryTask.setPath(file.getPath());
}
@ -76,35 +95,28 @@ public class DownloadCheckService {
fileIterator = files.iterator();
}
//转格式队列
//压缩队列
for(GalleryTask galleryTask: queue)
if (galleryTask.is_download_complete())
convert_queue.add(galleryTask);
if (galleryTask.is_download_complete()) {
galleryTask.setStatus(GalleryTask.COMPRESSING);
compress_queue.add(galleryTask);
}
return true;
return result;
}
/**
* 转换线程将转换队列的任务复制一份进行转换
* 压缩线程将压缩队列的任务复制一份进行转换
*/
public void convert() {
if(convert_queue.isEmpty())
public void compress() {
if(compress_queue.isEmpty())
return;
ConvertCmd convertCmd = new ConvertCmd(true);
ReentrantLock reentrantLock = new ReentrantLock();
reentrantLock.lock();
ArrayList<GalleryTask> galleryTasks = new ArrayList<>(convert_queue);
convert_queue.clear();
ArrayList<GalleryTask> galleryTasks = new ArrayList<>(compress_queue);
compress_queue.clear();
reentrantLock.unlock();
for (GalleryTask galleryTask : galleryTasks) {
File[] images = new File(galleryTask.getPath()).listFiles((dir, name) -> name.endsWith(".jpg") || name.endsWith(".png"));
if (images == null) {
galleryTask.setStatus(ErrorCode.COMPRESS_ERROR);
continue;
}
//长度相同比较字典序否则比较长度
images = Arrays.stream(images).sorted(Comparator.naturalOrder()).toArray(File[]::new);
//创建文件夹
File file = new File(storagePath + galleryTask.getName());
if (file.isDirectory() || file.mkdirs()) {
@ -114,43 +126,12 @@ public class DownloadCheckService {
continue;
}
//thumbnail
IMOperation operation = new IMOperation();
operation.addImage(images[0].getAbsolutePath());
operation.resize(500, 500);
operation.format("webp");
operation.addImage(storagePath + galleryTask.getName() + "/thumbnail.webp");
try {
log.info("文件" + images[0].getName() + "转换为thumbnail.webp");
convertCmd.run(operation);
} catch (IOException | IM4JavaException | InterruptedException e) {
log.error("创建" + galleryTask.getName() + "缩略图失败");
galleryTask.setStatus(ErrorCode.COMPRESS_ERROR);
continue;
}
//生成压缩包
ZipUtil.zip(galleryTask.getPath(), storagePath + galleryTask.getName() + "/" + galleryTask.getName() + ".zip");
log.info(galleryTask.getName() + "压缩完成" );
if ((galleryTask.getType() & GalleryTask.DOWNLOAD_PREVIEW) != 0)
for (int i = 0; i < images.length; i++) {
log.info("文件" + images[i].getName() + "转换为webp[" + i + "/" + images.length + "]");
operation = new IMOperation();
operation.addImage(images[i].getAbsolutePath());
operation.format("webp");
operation.addImage(storagePath + galleryTask.getName() + "/" + images[i].getName().replace(".png", ".webp").replace(".jpg", ".webp"));
try {
convertCmd.run(operation);
} catch (IOException | InterruptedException | IM4JavaException e) {
log.error("文件" + images[i].getName() + "转换失败");
galleryTask.setStatus(ErrorCode.COMPRESS_ERROR);
break;
}
}
if ((galleryTask.getType() & GalleryTask.DOWNLOAD_SOURCE) != 0) {
ZipUtil.zip(galleryTask.getPath(), storagePath + galleryTask.getName() + "/" + galleryTask.getName() + ".zip");
log.info(galleryTask.getName() + "压缩完成" );
}
FileUtil.del(galleryTask.getPath());
galleryTask.setStatus(GalleryTask.DOWNLOAD_COMPLETE);
galleryTask.setStatus(GalleryTask.COMPRESS_COMPLETE);
}
}
}

View File

@ -1,18 +1,18 @@
package lion;
import io.netty.util.concurrent.Promise;
import lion.Domain.GalleryTask;
import lion.Message.*;
import lion.Message.Main.*;
import lion.Service.DeleteService;
import lion.Service.DeliveryService;
import lion.Service.DownloadCheckService;
import cn.hutool.core.io.FileUtil;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.*;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import lombok.extern.log4j.Log4j;
import lombok.extern.slf4j.Slf4j;
import java.io.FileOutputStream;
@ -20,17 +20,19 @@ import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.ArrayList;
import java.util.ListIterator;
import java.util.HashMap;
import java.util.concurrent.*;
import java.util.concurrent.locks.ReentrantLock;
@Log4j
@Slf4j
public class storageNode {
ChannelFuture channelFuture;
Channel server;
Channel node;
DownloadCheckService downloadCheckService;
ArrayList<GalleryTask> queue;
@ -39,6 +41,8 @@ public class storageNode {
ScheduledExecutorService checkThreadPool;
HashMap<Integer, Promise<AbstractMessage>> promises;
int counter;
ReentrantLock lock;
@ -50,6 +54,7 @@ public class storageNode {
tempQueue = new ArrayList<>(0);
lock = new ReentrantLock();
counter = 0;
promises = new HashMap<>();
channelFuture = new ServerBootstrap()
.channel(NioServerSocketChannel.class)
@ -60,17 +65,14 @@ public class storageNode {
channel.pipeline().addLast(new LengthFieldBasedFrameDecoder(100000000, 1, 4));
channel.pipeline().addLast(new MessageCodec());
channel.pipeline().addLast(new MyChannelInboundHandlerAdapter(tempQueue));
}
}).option(ChannelOption.SO_BACKLOG, 128)
.childOption(ChannelOption.SO_KEEPALIVE, true)
})
.bind(26321);
log.info("listen port:8080");
try(Socket socket = new Socket()) {
try(Socket socket = new Socket()){
socket.connect(new InetSocketAddress("lionwebsite.xyz", 26322));
} catch (Exception ignored) {}
downloadCheckService = new DownloadCheckService(queue);
downloadCheckService = new DownloadCheckService(queue, promises);
checkThreadPool = Executors.newScheduledThreadPool(1);
checkThreadPool.scheduleAtFixedRate(this::mainThread, 5, 5, TimeUnit.SECONDS);
}
@ -83,29 +85,35 @@ public class storageNode {
tempQueue.clear();
}
lock.unlock();
//检查
//检查当任务状态发生变化即方法返回true时再更新否则return
if (!downloadCheckService.downloadCheck()) {
counter++;
if(server != null && server.isActive() && counter > 10) {
server.writeAndFlush(new MaintainMessage());
counter = 0;
boolean isSkip = true;
//返回false之后还要额外检查是否有压缩完成的任务
if(!queue.isEmpty())
for (GalleryTask galleryTask : queue)
if (galleryTask.is_compress_complete()) {
isSkip = false;
break;
}
if(isSkip) {
counter++;
if (server != null && server.isActive() && counter > 10) {
server.writeAndFlush(new MaintainMessage());
counter = 0;
}
return;
}
return;
}
//发送
//上锁后再发送避免出现发送完之后再下载完成
lock.lock();
DownloadStatusMessage downloadStatusMessage = new DownloadStatusMessage();
downloadStatusMessage.setGalleryTasks(queue.toArray(GalleryTask[]::new));
server.writeAndFlush(downloadStatusMessage);
ListIterator<GalleryTask> listIterator = queue.listIterator();
lock.lock();
while (listIterator.hasNext()) {
GalleryTask galleryTask = listIterator.next();
if (galleryTask.is_download_complete())
listIterator.remove();
}
lock.unlock();
log.info("任务状态发送完成");
queue.removeIf(GalleryTask::is_compress_complete);
lock.unlock();
}catch (Exception e){
e.printStackTrace();
try (OutputStream outputStream = new FileOutputStream("/root/gallery/storageNode/err.txt")){
@ -122,66 +130,45 @@ public class storageNode {
ArrayList<GalleryTask> queue;
public MyChannelInboundHandlerAdapter(ArrayList<GalleryTask> queue) {
super();
this.queue = queue;
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
System.out.println(msg);
AbstractMessage abstractMessage = (AbstractMessage) msg;
if(msg instanceof IdentityMessage) {
server = ctx.channel();
log.info("server 上线");
//提交下载
}else if(msg instanceof DownloadPostMessage dpm){
lock.lock();
queue.add(dpm.getGalleryTask());
System.out.println(queue);
lock.unlock();
ctx.writeAndFlush(new ResponseMessage(dpm.messageId, (byte) 0));
//删除本子 全部/预览/源文件
}else if(msg instanceof DeleteGalleryMessage deleteGalleryMessage){
byte result = switch (deleteGalleryMessage.getDeleteType()){
case DeleteGalleryMessage.DELETE_ALL ->
DeleteService.deleteAll(storagePath + deleteGalleryMessage.getGalleryName());
case DeleteGalleryMessage.DELETE_PREVIEW ->
DeleteService.deletePreview(storagePath + deleteGalleryMessage.getGalleryName());
case DeleteGalleryMessage.DELETE_SOURCE ->
DeleteService.deleteSource(storagePath + deleteGalleryMessage.getGalleryName());
default -> -1;
};
ResponseMessage responseMessage = new ResponseMessage(deleteGalleryMessage.messageId, result);
ctx.writeAndFlush(responseMessage);
//请求预览
}else if(msg instanceof GalleryRequestMessage grm){
byte result = DeliveryService.deliveryPreview(grm.getGalleryName(), grm.getPage(), grm.getPort());
ResponseMessage responseMessage = new ResponseMessage();
responseMessage.messageId = grm.messageId;
responseMessage.setResult(result);
ctx.writeAndFlush(responseMessage);
//更新本子 删除原有文件然后加入到队列
}else if(msg instanceof UpdateGalleryMessage ugm){
FileUtil.del(storagePath + ugm.getGalleryTask().getName());
lock.lock();
queue.add(ugm.getGalleryTask());
System.out.println(queue);
lock.unlock();
ctx.writeAndFlush(new ResponseMessage(ugm.messageId, (byte) 0));
}else if(msg instanceof GalleryPageQueryMessage gpqm){
byte result = DeliveryService.pageQuery(gpqm);
gpqm.setResult(result);
ctx.writeAndFlush(gpqm);
switch (abstractMessage.messageType){
case AbstractMessage.IDENTITY_MESSAGE -> {
IdentityMessage identityMessage = (IdentityMessage) abstractMessage;
if(identityMessage.getIdentity().equals("lionwebsite")) {
server = ctx.channel();
log.info("server 上线");
} else if(identityMessage.getIdentity().equals("lionwebsiteside")){
node = ctx.channel();
log.info("node上线");
downloadCheckService.setNode(node);
}
}
case AbstractMessage.DOWNLOAD_POST_MESSAGE -> {
DownloadPostMessage dpm = (DownloadPostMessage) abstractMessage;
lock.lock();
queue.add(dpm.getGalleryTask());
System.out.println(queue);
lock.unlock();
ctx.writeAndFlush(new ResponseMessage(dpm.messageId, (byte) 0));
}
case AbstractMessage.DELETE_GALLERY_MESSAGE -> {
DeleteGalleryMessage deleteGalleryMessage = (DeleteGalleryMessage) abstractMessage;
byte result = DeleteService.deleteAll(storagePath + deleteGalleryMessage.getGalleryName());
ResponseMessage responseMessage = new ResponseMessage(deleteGalleryMessage.messageId, result);
ctx.writeAndFlush(responseMessage);
}
}
//修复预览
//重新生成压缩包
//
// //修复预览
//
// //重新生成压缩包
}
@Override
@ -189,6 +176,10 @@ public class storageNode {
if(ctx.channel().equals(server)) {
log.info("server 下线");
server = null;
} else if(ctx.channel().equals(node)){
log.info("node 下线");
node = null;
downloadCheckService.setNode(null);
}
}
}

View File

@ -0,0 +1,74 @@
[
{
"name": "lion.Message.Main.DeleteGalleryMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.Main.DownloadPostMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.Main.DownloadStatusMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.Main.IdentityMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.Main.MaintainMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.Main.ResponseMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Message.AbstractMessage",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
},
{
"name": "lion.Domain.GalleryTask",
"allDeclaredConstructors" : true,
"allPublicConstructors" : true,
"allDeclaredMethods" : true,
"allPublicMethods" : true,
"allDeclaredFields" : true,
"allPublicFields" : true
}
]