lywsvip/ChopperBot
1
0
Fork 0
mirror of https://github.com/Geniusay/ChopperBot.git synced 2026-05-30 02:20:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

爬虫请求分发类（TaskHandler）：其他模块对爬虫模块发送请求（并不是网络请求，就是调用一个方法），请求分发类根据请求类型确定爬虫任务并塞入爬虫LoadTask

Browse Source 
爬虫任务中心（TaskCenter）：爬虫任务中心有如下功能：1，记录爬虫任务 2，执行爬虫任务 3，恢复异常关闭导致的爬虫任务 4，爬虫日志（也是恢复日志）
爬虫模块初始化：爬虫模块已经加入模块初始化队列中，目前的插件（爬虫配置文件，爬虫任务中心）
InitMachine：InitMachine又添加新方法，afterInit在初始化后执行的方法
其他更新（更新日志细说）：各种配置文件，一些工具类，一些爬虫方法更新和优化
This commit is contained in:
userA
2023-07-29 03:01:54 +08:00
parent a4c7d704f8
commit d274d48423
72 changed files with 1346 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
FileModule/src/main/java/org/example/cache/FileCacheManager.java vendored
View File

@@ -1,14 +1,11 @@
package org.example.cache;
import org.example.log.FileModuleLogger;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.util.TimeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import static java.lang.Thread.sleep;
@@ -75,7 +72,7 @@ public class FileCacheManager {
fileCaches.add(fileCache);
fileCacheMap.put(fileCache.getFullFilePath(),fileCache);
initSleepTime();
FileModuleLogger.logger.debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath());
ChopperLogFactory.getLogger(LoggerType.File).debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath());
}
return false;
}
@@ -84,6 +81,10 @@ public class FileCacheManager {
return fileCacheMap.get(filePath);
}
public boolean deleteFileCache(String filePath){
return fileCaches.remove(fileCacheMap.get(filePath)) && fileCacheMap.remove(filePath)!=null;
}
public List<FileCache> getRunnableFileCaches(){
return this.fileCaches;
}
@@ -98,7 +99,7 @@ public class FileCacheManager {
BlockingQueue fileChannel = cache.getFileChannel();
if(fileChannel.isEmpty()){
if(cache.needAutoSync()){
FileModuleLogger.logger.debug("检测到需要强制刷新的文件 {}",cache.getFileName());
ChopperLogFactory.getLogger(LoggerType.File).debug("检测到需要强制刷新的文件 {}",cache.getFileName());
autoSyncer.submit(new AutoSyncer(cache));
}
}

39
FileModule/src/main/java/org/example/init/ConfigInitMachine.java Normal file
View File

@@ -0,0 +1,39 @@
package org.example.init;
import org.example.bean.ConfigFile;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.exception.FileCacheException;
import org.example.util.ConfigFileUtil;
import org.example.util.FileUtil;
import org.example.util.JsonFileUtil;
import org.slf4j.Logger;
import java.nio.file.Paths;
import java.util.Map;
/**
* @author Genius
* @date 2023/07/29 00:08
**/
public abstract class ConfigInitMachine<T extends ConfigFile> extends CommonInitMachine{
private T configFile;
private String filePath;
public ConfigInitMachine(T configFile, Logger logger) {
super(logger);
this.configFile = configFile;
filePath = Paths.get(configFile.getFilePath(), configFile.getFileName()).toString();
}
@Override
public boolean init() {
return ConfigFileUtil.createConfigFile(filePath,configFile,logger,this.getClass().getName());
}
}

6
FileModule/src/main/java/org/example/init/FileCacheManagerInit.java
View File

@@ -1,8 +1,8 @@
package org.example.init;
import org.example.cache.FileCacheManagerInstance;
import org.example.log.FileModuleLogger;
import org.slf4j.Logger;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
/**
* @author Genius
@@ -11,7 +11,7 @@ import org.slf4j.Logger;
public class FileCacheManagerInit extends CommonInitMachine{
public FileCacheManagerInit() {
super(FileModuleLogger.logger);
super( ChopperLogFactory.getLogger(LoggerType.File));
}
@Override

6
FileModule/src/main/java/org/example/init/FileModuleInitMachine.java
View File

@@ -1,7 +1,7 @@
package org.example.init;
import org.example.log.FileModuleLogger;
import org.slf4j.Logger;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import java.util.List;
@@ -15,7 +15,7 @@ public class FileModuleInitMachine extends ModuleInitMachine{
super(List.of(
new ModuleSrcConfigFileInit(),
new FileCacheManagerInit()
), "FileModule", FileModuleLogger.logger);
), "FileModule", ChopperLogFactory.getLogger(LoggerType.File));
}
}

8
FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java
View File

@@ -1,11 +1,11 @@
package org.example.init;
import org.example.log.FileModuleLogger;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.pojo.configfile.ModuleSrcConfigFile;
import org.example.util.FileUtil;
import org.example.util.JsonFileUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -21,7 +21,7 @@ public class ModuleSrcConfigFileInit extends CommonInitMachine {
ModuleSrcConfigFile moduleSrcConfigFile;
public ModuleSrcConfigFileInit() {
super(FileModuleLogger.logger);
super( ChopperLogFactory.getLogger(LoggerType.File));
moduleSrcConfigFile = new ModuleSrcConfigFile();
}

51
FileModule/src/main/java/org/example/util/ConfigFileUtil.java Normal file
View File

@@ -0,0 +1,51 @@
package org.example.util;
import org.example.bean.ConfigFile;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.exception.FileCacheException;
import org.slf4j.Logger;
import java.util.Map;
/**
* @author Genius
* @date 2023/07/29 01:40
**/
public class ConfigFileUtil {
public static boolean createConfigFile(String filePath, ConfigFile configFile, Logger logger,String pluginName,boolean isNeedLog) {
if(!FileUtil.isFileExist(filePath)){
JsonFileUtil.writeJsonFile(filePath,configFile.packageConfig());
try {
FileCache fileCache = new FileCache(configFile);
FileCacheManagerInstance.getInstance().addFileCache(fileCache);
} catch (FileCacheException e) {
if(isNeedLog) logger.error(e.toString());
return false;
}
if(isNeedLog) logger.info(String.format("[✔]%s is created,{%s} plugin init success!", filePath, pluginName));
return true;
}else{
Map<String, Object> data = JsonFileUtil.readJsonFile(filePath);
configFile.setData(data);
try {
FileCache fileCache = new FileCache(configFile);
FileCacheManagerInstance.getInstance().addFileCache(fileCache);
} catch (FileCacheException e) {
if(isNeedLog) logger.error(e.toString());
return false;
}
}
if(isNeedLog) logger.info(String.format("[✔]%s read in disk,{%s} plugin init success!", filePath, pluginName));
return true;
}
public static boolean createConfigFile(String filePath, ConfigFile configFile, Logger logger,String pluginName){
return createConfigFile(filePath,configFile,logger,pluginName,true);
}
public static boolean createConfigFile(String filePath, ConfigFile configFile){
return createConfigFile(filePath,configFile,null,null,false);
}
}