diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..604ee66 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,12 @@ + + + + + mysql.8 + true + com.mysql.cj.jdbc.Driver + jdbc:mysql://62.234.54.176:3306/rsamodel + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/CreeperModule/src/main/java/org/example/config/CreeperConfigFile.java b/CreeperModule/src/main/java/org/example/config/CreeperConfigFile.java new file mode 100644 index 0000000..1eb3f80 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/config/CreeperConfigFile.java @@ -0,0 +1,33 @@ +package org.example.config; + +import lombok.AllArgsConstructor; +import org.example.bean.ConfigFile; +import org.example.bean.FileType; +import org.example.constpool.CreeperModuleConstPool; + +import java.nio.file.Paths; +import java.util.Map; + +/** + * @author Genius + * @date 2023/07/28 23:54 + **/ +public class CreeperConfigFile extends ConfigFile> { + + private static final String filePath = CreeperModuleConstPool.CREEPER_ROOT; + + private static final String fileName = "creeperConfig.json"; + + public CreeperConfigFile() { + super(filePath, fileName, + Map.of("taskCenter",new TaskCenterConfig(10,50,1000)), + FileType.CREEPER); + } + + + + public static String getFullFilePath(){ + return Paths.get(filePath,fileName).toString(); + } + +} diff --git a/CreeperModule/src/main/java/org/example/config/CreeperLogConfigFile.java b/CreeperModule/src/main/java/org/example/config/CreeperLogConfigFile.java new file mode 100644 index 0000000..e41c6a4 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/config/CreeperLogConfigFile.java @@ -0,0 +1,34 @@ +package org.example.config; + +import org.example.bean.ConfigFile; +import org.example.bean.FileType; +import org.example.constpool.CreeperModuleConstPool; +import org.example.taskcenter.task.ReptileTask; +import org.example.util.TimeUtil; + +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +/** + * @author Genius + * @date 2023/07/29 00:03 + **/ +public class CreeperLogConfigFile extends ConfigFile>> { + private static final String filePath = CreeperModuleConstPool.CREEPER_ROOT +"/log"; + + public CreeperLogConfigFile(List data) { + super(filePath,"creeper-"+ TimeUtil.getToday_YMD()+".log" , + Map.of("task",data) + , FileType.LOG); + } + + //检查是不是需要更新日志 + public static boolean needNewDayLog(String fileName){ + return fileName.equals("creeper-"+ TimeUtil.getToday_YMD()+".log"); + } + + public static String getFullFilePath(){ + return Paths.get(filePath,"creeper-"+ TimeUtil.getToday_YMD()+".log").toString(); + } +} diff --git a/CreeperModule/src/main/java/org/example/config/TaskCenterConfig.java b/CreeperModule/src/main/java/org/example/config/TaskCenterConfig.java new file mode 100644 index 0000000..759f4e9 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/config/TaskCenterConfig.java @@ -0,0 +1,16 @@ +package org.example.config; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @author Genius + * @date 2023/07/29 01:26 + **/ +@Data +@AllArgsConstructor +public class TaskCenterConfig{ + private int threads; + private int queueCapacity; + private int waitingTime; +} diff --git a/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java b/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java index b03075d..9615301 100644 --- a/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java +++ b/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java @@ -13,6 +13,8 @@ public class CreeperModuleConstPool { //弹幕保存文件的根目录 public static final String BARRAGE_ROOT = (String)GlobalFileCache.ModuleSrcConfigFile.get("barrage","src"); + public static final String CREEPER_ROOT = (String)GlobalFileCache.ModuleSrcConfigFile.get("creeper","src"); + /** * 下载的方式 */ diff --git a/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java index 04a8e83..0d98a47 100644 --- a/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java @@ -1,6 +1,7 @@ package org.example.core.control; -import org.example.log.HotModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import us.codecraft.webmagic.ResultItems; import us.codecraft.webmagic.Spider; @@ -8,7 +9,7 @@ import us.codecraft.webmagic.Spider; * @author Genius * @date 2023/07/21 10:22 **/ -public abstract class HotModuleLoadTask{ +public abstract class HotModuleLoadTask implements LoadTask{ public enum FinishFlag{ FINISH,NOT_FINISH,FAIL } @@ -23,7 +24,7 @@ public abstract class HotModuleLoadTask{ protected abstract T start0(); protected void fail(Exception e){ finishFlag = FinishFlag.FAIL; - HotModuleLogger.logger.error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage()); + ChopperLogFactory.getLogger(LoggerType.Hot).error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage()); } protected void success(){ @@ -41,4 +42,24 @@ public abstract class HotModuleLoadTask{ } public void clearFinishFlag(){finishFlag = FinishFlag.NOT_FINISH;} + + @Override + public void end() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public int getCacheSize() { + return 0; + } + + @Override + public int flushCacheAndSave(String key) { + return 0; + } } diff --git a/CreeperModule/src/main/java/org/example/core/control/LoadTask.java b/CreeperModule/src/main/java/org/example/core/control/LoadTask.java index dd8bb0a..7f9d0d5 100644 --- a/CreeperModule/src/main/java/org/example/core/control/LoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/LoadTask.java @@ -5,10 +5,10 @@ package org.example.core.control; * @author 燧枫 * @date 2023/4/23 18:14 */ -public interface LoadTask { +public interface LoadTask { // 开始爬虫 - void start(); + T start(); // 结束爬虫 void end(); diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java index 1a071fc..7be2402 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java @@ -31,7 +31,7 @@ public class BilibiliLiveLoadTask implements LoadTask { } @Override - public void start() { + public Object start() { Spider.create(bilibiliLiveProcessor) // 设置起始Request .addRequest(new Request(CreeperModuleConstPool.OCCUURL)) @@ -42,6 +42,7 @@ public class BilibiliLiveLoadTask implements LoadTask { .setEmptySleepTime(emptySleepTime) // 开始抓取 .start(); + return null; } @Override diff --git a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java similarity index 69% rename from CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java rename to CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java index 10386cc..888bbf7 100644 --- a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java @@ -1,10 +1,9 @@ -package org.example.core.control.hotmodule; +package org.example.core.control.impl; -import org.example.bean.HotLive; +import org.example.bean.Live; +import org.example.bean.hotmodule.DouyuLive; import org.example.core.control.HotModuleLoadTask; -import org.example.core.control.LoadTask; import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; -import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Spider; import java.util.List; @@ -15,7 +14,7 @@ import static org.example.constpool.ApiPool.*; * @author Genius * @date 2023/07/19 02:42 **/ -public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { +public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { @@ -29,7 +28,7 @@ public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { * 获取Douyu某个模块下的热门直播 * @param moduleId */ - public List start(int moduleId){ + public List start(int moduleId){ clearFinishFlag(); douyuHotLiveProcessor.setModuleId(moduleId); return this.start(String.format(DOUYU_HOT_MODULE_LIVES_API,moduleId)); @@ -39,12 +38,12 @@ public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { * 获取Douyu当前最热直播 */ @Override - protected List start0() { + protected List start0() { return this.start(DOUYU_HOT_LIVES_API); } - private List start(String url){ - List lives; + private List start(String url){ + List lives; try { lives = getData(Spider.create(douyuHotLiveProcessor),url); }catch (Exception e){ @@ -55,9 +54,4 @@ public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { return lives; } - public static void main(String[] args) { - List start = new DouyuHotLiveLoadTask().start(); - System.out.println(start); - } - } diff --git a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java similarity index 95% rename from CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java rename to CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java index 21449f1..fa5146a 100644 --- a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java @@ -1,4 +1,4 @@ -package org.example.core.control.hotmodule; +package org.example.core.control.impl; import org.example.bean.hotmodule.HotModuleList; import org.example.core.control.HotModuleLoadTask; diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuLiveOnlineLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuLiveOnlineLoadTask.java new file mode 100644 index 0000000..fd05bdf --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuLiveOnlineLoadTask.java @@ -0,0 +1,49 @@ +package org.example.core.control.impl; + +import org.example.bean.hotmodule.DouyuLive; +import org.example.core.control.LoadTask; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.pojo.download.assign.DouyuLiveOnlineConfig; +import org.slf4j.LoggerFactory; + +/** + * @author Genius + * @date 2023/07/28 23:14 + **/ +public class DouyuLiveOnlineLoadTask implements LoadTask { + + + private DouyuLive douyuLive; + public DouyuLiveOnlineLoadTask(DouyuLiveOnlineConfig douyuLiveOnlineConfig) { + this.douyuLive = douyuLiveOnlineConfig.getLive(); + } + + //TODO 需要开发斗鱼在线直播爬取功能 + @Override + public Object start() { + ChopperLogFactory + .getLogger(LoggerType.Creeper).info("正在爬取主播：{},直播间:{},直播间id:{}",douyuLive.getLiver(),douyuLive.getLiveName(),douyuLive.getLiveId()); + return null; + } + + @Override + public void end() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public int getCacheSize() { + return 0; + } + + @Override + public int flushCacheAndSave(String key) { + return 0; + } +} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java index 50097f4..2f047e8 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java @@ -31,7 +31,7 @@ public class DouyuRecordLoadTask implements LoadTask { } @Override - public void start() { + public Object start() { Spider.create(douyuRecordProcessor) // 设置起始Request .addRequest(new Request(CreeperModuleConstPool.OCCUURL)) @@ -42,6 +42,7 @@ public class DouyuRecordLoadTask implements LoadTask { .setEmptySleepTime(emptySleepTime) // 开始抓取 .start(); + return null; } @Override diff --git a/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java b/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java index c967ece..4193962 100644 --- a/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java +++ b/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java @@ -1,12 +1,11 @@ package org.example.core.factory; import org.example.core.control.LoadTask; -import org.example.core.control.impl.BilibiliLiveLoadTask; -import org.example.core.control.impl.DouyuRecordLoadTask; +import org.example.core.control.impl.*; import org.example.exception.FileCacheException; -import org.example.pojo.download.LoadBarrageConfig; -import org.example.pojo.download.assign.BilibiliLiveLoadBarrageConfig; -import org.example.pojo.download.assign.DouyuRecordLoadBarrageConfig; +import org.example.pojo.download.assign.*; +import org.example.pojo.download.LoadConfig; + /** * 弹幕下载任务工厂 @@ -17,22 +16,33 @@ public class LoadTaskFactory { /** * 通过配置信息来获取一个任务 - * @param loadBarrageConfig + * @param loadConfig * @return LoadTask */ - public LoadTask getLoadTask(LoadBarrageConfig loadBarrageConfig) throws FileCacheException { + public static LoadTask getLoadTask(LoadConfig loadConfig){ - if (loadBarrageConfig == null) { + if (loadConfig == null) { return null; } // 斗鱼录播 - if (loadBarrageConfig instanceof DouyuRecordLoadBarrageConfig) { - return new DouyuRecordLoadTask((DouyuRecordLoadBarrageConfig) loadBarrageConfig); + if (loadConfig instanceof DouyuRecordLoadBarrageConfig) { + return new DouyuRecordLoadTask((DouyuRecordLoadBarrageConfig) loadConfig); } // B站直播 - else if (loadBarrageConfig instanceof BilibiliLiveLoadBarrageConfig) { - return new BilibiliLiveLoadTask((BilibiliLiveLoadBarrageConfig) loadBarrageConfig); + else if (loadConfig instanceof BilibiliLiveLoadBarrageConfig) { + return new BilibiliLiveLoadTask((BilibiliLiveLoadBarrageConfig) loadConfig); + } + // 斗鱼热门模块 + else if(loadConfig instanceof DouyuHotModuleConfig){ + return new DouyuHotModuleLoadTask(); + // 斗鱼热门直播 + }else if(loadConfig instanceof DouyuHotLiveConfig){ + return new DouyuHotLiveLoadTask(); + } + // 斗鱼在线直播 + else if(loadConfig instanceof DouyuLiveOnlineConfig){ + return new DouyuLiveOnlineLoadTask((DouyuLiveOnlineConfig) loadConfig); } return null; diff --git a/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java b/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java index 73de346..6d6c432 100644 --- a/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java +++ b/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java @@ -19,18 +19,15 @@ import java.util.stream.Collectors; public class LoadTaskManager { private final ConcurrentHashMap taskMap; - private final LoadTaskFactory loadTaskFactory; public LoadTaskManager() { this.taskMap = new ConcurrentHashMap<>(); // 默认任务工场 - this.loadTaskFactory = new LoadTaskFactory(); } public LoadTaskManager(LoadTaskFactory customLoadTaskFactory) { this.taskMap = new ConcurrentHashMap<>(); // 自定义任务工场 - this.loadTaskFactory = customLoadTaskFactory; } // 创建一个任务并返回唯一的 key @@ -39,7 +36,7 @@ public class LoadTaskManager { String key = generateKey(loadBarrageConfig); // 创建任务并将其添加到任务映射中 - LoadTask loadTask = loadTaskFactory.getLoadTask(loadBarrageConfig); + LoadTask loadTask = LoadTaskFactory.getLoadTask(loadBarrageConfig); taskMap.put(key, loadTask); // 返回生成的 key diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java index b8aca9c..c73002c 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java @@ -3,8 +3,8 @@ package org.example.core.processor.hotmodule; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import org.example.bean.HotLive; -import org.example.bean.hotmodule.DouyuHotLive; +import org.example.bean.Live; +import org.example.bean.hotmodule.DouyuLive; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.processor.PageProcessor; @@ -29,14 +29,14 @@ public class DouyuHotLiveProcessor implements PageProcessor { @Override public void process(Page page) { - List hotLiveList = new ArrayList<>(); + List liveList = new ArrayList<>(); try{ JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONObject("data").getJSONArray("rl"); for (Object live : Lives) { if(live instanceof JSONObject){ JSONObject jsonLive = (JSONObject) live; - hotLiveList.add(new DouyuHotLive( + liveList.add(new DouyuLive( jsonLive.getInteger("ol"), jsonLive.getInteger("rid"), jsonLive.getString("rn"), @@ -54,7 +54,7 @@ public class DouyuHotLiveProcessor implements PageProcessor { }catch (Exception e){ throw e; } - page.putField("data",hotLiveList); + page.putField("data", liveList); } } diff --git a/CreeperModule/src/main/java/org/example/init/CreeperConfigInitMachine.java b/CreeperModule/src/main/java/org/example/init/CreeperConfigInitMachine.java new file mode 100644 index 0000000..613db97 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/init/CreeperConfigInitMachine.java @@ -0,0 +1,37 @@ +package org.example.init; + +import org.example.config.CreeperConfigFile; +import org.example.constpool.CreeperModuleConstPool; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.util.FileUtil; +import org.slf4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author Genius + * @date 2023/07/29 01:56 + **/ +public class CreeperConfigInitMachine extends ConfigInitMachine { + + public CreeperConfigInitMachine() { + super(new CreeperConfigFile(), ChopperLogFactory.getLogger(LoggerType.Creeper)); + } + + @Override + public boolean init() { + try { + Path path = Paths.get(CreeperModuleConstPool.CREEPER_ROOT, "log"); + if(!FileUtil.isFileExist(path.toString())){ + Files.createDirectory(path); + } + }catch (IOException e) { + return false; + } + return super.init(); + } +} diff --git a/CreeperModule/src/main/java/org/example/init/CreeperModuleInitMachine.java b/CreeperModule/src/main/java/org/example/init/CreeperModuleInitMachine.java new file mode 100644 index 0000000..ff3362a --- /dev/null +++ b/CreeperModule/src/main/java/org/example/init/CreeperModuleInitMachine.java @@ -0,0 +1,24 @@ +package org.example.init; + +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/29 01:58 + **/ +public class CreeperModuleInitMachine extends ModuleInitMachine{ + + public CreeperModuleInitMachine() { + super(List.of( + new CreeperConfigInitMachine(), + new TaskCenterInitMachine() + ), "Creeper", ChopperLogFactory.getLogger(LoggerType.Creeper)); + } + + +} diff --git a/CreeperModule/src/main/java/org/example/init/TaskCenterInitMachine.java b/CreeperModule/src/main/java/org/example/init/TaskCenterInitMachine.java new file mode 100644 index 0000000..52ff58e --- /dev/null +++ b/CreeperModule/src/main/java/org/example/init/TaskCenterInitMachine.java @@ -0,0 +1,43 @@ +package org.example.init; + +import org.example.taskcenter.TaskCenter; +import org.example.thread.oddjob.OddJobBoy; + +import java.util.Objects; + +/** + * @author Genius + * @date 2023/07/28 23:49 + **/ +public class TaskCenterInitMachine extends CommonInitMachine{ + + @Override + public boolean init() { + try { + TaskCenter center = TaskCenter.center(); + if(center==null){ + return false; + } + OddJobBoy.Boy().addWork( + center::work + ); + }catch (InterruptedException e){ + return false; + } + return true; + } + + @Override + public void shutdown() { + TaskCenter center = TaskCenter.center(); + if(center!=null){ + center.shutdown(); + } + } + + @Override + public void afterInit() { + //爬虫任务恢复 + Objects.requireNonNull(TaskCenter.center()).restoreTaskCenter(); + } +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java index 0bc2b74..af997fc 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java @@ -8,8 +8,7 @@ import org.example.utils.FormatUtil; * @author 燧枫 * @date 2023/4/23 16:03 */ -@Data -public class LoadBarrageConfig { +public class LoadBarrageConfig extends LoadConfig{ // 爬取的平台 protected String platform; @@ -21,12 +20,34 @@ public class LoadBarrageConfig { protected String anchorName; // 开始时间 - protected String startTime; public LoadBarrageConfig(String platform, String action, String anchorName) { this.platform = platform; this.action = action; this.anchorName = anchorName; - this.startTime = FormatUtil.getNowDate(); + } + + public String getPlatform() { + return platform; + } + + public void setPlatform(String platform) { + this.platform = platform; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public String getAnchorName() { + return anchorName; + } + + public void setAnchorName(String anchorName) { + this.anchorName = anchorName; } } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java new file mode 100644 index 0000000..f8859af --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java @@ -0,0 +1,26 @@ +package org.example.pojo.download; + +import org.example.utils.FormatUtil; + +/** + * @author Genius + * @date 2023/07/26 15:25 + **/ +public abstract class LoadConfig { + + protected String startTime; + + public LoadConfig() { + this.startTime = FormatUtil.getNowDate(); + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java index 5042732..1154a63 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java @@ -4,5 +4,7 @@ package org.example.pojo.download; * @author Genius * @date 2023/06/01 22:18 **/ -public class LoadHotModuleConfig { +public class LoadHotModuleConfig extends LoadConfig{ + public LoadHotModuleConfig() { + } } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadLiveConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadLiveConfig.java new file mode 100644 index 0000000..103ff27 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadLiveConfig.java @@ -0,0 +1,24 @@ +package org.example.pojo.download; + +import org.example.bean.Live; +import org.example.pojo.download.LoadConfig; + +/** + * @author Genius + * @date 2023/07/28 23:16 + **/ +public class LoadLiveConfig extends LoadConfig { + private T live; + + public LoadLiveConfig(T live) { + this.live = live; + } + + public T getLive() { + return live; + } + + public void setLive(T live) { + this.live = live; + } +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotLiveConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotLiveConfig.java new file mode 100644 index 0000000..28a30a0 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotLiveConfig.java @@ -0,0 +1,12 @@ +package org.example.pojo.download.assign; + +import org.example.pojo.download.LoadHotModuleConfig; + +/** + * @author Genius + * @date 2023/07/26 15:33 + **/ +public class DouyuHotLiveConfig extends LoadHotModuleConfig { + public DouyuHotLiveConfig() { + } +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java index c77b9a5..0472262 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java @@ -1,12 +1,14 @@ package org.example.pojo.download.assign; import lombok.Data; +import org.example.pojo.download.LoadHotModuleConfig; /** * @author Genius * @date 2023/06/01 22:16 **/ -@Data -public class DouyuHotModuleConfig { +public class DouyuHotModuleConfig extends LoadHotModuleConfig { + public DouyuHotModuleConfig() { + } } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuLiveOnlineConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuLiveOnlineConfig.java new file mode 100644 index 0000000..f8e0cf9 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuLiveOnlineConfig.java @@ -0,0 +1,16 @@ +package org.example.pojo.download.assign; + +import org.example.bean.Live; +import org.example.bean.hotmodule.DouyuLive; +import org.example.pojo.download.LoadLiveConfig; + +/** + * @author Genius + * @date 2023/07/28 23:17 + **/ +public class DouyuLiveOnlineConfig extends LoadLiveConfig { + + public DouyuLiveOnlineConfig(DouyuLive live) { + super(live); + } +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/TaskCenter.java b/CreeperModule/src/main/java/org/example/taskcenter/TaskCenter.java new file mode 100644 index 0000000..03ca13b --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/TaskCenter.java @@ -0,0 +1,200 @@ +package org.example.taskcenter; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import org.example.cache.FileCache; +import org.example.cache.FileCacheManagerInstance; +import org.example.config.CreeperConfigFile; +import org.example.config.CreeperLogConfigFile; +import org.example.config.TaskCenterConfig; +import org.example.exception.FileCacheException; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.taskcenter.handler.BootStrapTaskHandler; +import org.example.taskcenter.request.ReptileRequest; +import org.example.taskcenter.task.ReptileTask; +import org.example.util.ConfigFileUtil; +import org.example.util.FileUtil; +import org.example.util.JsonFileUtil; + +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author Genius + * @date 2023/07/28 21:57 + **/ + +/** + * 爬虫任务中心包含了以下功能 + * 1，请求分发和包装爬虫任务 + * 2，记录正在运行的爬虫任务 + * 3，未完成的爬虫任务修复 + */ +public class TaskCenter { + + private long waitingQueueTime; //等待队列时间 + + private int threads; //处理任务的线程数量 + + private volatile static TaskCenter taskCenter; + + private Map runningTask; //正在运行的任务 + + private BlockingQueue waitingTask; //正在等待运行的任务 + + private ExecutorService taskPool; //任务池子 + + private BootStrapTaskHandler bootStrapTaskHandler; // 请求处理封装类 + + private FileCache creeperLogFileCache; //爬虫日志文件 + + private ReentrantLock lock = new ReentrantLock(); + + + public TaskCenter() { + } + + private TaskCenter(int waitingQueueTime, int threads, int capacity){ + this.runningTask = new ConcurrentHashMap<>(); + this.waitingTask = new ArrayBlockingQueue<>(capacity); + this.bootStrapTaskHandler = new BootStrapTaskHandler(); + this.taskPool = Executors.newFixedThreadPool(threads); + this.waitingQueueTime = waitingQueueTime; + this.threads = threads; + } + + public static TaskCenter center(){ + if(taskCenter==null){ + synchronized (TaskCenter.class){ + if(taskCenter==null){ + if (!init()) { + return null; + } + } + } + } + return taskCenter; + } + + private static boolean init(){ + FileCache configFileCache = FileCacheManagerInstance.getInstance().getFileCache(CreeperConfigFile.getFullFilePath()); + Object obj = configFileCache.get("taskCenter"); + if (obj!=null) { + TaskCenterConfig taskCenterConfig = JSONObject.parseObject(obj.toString(), TaskCenterConfig.class); + taskCenter = new TaskCenter( + taskCenterConfig.getWaitingTime(), + taskCenterConfig.getThreads(), + taskCenterConfig.getQueueCapacity() + ); + return taskCenter.newLogFile(new CreeperLogConfigFile(new ArrayList<>())); + } + return false; + } + + public boolean addTask(ReptileTask task){ + try { + if(runningTask.containsKey(task.getTaskId())){ + return false; + } + return waitingTask.offer(task,waitingQueueTime,TimeUnit.MILLISECONDS); + }catch (InterruptedException e){ + //TODO 待完善错误处理 + return false; + } + } + + public void work(){ + ChopperLogFactory.getLogger(LoggerType.Creeper) + .info("TaskCenter start to work.threads:{},waitingTime:{}s",threads,waitingQueueTime/1000); + while(true){ + try { + ReptileTask task = waitingTask.take(); + runningTask.put(task.getTaskId(),task); + taskPool.submit(task::reptile); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + public boolean finishTask(String taskId){ + ReptileTask reptileTask; + if ((reptileTask=runningTask.remove(taskId))!=null) { + try { + //查看是不是需要更新日志 + try { + lock.lock(); + if (CreeperLogConfigFile.needNewDayLog(creeperLogFileCache.getFileName())) { + updateLogFile(); + } + creeperLogFileCache.append(reptileTask,"task","-1"); + }finally { + lock.unlock(); + } + }catch (FileCacheException | InterruptedException e){ + return false; + } + } + return false; + } + + /** + * 如果时间变为新的一天则创建新的日志文件，并清理缓存中的日志文件 + */ + private void updateLogFile(){ + CreeperLogConfigFile configFile = new CreeperLogConfigFile(new ArrayList<>()); + String oldFilePath = creeperLogFileCache.getFullFilePath(); + if (newLogFile(configFile)) { + FileCacheManagerInstance.getInstance().deleteFileCache(oldFilePath); + } + } + + public boolean newLogFile(CreeperLogConfigFile configFile){ + if (ConfigFileUtil.createConfigFile(CreeperLogConfigFile.getFullFilePath(),configFile)) { + creeperLogFileCache = FileCacheManagerInstance.getInstance().getFileCache(CreeperConfigFile.getFullFilePath()); + return true; + } + return false; + } + + /** + * 恢复当日因异常关闭而没有完成的任务 + */ + public void restoreTaskCenter(){ + JSONArray tasks = (JSONArray)creeperLogFileCache.get("task"); + int restoreNum = 0; + if(tasks.size()>0){ + for (Object task : tasks) { + if(task instanceof JSONObject){ + ReptileTask reptileTask = JSONObject.parseObject(task.toString(), ReptileTask.class); + if(reptileTask.getType().equals(ReptileTask.TaskStatus.Running)){ + restoreNum++; + try { + waitingTask.put(reptileTask); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + } + ChopperLogFactory.getLogger(LoggerType.Creeper).info(" Find {} reptile task need restore,already insert waiting queue",restoreNum); + } + + public boolean shutdown(){ + taskPool.shutdown(); + return taskPool.isShutdown(); + } + + public void request(ReptileRequest request){ + ReptileTask task = bootStrapTaskHandler.distribute(request); + if(task!=null){ + addTask(task); + } + } + +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/handler/BootStrapTaskHandler.java b/CreeperModule/src/main/java/org/example/taskcenter/handler/BootStrapTaskHandler.java new file mode 100644 index 0000000..33c851d --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/handler/BootStrapTaskHandler.java @@ -0,0 +1,32 @@ +package org.example.taskcenter.handler; + +import org.example.taskcenter.TaskCenter; +import org.example.taskcenter.request.BarrageReptileRequest; +import org.example.taskcenter.request.LiveReptileRequest; +import org.example.taskcenter.request.ReptileRequest; +import org.example.taskcenter.task.ReptileTask; + +/** + * @author Genius + * @date 2023/07/28 17:37 + **/ +public class BootStrapTaskHandler implements TaskHandler { + + private LiveTaskHandler liveTaskHandler; + + public BootStrapTaskHandler() { + this.liveTaskHandler = new LiveTaskHandler(); + } + + @Override + public ReptileTask distribute(ReptileRequest request) { + ReptileTask task = null; + if(request instanceof LiveReptileRequest){ + task = liveTaskHandler.distribute((LiveReptileRequest) request); + }else if(request instanceof BarrageReptileRequest){ + + } + return task; + } + +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/handler/LiveTaskHandler.java b/CreeperModule/src/main/java/org/example/taskcenter/handler/LiveTaskHandler.java new file mode 100644 index 0000000..569808d --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/handler/LiveTaskHandler.java @@ -0,0 +1,33 @@ +package org.example.taskcenter.handler; + +import org.example.bean.Live; +import org.example.bean.hotmodule.DouyuLive; +import org.example.core.factory.LoadTaskFactory; +import org.example.pojo.download.assign.DouyuLiveOnlineConfig; +import org.example.taskcenter.request.LiveReptileRequest; +import org.example.taskcenter.task.ReptileTask; + +/** + * @author Genius + * @date 2023/07/28 17:39 + **/ + +/** + * + */ +public class LiveTaskHandler implements TaskHandler { + + @Override + public ReptileTask distribute(LiveReptileRequest request) { + Live live = request.getRequestObj(); + ReptileTask task = null; + if(live instanceof DouyuLive){ + if(request.isOnline()){ + task = new ReptileTask(LoadTaskFactory.getLoadTask(new DouyuLiveOnlineConfig((DouyuLive) live)),request); + }else{ + //Todo task = new ReptileTask(录播LoadTask,this); + } + } + return task; + } +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/handler/TaskHandler.java b/CreeperModule/src/main/java/org/example/taskcenter/handler/TaskHandler.java new file mode 100644 index 0000000..950f469 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/handler/TaskHandler.java @@ -0,0 +1,9 @@ +package org.example.taskcenter.handler; + +import org.example.taskcenter.request.ReptileRequest; +import org.example.taskcenter.task.ReptileTask; + +public interface TaskHandler { + + ReptileTask distribute(T request); +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/request/BarrageReptileRequest.java b/CreeperModule/src/main/java/org/example/taskcenter/request/BarrageReptileRequest.java new file mode 100644 index 0000000..988b348 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/request/BarrageReptileRequest.java @@ -0,0 +1,18 @@ +package org.example.taskcenter.request; + +import org.example.pojo.Barrage; + +import java.util.function.Consumer; + +/** + * @author Genius + * @date 2023/07/28 23:06 + **/ + +//TODO 弹幕请求待完善 +public class BarrageReptileRequest extends ReptileRequest{ + + public BarrageReptileRequest(Barrage requestObj, Consumer callback) { + super(requestObj, callback); + } +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/request/LiveReptileRequest.java b/CreeperModule/src/main/java/org/example/taskcenter/request/LiveReptileRequest.java new file mode 100644 index 0000000..f9a5ab2 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/request/LiveReptileRequest.java @@ -0,0 +1,43 @@ +package org.example.taskcenter.request; + +import org.example.bean.Live; + +import java.util.function.Consumer; + +/** + * @author Genius + * @date 2023/07/28 17:33 + **/ + +//直播爬取请求 +public class LiveReptileRequest extends ReptileRequest{ + + public enum LiveType{ + Online,//在线直播爬取 + Record //录播爬取 + } + + private LiveType type; + + public LiveReptileRequest(Live requestObj, LiveType type, Consumer callBack) { + super(requestObj,callBack); + this.type = type; + } + + public LiveType getType() { + return type; + } + + public void setType(LiveType type) { + this.type = type; + } + + public boolean isOnline(){ + return type==LiveType.Online; + } + + @Override + public String GenerateTaskId() { + return requestObj.getPlatform()+"-"+requestObj.getLiver(); + } +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/request/ReptileRequest.java b/CreeperModule/src/main/java/org/example/taskcenter/request/ReptileRequest.java new file mode 100644 index 0000000..a518ea2 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/request/ReptileRequest.java @@ -0,0 +1,35 @@ +package org.example.taskcenter.request; + +import java.util.function.Consumer; + +/** + * @author Genius + * @date 2023/07/28 17:32 + **/ +public abstract class ReptileRequest { + + protected Consumer callback; + protected T requestObj; + + public ReptileRequest(T requestObj,Consumer callback) { + + this.requestObj = requestObj; + this.callback = callback; + } + + public T getRequestObj() { + return requestObj; + } + + public void setRequestObj(T requestObj) { + this.requestObj = requestObj; + } + + public String GenerateTaskId(){ + return String.valueOf(requestObj.hashCode()); + } + + public void response(Object obj){ + callback.accept(obj); + } +} diff --git a/CreeperModule/src/main/java/org/example/taskcenter/task/ReptileTask.java b/CreeperModule/src/main/java/org/example/taskcenter/task/ReptileTask.java new file mode 100644 index 0000000..68286d3 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/taskcenter/task/ReptileTask.java @@ -0,0 +1,56 @@ +package org.example.taskcenter.task; + +import lombok.Data; +import org.example.core.control.LoadTask; +import org.example.taskcenter.TaskCenter; +import org.example.taskcenter.request.ReptileRequest; +import org.example.util.TimeUtil; + +import java.time.LocalDateTime; + +import static org.example.constpool.ConstPool.NULL_TIME; + +/** + * @author Genius + * @date 2023/07/28 21:51 + **/ +@Data +public class ReptileTask { + + public enum TaskStatus{ + Already,//准备就绪 + Running,//正在运行 + Finish,//完成的任务 + No_Status //没有状态，用于不需要恢复的任务 + } + private String taskId; + private ReptileRequest request; + private LoadTask loadTask; + + private String startTime; + private String endTime; + private TaskStatus type; + + + public ReptileTask(LoadTask loadTask,ReptileRequest request) { + this.loadTask = loadTask; + this.request =request; + this.taskId = request.GenerateTaskId(); + this.type = TaskStatus.Already; + this.endTime = NULL_TIME; + } + + public void reptile(){ + //开始任务 + this.startTime = TimeUtil.getNowTime_YMDHMS(); + this.type = TaskStatus.Running; + Object res = loadTask.start(); + + //完成任务 + request.response(res); //让请求响应结果 + TaskCenter.center().finishTask(taskId); + this.type = TaskStatus.Finish; + this.endTime = TimeUtil.getNowTime_YMDHMS(); + } + +} diff --git a/FileModule/src/main/java/org/example/cache/FileCacheManager.java b/FileModule/src/main/java/org/example/cache/FileCacheManager.java index 636bc32..d58b925 100644 --- a/FileModule/src/main/java/org/example/cache/FileCacheManager.java +++ b/FileModule/src/main/java/org/example/cache/FileCacheManager.java @@ -1,14 +1,11 @@ package org.example.cache; -import org.example.log.FileModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import org.example.util.TimeUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.time.LocalDateTime; import java.util.List; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static java.lang.Thread.sleep; @@ -75,7 +72,7 @@ public class FileCacheManager { fileCaches.add(fileCache); fileCacheMap.put(fileCache.getFullFilePath(),fileCache); initSleepTime(); - FileModuleLogger.logger.debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath()); + ChopperLogFactory.getLogger(LoggerType.File).debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath()); } return false; } @@ -84,6 +81,10 @@ public class FileCacheManager { return fileCacheMap.get(filePath); } + public boolean deleteFileCache(String filePath){ + return fileCaches.remove(fileCacheMap.get(filePath)) && fileCacheMap.remove(filePath)!=null; + } + public List getRunnableFileCaches(){ return this.fileCaches; } @@ -98,7 +99,7 @@ public class FileCacheManager { BlockingQueue fileChannel = cache.getFileChannel(); if(fileChannel.isEmpty()){ if(cache.needAutoSync()){ - FileModuleLogger.logger.debug("检测到需要强制刷新的文件 {}",cache.getFileName()); + ChopperLogFactory.getLogger(LoggerType.File).debug("检测到需要强制刷新的文件 {}",cache.getFileName()); autoSyncer.submit(new AutoSyncer(cache)); } } diff --git a/FileModule/src/main/java/org/example/init/ConfigInitMachine.java b/FileModule/src/main/java/org/example/init/ConfigInitMachine.java new file mode 100644 index 0000000..c878813 --- /dev/null +++ b/FileModule/src/main/java/org/example/init/ConfigInitMachine.java @@ -0,0 +1,39 @@ +package org.example.init; + +import org.example.bean.ConfigFile; +import org.example.cache.FileCache; +import org.example.cache.FileCacheManagerInstance; +import org.example.exception.FileCacheException; +import org.example.util.ConfigFileUtil; +import org.example.util.FileUtil; +import org.example.util.JsonFileUtil; +import org.slf4j.Logger; + +import java.nio.file.Paths; +import java.util.Map; + +/** + * @author Genius + * @date 2023/07/29 00:08 + **/ +public abstract class ConfigInitMachine extends CommonInitMachine{ + + private T configFile; + + private String filePath; + + public ConfigInitMachine(T configFile, Logger logger) { + super(logger); + this.configFile = configFile; + filePath = Paths.get(configFile.getFilePath(), configFile.getFileName()).toString(); + } + + + + @Override + public boolean init() { + return ConfigFileUtil.createConfigFile(filePath,configFile,logger,this.getClass().getName()); + } + + +} diff --git a/FileModule/src/main/java/org/example/init/FileCacheManagerInit.java b/FileModule/src/main/java/org/example/init/FileCacheManagerInit.java index eaab706..cb87db5 100644 --- a/FileModule/src/main/java/org/example/init/FileCacheManagerInit.java +++ b/FileModule/src/main/java/org/example/init/FileCacheManagerInit.java @@ -1,8 +1,8 @@ package org.example.init; import org.example.cache.FileCacheManagerInstance; -import org.example.log.FileModuleLogger; -import org.slf4j.Logger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; /** * @author Genius @@ -11,7 +11,7 @@ import org.slf4j.Logger; public class FileCacheManagerInit extends CommonInitMachine{ public FileCacheManagerInit() { - super(FileModuleLogger.logger); + super( ChopperLogFactory.getLogger(LoggerType.File)); } @Override diff --git a/FileModule/src/main/java/org/example/init/FileModuleInitMachine.java b/FileModule/src/main/java/org/example/init/FileModuleInitMachine.java index 11388be..1ce9fcc 100644 --- a/FileModule/src/main/java/org/example/init/FileModuleInitMachine.java +++ b/FileModule/src/main/java/org/example/init/FileModuleInitMachine.java @@ -1,7 +1,7 @@ package org.example.init; -import org.example.log.FileModuleLogger; -import org.slf4j.Logger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import java.util.List; @@ -15,7 +15,7 @@ public class FileModuleInitMachine extends ModuleInitMachine{ super(List.of( new ModuleSrcConfigFileInit(), new FileCacheManagerInit() - ), "FileModule", FileModuleLogger.logger); + ), "FileModule", ChopperLogFactory.getLogger(LoggerType.File)); } } diff --git a/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java b/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java index 0a809ba..33a68f6 100644 --- a/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java +++ b/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java @@ -1,11 +1,11 @@ package org.example.init; -import org.example.log.FileModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import org.example.pojo.configfile.ModuleSrcConfigFile; import org.example.util.FileUtil; import org.example.util.JsonFileUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,7 +21,7 @@ public class ModuleSrcConfigFileInit extends CommonInitMachine { ModuleSrcConfigFile moduleSrcConfigFile; public ModuleSrcConfigFileInit() { - super(FileModuleLogger.logger); + super( ChopperLogFactory.getLogger(LoggerType.File)); moduleSrcConfigFile = new ModuleSrcConfigFile(); } diff --git a/FileModule/src/main/java/org/example/util/ConfigFileUtil.java b/FileModule/src/main/java/org/example/util/ConfigFileUtil.java new file mode 100644 index 0000000..92a8d77 --- /dev/null +++ b/FileModule/src/main/java/org/example/util/ConfigFileUtil.java @@ -0,0 +1,51 @@ +package org.example.util; + +import org.example.bean.ConfigFile; +import org.example.cache.FileCache; +import org.example.cache.FileCacheManagerInstance; +import org.example.exception.FileCacheException; +import org.slf4j.Logger; + +import java.util.Map; + +/** + * @author Genius + * @date 2023/07/29 01:40 + **/ +public class ConfigFileUtil { + + public static boolean createConfigFile(String filePath, ConfigFile configFile, Logger logger,String pluginName,boolean isNeedLog) { + if(!FileUtil.isFileExist(filePath)){ + JsonFileUtil.writeJsonFile(filePath,configFile.packageConfig()); + try { + FileCache fileCache = new FileCache(configFile); + FileCacheManagerInstance.getInstance().addFileCache(fileCache); + } catch (FileCacheException e) { + if(isNeedLog) logger.error(e.toString()); + return false; + } + if(isNeedLog) logger.info(String.format("[✔]%s is created,{%s} plugin init success!", filePath, pluginName)); + return true; + }else{ + Map data = JsonFileUtil.readJsonFile(filePath); + configFile.setData(data); + try { + FileCache fileCache = new FileCache(configFile); + FileCacheManagerInstance.getInstance().addFileCache(fileCache); + } catch (FileCacheException e) { + if(isNeedLog) logger.error(e.toString()); + return false; + } + } + if(isNeedLog) logger.info(String.format("[✔]%s read in disk,{%s} plugin init success!", filePath, pluginName)); + return true; + } + + public static boolean createConfigFile(String filePath, ConfigFile configFile, Logger logger,String pluginName){ + return createConfigFile(filePath,configFile,logger,pluginName,true); + } + + public static boolean createConfigFile(String filePath, ConfigFile configFile){ + return createConfigFile(filePath,configFile,null,null,false); + } +} diff --git a/HotModule/src/main/java/org/example/api/HotModuleApi.java b/HotModule/src/main/java/org/example/api/HotModuleApi.java index 3cf5c4d..dd83ab9 100644 --- a/HotModule/src/main/java/org/example/api/HotModuleApi.java +++ b/HotModule/src/main/java/org/example/api/HotModuleApi.java @@ -1,9 +1,13 @@ package org.example.api; -import org.example.bean.HotLive; +import org.example.bean.Live; +import org.example.bean.hotmodule.DouyuLive; import org.example.bean.hotmodule.HotModuleList; -import org.example.core.control.hotmodule.DouyuHotLiveLoadTask; -import org.example.core.control.hotmodule.DouyuHotModuleLoadTask; +import org.example.core.control.impl.DouyuHotLiveLoadTask; +import org.example.core.control.impl.DouyuHotModuleLoadTask; +import org.example.core.factory.LoadTaskFactory; +import org.example.pojo.download.assign.DouyuHotLiveConfig; +import org.example.pojo.download.assign.DouyuHotModuleConfig; import java.util.List; @@ -12,18 +16,17 @@ import java.util.List; * @date 2023/07/21 17:53 **/ public class HotModuleApi { - private static DouyuHotModuleLoadTask douyuHotModuleLoadTask = new DouyuHotModuleLoadTask(); - private static DouyuHotLiveLoadTask douyuHotLiveLoadTask = new DouyuHotLiveLoadTask(); + public static HotModuleList getDouyuAllHotModule(){ - return douyuHotModuleLoadTask.start(); + return (HotModuleList)LoadTaskFactory.getLoadTask(new DouyuHotModuleConfig()).start(); } - public static List getDouyuHotLive(){ - return douyuHotLiveLoadTask.start(); + public static List getDouyuHotLive(){ + return ((DouyuHotLiveLoadTask)LoadTaskFactory.getLoadTask(new DouyuHotLiveConfig())).start(); } - public static List getDouyuHotLive(int moduleId){ - return douyuHotLiveLoadTask.start(moduleId); + public static List getDouyuHotLive(int moduleId){ + return ((DouyuHotLiveLoadTask)LoadTaskFactory.getLoadTask(new DouyuHotLiveConfig())).start(moduleId); } } diff --git a/HotModule/src/main/java/org/example/config/HotModuleConfig.java b/HotModule/src/main/java/org/example/config/HotModuleConfig.java index 94b149a..fd2b1c9 100644 --- a/HotModule/src/main/java/org/example/config/HotModuleConfig.java +++ b/HotModule/src/main/java/org/example/config/HotModuleConfig.java @@ -1,6 +1,7 @@ package org.example.config; import org.example.bean.ConfigFile; +import org.example.bean.FileType; import org.example.constpool.CreeperModuleConstPool; import org.example.constpool.HotModuleConstPool; @@ -31,7 +32,7 @@ public class HotModuleConfig extends ConfigFile> { List.of(allLiveDog()), OneDay, FiveMinute) ), "GuardNum",10 - ) + ), FileType.HOT ); } diff --git a/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java b/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java index 60a9ff9..4487177 100644 --- a/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java +++ b/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java @@ -11,6 +11,6 @@ public class HotModuleConstPool { public static final String HOT_MODULE_CONFIG_ROOT = (String) GlobalFileCache.ModuleSrcConfigFile.get("hot","src"); //热门模块配置文件路径 - public static final String LOAD_TASK_CLASS_ROOT = "org.example.core.control.hotmodule"; //各个平台爬虫任务包路径 + public static final String LOAD_TASK_CLASS_ROOT = "org.example.core.control.impl"; //各个平台爬虫任务包路径 } diff --git a/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java b/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java index 422be63..fe6c208 100644 --- a/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java +++ b/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java @@ -3,15 +3,19 @@ package org.example.core; import lombok.AllArgsConstructor; import lombok.Data; import org.example.api.HotModuleApi; -import org.example.bean.HotLive; +import org.example.bean.Live; import org.example.bean.HotModule; import org.example.bean.hotmodule.HotModuleList; import org.example.constpool.ConstPool; -import org.example.log.HotModuleLogger; +import org.example.init.CommonInitMachine; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.taskcenter.request.ReptileRequest; import org.example.thread.oddjob.OddJobBoy; import org.example.util.TimeUtil; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -23,7 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * 热门模块的数据中心 */ -public class HotModuleDataCenter { +public class HotModuleDataCenter extends CommonInitMachine { /** * 设计这个参数的原因是，虽然HotModule可以存放HotLive数组，但是每天模块更新后会将整个hotModuleListPool替换，导致模块下的热门直播清空， @@ -60,19 +64,19 @@ public class HotModuleDataCenter { public ConcurrentHashMap> hotModuleLivePool; //各个平台热门模块直播列表 - private ConcurrentHashMap> hotLiveListPool; //各个平台热门直播列表 + private ConcurrentHashMap> hotLiveListPool; //各个平台热门直播列表 public void addModuleList(String platform,HotModuleList hotModuleList){ hotModuleListPool.put(platform,hotModuleList); } - public void addLiveList(String platform,List hotLiveList){ - hotLiveListPool.put(platform,hotLiveList); + public void addLiveList(String platform,List liveList){ + hotLiveListPool.put(platform, liveList); } - public void addModuleLiveList(String platform,HotModule hotModule,List hotLiveList){ - ModuleLives moduleLives = new ModuleLives(hotLiveList, LocalDateTime.now()); - hotModule.setHotLives(hotLiveList); + public void addModuleLiveList(String platform,HotModule hotModule,List liveList){ + ModuleLives moduleLives = new ModuleLives(liveList, LocalDateTime.now()); + hotModule.setHotLives(liveList); hotModuleLivePool.get(platform).put(hotModule,moduleLives); } @@ -80,35 +84,52 @@ public class HotModuleDataCenter { return hotModuleListPool.get(platform); } - public List getLiveList(String platform){ + public List getLiveList(String platform){ return hotLiveListPool.get(platform); } - public List getModuleLiveList(String platform,HotModule hotModule) throws InterruptedException { + public List getModuleLiveList(String platform, HotModule hotModule) throws InterruptedException { ModuleLives moduleLives = hotModuleLivePool.get(platform).get(hotModule); if(moduleLives!=null){ if (moduleLives.isExpire()) { + //TODO 改为爬虫请求 OddJobBoy.Boy().addWork(()->{ addModuleLiveList(platform,hotModule, HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId()))); - HotModuleLogger.logger.info("platform:{} module:{} hot lives refresh~",platform,hotModule.getTagName()); + ChopperLogFactory.getLogger(LoggerType.Hot).info("platform:{} module:{} hot lives refresh~",platform,hotModule.getTagName()); }); } - return moduleLives.hotLives; + return moduleLives.lives; }else{ //TODO多次访问改方法时会导致链接中断 - - - List hotLives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId())); - addModuleLiveList(platform,hotModule,hotLives); - return hotLives; + List lives = new ArrayList<>(); + if(ConstPool.PLATFORM.DOUYU.getName().equals(platform)){ + lives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId())); + } + addModuleLiveList(platform,hotModule, lives); + return lives; } } + //根据平台的模块名获取某模块 + public HotModule getModule(String platform,String moduleName){ + return this.getModuleList(platform).findHotModule(moduleName); + } + + //根据平台的模块ID获取某模块 + public HotModule getModule(String platform,int moduleId){ + return this.getModuleList(platform).findHotModule(moduleId); + } + + @Override + public boolean init() { + return HotModuleDataCenter.DataCenter()!=null; + } + @Data @AllArgsConstructor class ModuleLives{ - private List hotLives; + private List lives; private LocalDateTime updateTime; public boolean isExpire(){ diff --git a/HotModule/src/main/java/org/example/core/recommend/HeatRecommendation.java b/HotModule/src/main/java/org/example/core/recommend/HeatRecommendation.java new file mode 100644 index 0000000..23832ea --- /dev/null +++ b/HotModule/src/main/java/org/example/core/recommend/HeatRecommendation.java @@ -0,0 +1,140 @@ +package org.example.core.recommend; + +/** + * @author Genius + * @date 2023/07/22 23:20 + **/ + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.example.bean.Live; +import org.example.cache.FileCacheManagerInstance; +import org.example.config.FollowDog; +import org.example.config.HotModuleConfig; +import org.example.config.HotModuleSetting; +import org.example.core.HotModuleDataCenter; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.taskcenter.TaskCenter; +import org.example.taskcenter.request.LiveReptileRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +/** + * 根据热门的直播以及看门狗设置来推荐热门直播间到系统 + */ +public class HeatRecommendation { + + private Map recommendationLog; //推送日志 + + private Map> platformFollowDogMap ; //每个平台的跟风列表 + + private BlockingQueue hotEventList; //用于接收每一次的热榜更新信息，并进行跟风狗推送 + + private Reference reference; + + public HeatRecommendation(){ + recommendationLog = new ConcurrentHashMap<>(); + platformFollowDogMap = new ConcurrentHashMap<>(); + hotEventList = new ArrayBlockingQueue<>(1024); + List modules = new ArrayList<>(); + JSONArray jsonModules = (JSONArray) FileCacheManagerInstance + .getInstance() + .getFileCache(HotModuleConfig.getFullFilePath()) + .get("Module"); + jsonModules.forEach(jsonModule->{ + modules.add(JSONObject.parseObject(jsonModule.toString(),HotModuleSetting.class)); + }); + + for (HotModuleSetting module : modules) { + if (module.isFollowDogEnable()) { + platformFollowDogMap.put(module.getPlatform(),module.getFollowDogs()); + } + } + + this.reference = new Reference(); + } + + private void handlerHotEvent() throws InterruptedException { + while(true){ + String platform = hotEventList.take(); + List followDogList; + if(platformFollowDogMap.containsKey(platform) + &&(followDogList=platformFollowDogMap.get(platform)).size()>0){ + //发送给爬虫队列 + for (FollowDog followDog : followDogList) { + String moduleName = followDog.getModuleName(); + List lives; + if(FollowDog.ALL_LIVES.equals(moduleName)){ + lives = HotModuleDataCenter.DataCenter().getLiveList(platform); + }else{ + lives = HotModuleDataCenter.DataCenter().getModule(platform,moduleName).getHotLives(); + } + //TODO 待完善 1，需要发送弹幕爬虫请求 2，callback更改 + for (Live live : needRecommend(lives, followDog.getBanLiver(), followDog.getTop())) { + + Objects.requireNonNull(TaskCenter.center()). + request( new LiveReptileRequest(live, LiveReptileRequest.LiveType.Online,(t)->{ + ChopperLogFactory.getLogger(LoggerType.Hot).info("成功推荐主播,结果为:T"); + })); + + } + } + } + } + } + + private List needRecommend(List lives,List banLivers,int top){ + List recommendLive = new ArrayList<>(); + int num = 0; + + for (Live live : lives) { + if(recommendationLog.containsKey(live.getLiver())){ + if(top recommend {} lives,lives info:",recommendLive); + return recommendLive; + } + + private boolean isBan(String liver,List banLivers){ + for (String banLiver : banLivers) { + if (Pattern.compile(banLiver).matcher(liver).find()) { + return true; + } + } + return false; + } + + public Reference getReference(){ + return reference; + } + + + + class Reference implements Runnable{ + + @Override + public void run() { + try { + handlerHotEvent(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + +} diff --git a/HotModule/src/main/java/org/example/guard/Guard.java b/HotModule/src/main/java/org/example/guard/Guard.java index 4812860..ff94b96 100644 --- a/HotModule/src/main/java/org/example/guard/Guard.java +++ b/HotModule/src/main/java/org/example/guard/Guard.java @@ -2,19 +2,14 @@ package org.example.guard; import lombok.AllArgsConstructor; import lombok.Data; -import org.example.bean.HotLive; +import org.example.bean.Live; import org.example.bean.hotmodule.HotModuleList; -import org.example.constpool.ConstPool; import org.example.core.HotModuleDataCenter; import org.example.core.control.HotModuleLoadTask; -import org.example.core.control.LoadTask; -import org.example.guard.HotModuleGuardInstance; -import org.example.log.HotModuleLogger; import org.example.log.ResultLogger; +import org.slf4j.Logger; import java.util.List; -import java.util.Objects; -import java.util.concurrent.*; /** * @author Genius @@ -28,6 +23,7 @@ import java.util.concurrent.*; @AllArgsConstructor public class Guard implements Runnable, ResultLogger { + private Logger logger; private String guardName; private HotModuleLoadTask task; @@ -63,13 +59,13 @@ public class Guard implements Runnable, ResultLogger { if(clazzName.contains("module")){ HotModuleDataCenter.DataCenter().addModuleList(platform,(HotModuleList) data); }else if(clazzName.contains("live")){ - HotModuleDataCenter.DataCenter().addLiveList(platform,(List) data); + HotModuleDataCenter.DataCenter().addLiveList(platform,(List) data); } } @Override public void successLog() { - HotModuleLogger.logger.info("{} successfully finish!",guardName); + logger.info("{} successfully finish!",guardName); } @Override @@ -79,12 +75,12 @@ public class Guard implements Runnable, ResultLogger { @Override public void failLog() { - HotModuleLogger.logger.error("{} finish error,cancel this task!",guardName); + logger.error("{} finish error,cancel this task!",guardName); } @Override public void failLog(String str) { - HotModuleLogger.logger.error("{} fail try to redo,retry times:{}!",guardName,str); + logger.error("{} fail try to redo,retry times:{}!",guardName,str); } } diff --git a/HotModule/src/main/java/org/example/guard/HotModuleGuard.java b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java index 9ff03e2..1adbe88 100644 --- a/HotModule/src/main/java/org/example/guard/HotModuleGuard.java +++ b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java @@ -1,13 +1,12 @@ package org.example.guard; -import org.example.bean.HotLive; -import org.example.bean.hotmodule.HotModuleList; import org.example.cache.FileCache; -import org.example.cache.FileCacheManager; import org.example.cache.FileCacheManagerInstance; import org.example.config.HotModuleConfig; import org.example.constpool.HotModuleConstPool; import org.example.core.control.HotModuleLoadTask; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import org.example.thread.NamedThreadFactory; import java.util.List; @@ -25,8 +24,6 @@ public class HotModuleGuard { private Map runningGuards; // 运行的热度监控守卫 - - protected HotModuleGuard(List guards,int guardNum){ this.guards = guards; this.hotModuleGuardPool = Executors.newScheduledThreadPool(guardNum, new NamedThreadFactory("HotModuleGuard")); @@ -66,6 +63,7 @@ public class HotModuleGuard { String timeName = isHotModule?"updateHotModuleTimes":"updateHotLivesTimes"; try { addGuard(new Guard( + ChopperLogFactory.getLogger(LoggerType.Hot), clazzName.toLowerCase(), (HotModuleLoadTask)Class.forName(clazz).getDeclaredConstructor().newInstance(), (long)fileCache.get(timeName), diff --git a/HotModule/src/main/java/org/example/init/HotModuleConfigInitMachine.java b/HotModule/src/main/java/org/example/init/HotModuleConfigInitMachine.java index 3768924..58a2f6e 100644 --- a/HotModule/src/main/java/org/example/init/HotModuleConfigInitMachine.java +++ b/HotModule/src/main/java/org/example/init/HotModuleConfigInitMachine.java @@ -7,50 +7,24 @@ package org.example.init; import org.example.cache.FileCache; import org.example.cache.FileCacheManagerInstance; +import org.example.config.CreeperLogConfigFile; import org.example.config.HotModuleConfig; -import org.example.constpool.HotModuleConstPool; import org.example.exception.FileCacheException; -import org.example.log.HotModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import org.example.util.FileUtil; import org.example.util.JsonFileUtil; import org.slf4j.Logger; -import java.nio.file.Paths; import java.util.Map; /** * 热门模块配置文件初始化机器 */ -public class HotModuleConfigInitMachine extends CommonInitMachine { - - private String filePath = HotModuleConfig.getFullFilePath(); +public class HotModuleConfigInitMachine extends ConfigInitMachine { public HotModuleConfigInitMachine() { - super(HotModuleLogger.logger); + super(new HotModuleConfig(), ChopperLogFactory.getLogger(LoggerType.Hot)); } - @Override - public boolean init() { - HotModuleConfig hotModuleConfig = new HotModuleConfig(); - if(!FileUtil.isFileExist(filePath)){ - JsonFileUtil.writeJsonFile(filePath,hotModuleConfig.packageConfig()); - try { - FileCache fileCache = new FileCache(hotModuleConfig); - FileCacheManagerInstance.getInstance().addFileCache(fileCache); - } catch (FileCacheException e) { - return fail(e.toString()); - } - return success(String.format("[✔]%s is created,{%s} plugin init success!",filePath,this.getClass().toString())); - }else{ - Map data = JsonFileUtil.readJsonFile(filePath); - hotModuleConfig.setData(data); - try { - FileCache fileCache = new FileCache(hotModuleConfig); - FileCacheManagerInstance.getInstance().addFileCache(fileCache); - } catch (FileCacheException e) { - return fail(e.toString()); - } - } - return success(String.format("[✔]%s read in disk,{%s} plugin init success!",filePath,this.getClass().toString())); - } } diff --git a/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java b/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java index 7e1f5c5..46bd1d8 100644 --- a/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java +++ b/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java @@ -7,10 +7,10 @@ import org.example.cache.FileCacheManagerInstance; import org.example.config.HotModuleConfig; import org.example.config.HotModuleSetting; import org.example.core.control.HotModuleLoadTask; -import org.example.guard.HotModuleGuard; import org.example.guard.HotModuleGuardInstance; import org.example.guard.Guard; -import org.example.log.HotModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import org.example.util.ClassUtil; import java.util.ArrayList; @@ -28,7 +28,7 @@ public class HotModuleGuardInitMachine extends CommonInitMachine{ public HotModuleGuardInitMachine() { - super(HotModuleLogger.logger); + super(ChopperLogFactory.getLogger(LoggerType.Hot)); } private void envInit() throws Exception { @@ -54,11 +54,11 @@ public class HotModuleGuardInitMachine extends CommonInitMachine{ Class loadClazz = Class.forName(clazz); if(isHotModule&&hotModuleSetting.isEnableHotModule()){ HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance(); - guards.add(new Guard(clazzName,task, + guards.add(new Guard(logger,clazzName,task, hotModuleSetting.getUpdateHotModuleTimes(),hotModuleSetting.getFailRetryTimes())); }else if(hotModuleSetting.isEnableHotLive()){ HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance(); - guards.add(new Guard(clazzName,task, + guards.add(new Guard(logger,clazzName,task, hotModuleSetting.getUpdateHotLivesTimes(),hotModuleSetting.getFailRetryTimes())); } } diff --git a/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java b/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java index 1a1a0d1..1530808 100644 --- a/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java +++ b/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java @@ -1,6 +1,7 @@ package org.example.init; -import org.example.log.HotModuleLogger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; import java.util.List; @@ -18,7 +19,7 @@ public class HotModuleInitMachine extends ModuleInitMachine{ super( List.of(new HotModuleConfigInitMachine(),new HotModuleGuardInitMachine()), "HotModule", - HotModuleLogger.logger + ChopperLogFactory.getLogger(LoggerType.Hot) ); } } diff --git a/common/src/main/java/org/example/bean/ConfigFile.java b/common/src/main/java/org/example/bean/ConfigFile.java index d1d2143..5a98fdc 100644 --- a/common/src/main/java/org/example/bean/ConfigFile.java +++ b/common/src/main/java/org/example/bean/ConfigFile.java @@ -10,7 +10,7 @@ import java.util.Map; **/ //配置文件的抽象类，只负责构建配置文件最基础的架构，一般不用来存放配置文件本身的内容 -public abstract class ConfigFile { +public abstract class ConfigFile{ private FileType fileType; private String filePath; @@ -109,4 +109,7 @@ public abstract class ConfigFile { public void setFilePath(String filePath){ this.filePath = filePath; } + + + } diff --git a/common/src/main/java/org/example/bean/FileType.java b/common/src/main/java/org/example/bean/FileType.java index f1c9453..f2444f3 100644 --- a/common/src/main/java/org/example/bean/FileType.java +++ b/common/src/main/java/org/example/bean/FileType.java @@ -7,9 +7,12 @@ package org.example.bean; public enum FileType { CHOPPER_BOT("ChopperBot"), BARRAGE("弹幕设置"), + HOT("热门设置"), CREEPER("爬虫设置"), VIDEO("文件下载"), - COMMON("普通文件"); + COMMON("普通文件"), + + LOG("日志文件"); private final String name; FileType(String name){ this.name = name; diff --git a/common/src/main/java/org/example/bean/HotModule.java b/common/src/main/java/org/example/bean/HotModule.java index 5cfc1ca..7abeef4 100644 --- a/common/src/main/java/org/example/bean/HotModule.java +++ b/common/src/main/java/org/example/bean/HotModule.java @@ -1,7 +1,5 @@ package org.example.bean; -import lombok.Data; - import java.util.List; /** @@ -10,11 +8,11 @@ import java.util.List; **/ -public abstract class HotModule { +public abstract class HotModule { private String tagId; private String tagName; - private List hotLives; + private List lives; public String getTagId() { return tagId; @@ -32,12 +30,12 @@ public abstract class HotModule { this.tagName = tagName; } - public List getHotLives() { - return hotLives; + public List getHotLives() { + return lives; } - public void setHotLives(List hotLives) { - this.hotLives = hotLives; + public void setHotLives(List lives) { + this.lives = lives; } public HotModule(String tagId, String tagName){ diff --git a/common/src/main/java/org/example/bean/HotLive.java b/common/src/main/java/org/example/bean/Live.java similarity index 66% rename from common/src/main/java/org/example/bean/HotLive.java rename to common/src/main/java/org/example/bean/Live.java index 01f4bc5..45c9b60 100644 --- a/common/src/main/java/org/example/bean/HotLive.java +++ b/common/src/main/java/org/example/bean/Live.java @@ -8,7 +8,7 @@ package org.example.bean; /** * 热门直播抽象类 */ -public abstract class HotLive { +public abstract class Live { private int watcherNum; //直播间观众数目 private int liveId; //直播间ID private String liveName; //直播间名字 @@ -16,14 +16,16 @@ public abstract class HotLive { private String liver; //主播 private String description; //直播间简介 - public HotLive(int watcherNum, int liveId, String liveName, String description) { + private String platform; + + public Live(int watcherNum, int liveId, String liveName, String description) { this.watcherNum = watcherNum; this.liveId = liveId; this.liveName = liveName; this.description = description; } - public HotLive(int watcherNum, int liveId, String liveName, String liver, String description) { + public Live(int watcherNum, int liveId, String liveName, String liver, String description) { this.watcherNum = watcherNum; this.liveId = liveId; this.liveName = liveName; @@ -31,6 +33,15 @@ public abstract class HotLive { this.description = description; } + public Live(int watcherNum, int liveId, String liveName, String liver, String description, String platform) { + this.watcherNum = watcherNum; + this.liveId = liveId; + this.liveName = liveName; + this.liver = liver; + this.description = description; + this.platform = platform; + } + public int getWatcherNum() { return watcherNum; } @@ -70,4 +81,12 @@ public abstract class HotLive { public void setDescription(String description) { this.description = description; } + + public String getPlatform() { + return platform; + } + + public void setPlatform(String platform) { + this.platform = platform; + } } diff --git a/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java b/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java index 67784f5..cf002c0 100644 --- a/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java +++ b/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java @@ -2,13 +2,14 @@ package org.example.bean.hotmodule; import lombok.Data; import org.example.bean.HotModule; +import org.example.bean.Live; /** * @author Genius * @date 2023/07/18 20:49 **/ -public class BilibiliHotModule extends HotModule { +public class BilibiliHotModule extends HotModule { private BilibiliParentHotModule parentHotModule; diff --git a/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java b/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java index 5274fd4..827b421 100644 --- a/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java +++ b/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java @@ -7,7 +7,7 @@ import org.example.bean.HotModule; * @author Genius * @date 2023/07/18 20:46 **/ -public class DouyuHotModule extends HotModule { +public class DouyuHotModule extends HotModule { private String url; public DouyuHotModule(String tagId, String tagName, String url) { diff --git a/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java b/common/src/main/java/org/example/bean/hotmodule/DouyuLive.java similarity index 77% rename from common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java rename to common/src/main/java/org/example/bean/hotmodule/DouyuLive.java index 0afb48b..73ea7df 100644 --- a/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java +++ b/common/src/main/java/org/example/bean/hotmodule/DouyuLive.java @@ -1,12 +1,13 @@ package org.example.bean.hotmodule; -import org.example.bean.HotLive; +import org.example.bean.Live; +import org.example.constpool.ConstPool; /** * @author Genius * @date 2023/07/19 01:16 **/ -public class DouyuHotLive extends HotLive { +public class DouyuLive extends Live { private String liveModule; //直播板块 @@ -21,9 +22,9 @@ public class DouyuHotLive extends HotLive { private int moduleId; //直播板块Id - public DouyuHotLive(int watcherNum, int liveId, String liveName, String liver, String description, - String liveModule, String url, String roomCoverPic, int type, int uid,int moduleId) { - super(watcherNum, liveId, liveName, liver, description); + public DouyuLive(int watcherNum, int liveId, String liveName, String liver, String description, + String liveModule, String url, String roomCoverPic, int type, int uid, int moduleId) { + super(watcherNum, liveId, liveName, liver, description, ConstPool.PLATFORM.DOUYU.getName()); this.liveModule = liveModule; this.url = url; this.roomCoverPic = roomCoverPic; diff --git a/common/src/main/java/org/example/constpool/ConstPool.java b/common/src/main/java/org/example/constpool/ConstPool.java index 04164f5..07cf45d 100644 --- a/common/src/main/java/org/example/constpool/ConstPool.java +++ b/common/src/main/java/org/example/constpool/ConstPool.java @@ -18,6 +18,9 @@ public class ConstPool { public static final String HOT = "hot"; public static final String PUBLISH = "publish"; + /**其他**/ + public static final String NULL_TIME = "nil"; + /**图片格式**/ public static final List PIC_TYPES = List.of("jpg","jpeg","png","svg"); diff --git a/common/src/main/java/org/example/init/CommonInitMachine.java b/common/src/main/java/org/example/init/CommonInitMachine.java index 3aaaa07..4daefd4 100644 --- a/common/src/main/java/org/example/init/CommonInitMachine.java +++ b/common/src/main/java/org/example/init/CommonInitMachine.java @@ -71,4 +71,9 @@ public abstract class CommonInitMachine implements ComponentInitMachine{ private void shutdownLog(){ logger.info("[🆖] {} close success.",this.getClass().getName()); } + + @Override + public void afterInit() { + + } } diff --git a/common/src/main/java/org/example/init/InitMachine.java b/common/src/main/java/org/example/init/InitMachine.java index f35090e..a53e77a 100644 --- a/common/src/main/java/org/example/init/InitMachine.java +++ b/common/src/main/java/org/example/init/InitMachine.java @@ -13,5 +13,7 @@ public interface InitMachine extends ResultLogger { boolean init(); + void afterInit(); + void shutdown(); } diff --git a/common/src/main/java/org/example/init/ModuleInitMachine.java b/common/src/main/java/org/example/init/ModuleInitMachine.java index c064d4e..948afdc 100644 --- a/common/src/main/java/org/example/init/ModuleInitMachine.java +++ b/common/src/main/java/org/example/init/ModuleInitMachine.java @@ -35,6 +35,13 @@ public abstract class ModuleInitMachine extends CommonInitMachine{ }); } + @Override + public void afterInit() { + for(InitMachine initMachine:this.getInitMachines()){ + initMachine.afterInit(); + } + } + public ModuleInitMachine(List initMachines, String moduleName) { this.initMachines = initMachines; this.moduleName = moduleName; diff --git a/common/src/main/java/org/example/log/ChopperLogFactory.java b/common/src/main/java/org/example/log/ChopperLogFactory.java new file mode 100644 index 0000000..d4420d8 --- /dev/null +++ b/common/src/main/java/org/example/log/ChopperLogFactory.java @@ -0,0 +1,15 @@ +package org.example.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Genius + * @date 2023/07/28 23:22 + **/ +public class ChopperLogFactory { + + public static Logger getLogger(LoggerType loggerType){ + return LoggerFactory.getLogger(loggerType.getLoggerName()); + } +} diff --git a/common/src/main/java/org/example/log/FileModuleLogger.java b/common/src/main/java/org/example/log/FileModuleLogger.java deleted file mode 100644 index d4a69b3..0000000 --- a/common/src/main/java/org/example/log/FileModuleLogger.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.example.log; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Genius - * @date 2023/07/21 00:32 - **/ -public class FileModuleLogger { - public static final Logger logger = LoggerFactory.getLogger("FileModuleLogger"); -} diff --git a/common/src/main/java/org/example/log/HotModuleLogger.java b/common/src/main/java/org/example/log/HotModuleLogger.java deleted file mode 100644 index c660802..0000000 --- a/common/src/main/java/org/example/log/HotModuleLogger.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.example.log; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - - -/** - * @author Genius - * @date 2023/07/19 02:55 - **/ -public class HotModuleLogger { - public static final Logger logger = LoggerFactory.getLogger("HotModuleLogger "); -} diff --git a/common/src/main/java/org/example/log/LoggerType.java b/common/src/main/java/org/example/log/LoggerType.java new file mode 100644 index 0000000..b2f4551 --- /dev/null +++ b/common/src/main/java/org/example/log/LoggerType.java @@ -0,0 +1,21 @@ +package org.example.log; + +/** + * @author Genius + * @date 2023/07/28 23:22 + **/ +public enum LoggerType { + System("ChopperBot🤖"), + Creeper("Creeper "), + File("FileModule "), + Hot("HotModule "); + private String loggerName; + + LoggerType(String name){ + this.loggerName = name; + } + + public String getLoggerName(){ + return loggerName; + } +} diff --git a/common/src/main/java/org/example/log/SystemLogger.java b/common/src/main/java/org/example/log/SystemLogger.java deleted file mode 100644 index a000648..0000000 --- a/common/src/main/java/org/example/log/SystemLogger.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.example.log; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Genius - * @date 2023/07/22 17:53 - **/ -public class SystemLogger { - - public static final Logger logger = LoggerFactory.getLogger("ChopperBotLogger"); - -} diff --git a/common/src/main/java/org/example/thread/ChopperBotThreadPool.java b/common/src/main/java/org/example/thread/ChopperBotThreadPool.java new file mode 100644 index 0000000..28ec5d8 --- /dev/null +++ b/common/src/main/java/org/example/thread/ChopperBotThreadPool.java @@ -0,0 +1,19 @@ +package org.example.thread; + +/** + * @author Genius + * @date 2023/07/24 23:51 + **/ + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + + +/** + * ChopperBot系统的线程池 + * TODO 待完善 + */ +public class ChopperBotThreadPool { + + public static ExecutorService threadPool = Executors.newFixedThreadPool(10); +} diff --git a/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java b/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java index 9b0976b..0b38717 100644 --- a/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java +++ b/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java @@ -1,9 +1,8 @@ package org.example.thread.oddjob; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import org.example.thread.NamedThreadFactory; + +import java.util.concurrent.*; /** * @author Genius @@ -22,7 +21,7 @@ public class OddJobBoy { private ExecutorService home; private OddJobBoy(){ - home = Executors.newSingleThreadExecutor(); + home = Executors.newSingleThreadExecutor(new NamedThreadFactory("OddJobBoy")); oddjobs = new ArrayBlockingQueue<>(1024); } diff --git a/common/src/main/java/org/example/util/TimeUtil.java b/common/src/main/java/org/example/util/TimeUtil.java index 93ccac7..cec85ac 100644 --- a/common/src/main/java/org/example/util/TimeUtil.java +++ b/common/src/main/java/org/example/util/TimeUtil.java @@ -1,7 +1,9 @@ package org.example.util; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; /** * @author Genius @@ -16,4 +18,17 @@ public class TimeUtil { public static long getSecond(LocalDateTime dateTime){ return dateTime.toEpochSecond(ZoneOffset.of("+8")); } + + + public static String getToday_YMD(){ + return LocalDate.now().toString(); + } + + public static String getYesterday_YMD(){ + return LocalDate.now().minusDays(1).toString(); + } + + public static String getNowTime_YMDHMS() { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + } } diff --git a/console/src/main/java/org/example/controller/HotController.java b/console/src/main/java/org/example/controller/HotController.java index 43fd75a..03c79dd 100644 --- a/console/src/main/java/org/example/controller/HotController.java +++ b/console/src/main/java/org/example/controller/HotController.java @@ -2,7 +2,7 @@ package org.example.controller; import com.genius.assistant.common.Result; import org.example.api.HotModuleApi; -import org.example.bean.HotLive; +import org.example.bean.Live; import org.example.bean.HotModule; import org.example.bean.hotmodule.HotModuleList; import org.example.constpool.ConstPool; @@ -29,13 +29,13 @@ public class HotController { @GetMapping("/douyu/allHotLive") public Result getDouyuAllHotLive(@RequestParam(defaultValue = "0") int latest){ - List hotLives; + List lives; if(latest==1){ - hotLives = HotModuleApi.getDouyuHotLive(); + lives = HotModuleApi.getDouyuHotLive(); }else{ - hotLives = HotModuleDataCenter.DataCenter().getLiveList(ConstPool.PLATFORM.DOUYU.getName()); + lives = HotModuleDataCenter.DataCenter().getLiveList(ConstPool.PLATFORM.DOUYU.getName()); } - return Result.success(hotLives); + return Result.success(lives); } @GetMapping("/douyu/allHotModule") diff --git a/console/src/main/java/org/example/init/InitWorld.java b/console/src/main/java/org/example/init/InitWorld.java index fd28e96..76e0f85 100644 --- a/console/src/main/java/org/example/init/InitWorld.java +++ b/console/src/main/java/org/example/init/InitWorld.java @@ -1,5 +1,6 @@ package org.example.init; +import org.example.guard.HotModuleGuardInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@ -33,6 +34,8 @@ public class InitWorld { boolean isInit = world.init(); if(!isInit){ close(); + }else{ + world.afterInit(); } } diff --git a/console/src/main/java/org/example/init/SystemInitMachine.java b/console/src/main/java/org/example/init/SystemInitMachine.java index d119ba4..37e8768 100644 --- a/console/src/main/java/org/example/init/SystemInitMachine.java +++ b/console/src/main/java/org/example/init/SystemInitMachine.java @@ -1,7 +1,8 @@ package org.example.init; -import org.example.log.SystemLogger; -import org.slf4j.Logger; +import org.example.log.ChopperLogFactory; +import org.example.log.LoggerType; +import org.example.thread.oddjob.OddJobBoy; import java.util.List; import java.util.function.Supplier; @@ -16,10 +17,26 @@ public class SystemInitMachine extends ModuleInitMachine{ public SystemInitMachine() { super(List.of( new FileModuleInitMachine(), + new CreeperModuleInitMachine(), new HotModuleInitMachine() - ), "ChopperBot", SystemLogger.logger); + ), "ChopperBot", ChopperLogFactory.getLogger(LoggerType.System)); } + @Override + public void afterInit() { + try { + OddJobBoy.Boy().addWork( + ()->{ + getInitMachines().forEach( + InitMachine::afterInit + ); + } + ); + }catch (Exception e){ + e.printStackTrace(); + } + + } @Override protected boolean initLogger(Supplier init) { diff --git a/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java b/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java index 2ec7ac2..b636081 100644 --- a/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java +++ b/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java @@ -1,9 +1,8 @@ package org.example.service.impl; -import org.example.bean.HotLive; +import org.example.bean.Live; import org.example.bean.HotModule; import org.example.bean.hotmodule.HotModuleList; -import org.example.constpool.HotModuleConstPool; import org.example.core.HotModuleDataCenter; import org.example.service.HotModuleService; import org.springframework.stereotype.Service; @@ -32,7 +31,7 @@ public class HotModuleServiceImpl implements HotModuleService { HotModule hotModule = moduleList.findHotModule(moduleId); if(hotModule!=null){ try { - List moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule); + List moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule); hotModule.setHotLives(moduleLiveList); return hotModule; }catch (Exception e){