From e5e4cb41de0214fad81ab1cd02dbfcf168a445b8 Mon Sep 17 00:00:00 2001 From: userA Date: Sat, 22 Jul 2023 16:50:58 +0800 Subject: [PATCH] =?UTF-8?q?1=EF=BC=8C=E5=AE=8C=E6=88=90=E7=83=AD=E9=97=A8?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E7=9A=84=E6=95=B0=E6=8D=AE=E4=B8=AD=E5=BF=83?= =?UTF-8?q?=202=EF=BC=8C=E5=AE=8C=E6=88=90api=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/example/constpool/ApiPool.java | 6 +- ...tPool.java => CreeperModuleConstPool.java} | 2 +- .../core/control/HotModuleLoadTask.java | 44 +++++++ .../hotmodule/DouyuHotLiveLoadTask.java | 63 ++++++++++ .../hotmodule/DouyuHotModuleLoadTask.java | 32 +++++ .../control/impl/BilibiliLiveLoadTask.java | 4 +- .../control/impl/DouyuHotLiveLoadTask.java | 77 ------------ .../control/impl/DouyuHotModuleLoadTask.java | 48 ------- .../control/impl/DouyuRecordLoadTask.java | 4 +- .../hotmodule/DouyuHotLiveProcessor.java | 28 +---- .../hotmodule/DouyuHotModuleProcessor.java | 7 +- .../pojo/configfile/BarrageSaveFile.java | 2 +- .../assign/BilibiliLiveLoadBarrageConfig.java | 4 +- .../assign/DouyuRecordLoadBarrageConfig.java | 4 +- .../java/org/example/cache/FileCache.java | 4 +- .../org/example/cache/FileCacheManager.java | 4 +- .../java/org/example/util/JsonFileUtil.java | 1 - .../java/org/example/api/HotModuleApi.java | 29 +++++ .../org/example/config/HotModuleConfig.java | 24 ++-- ...duleSetting.java => HotModuleSetting.java} | 8 +- .../example/constpool/HotModuleConstPool.java | 5 +- .../org/example/core/HotModuleDataCenter.java | 119 ++++++++++++++++++ .../main/java/org/example/guard/Guard.java | 90 +++++++++++++ .../org/example/guard/HotModuleGuard.java | 94 ++++++++++---- .../example/guard/HotModuleGuardInstance.java | 34 +++++ .../java/org/example/guard/task/Guard.java | 8 -- .../init/HotModuleGuardInitMachine.java | 80 ++++++++++++ .../example/init/HotModuleInitMachine.java | 2 +- .../java/org/example/constpool/ConstPool.java | 18 +++ .../org/example/constpool/HotModulePool.java | 26 ---- .../org/example/init/CommonInitMachine.java | 2 +- .../java/org/example/init/InitMachine.java | 4 +- ...itMachineLogger.java => ResultLogger.java} | 2 +- .../example/thread/NamedThreadFactory.java | 28 +++++ .../org/example/thread/oddjob/OddJob.java | 9 ++ .../org/example/thread/oddjob/OddJobBoy.java | 68 ++++++++++ .../main/java/org/example/util/ClassUtil.java | 41 ++++++ .../org/example/controller/HotController.java | 57 +++++++++ .../java/org/example/service/FileService.java | 0 .../org/example/service/HotModuleService.java | 14 +++ .../example/service/impl/FileServiceImpl.java | 0 .../service/impl/HotModuleServiceImpl.java | 45 +++++++ console/src/main/resources/application.yaml | 1 + 43 files changed, 895 insertions(+), 247 deletions(-) rename CreeperModule/src/main/java/org/example/constpool/{ConstPool.java => CreeperModuleConstPool.java} (96%) create mode 100644 CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java create mode 100644 CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java create mode 100644 CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java delete mode 100644 CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java delete mode 100644 CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java create mode 100644 HotModule/src/main/java/org/example/api/HotModuleApi.java rename HotModule/src/main/java/org/example/config/{ModuleSetting.java => HotModuleSetting.java} (75%) create mode 100644 HotModule/src/main/java/org/example/core/HotModuleDataCenter.java create mode 100644 HotModule/src/main/java/org/example/guard/Guard.java create mode 100644 HotModule/src/main/java/org/example/guard/HotModuleGuardInstance.java delete mode 100644 HotModule/src/main/java/org/example/guard/task/Guard.java create mode 100644 HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java delete mode 100644 common/src/main/java/org/example/constpool/HotModulePool.java rename common/src/main/java/org/example/log/{InitMachineLogger.java => ResultLogger.java} (82%) create mode 100644 common/src/main/java/org/example/thread/NamedThreadFactory.java create mode 100644 common/src/main/java/org/example/thread/oddjob/OddJob.java create mode 100644 common/src/main/java/org/example/thread/oddjob/OddJobBoy.java create mode 100644 common/src/main/java/org/example/util/ClassUtil.java create mode 100644 console/src/main/java/org/example/controller/HotController.java rename {FileModule => console}/src/main/java/org/example/service/FileService.java (100%) create mode 100644 console/src/main/java/org/example/service/HotModuleService.java rename {FileModule => console}/src/main/java/org/example/service/impl/FileServiceImpl.java (100%) create mode 100644 console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java diff --git a/CreeperModule/src/main/java/org/example/constpool/ApiPool.java b/CreeperModule/src/main/java/org/example/constpool/ApiPool.java index 7ca268f..d8c199f 100644 --- a/CreeperModule/src/main/java/org/example/constpool/ApiPool.java +++ b/CreeperModule/src/main/java/org/example/constpool/ApiPool.java @@ -10,6 +10,10 @@ package org.example.constpool; * 存放每一个爬虫api的池子 */ public class ApiPool { - public final static String HOT_MODULE_LIST_API = "/japi/search/api/getHotList"; + + public static final String DOUYU_HOT_MODULE_API = "https://www.douyu.com/japi/weblist/apinc/header/cate"; + public static final String DOUYU_HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api + + public static final String DOUYU_HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api } diff --git a/CreeperModule/src/main/java/org/example/constpool/ConstPool.java b/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java similarity index 96% rename from CreeperModule/src/main/java/org/example/constpool/ConstPool.java rename to CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java index 14637f0..b03075d 100644 --- a/CreeperModule/src/main/java/org/example/constpool/ConstPool.java +++ b/CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java @@ -5,7 +5,7 @@ package org.example.constpool; * @author 燧枫 * @date 2023/4/23 16:12 */ -public class ConstPool { +public class CreeperModuleConstPool { /** * 文件路径 diff --git a/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java new file mode 100644 index 0000000..04a8e83 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java @@ -0,0 +1,44 @@ +package org.example.core.control; + +import org.example.log.HotModuleLogger; +import us.codecraft.webmagic.ResultItems; +import us.codecraft.webmagic.Spider; + +/** + * @author Genius + * @date 2023/07/21 10:22 + **/ +public abstract class HotModuleLoadTask{ + public enum FinishFlag{ + FINISH,NOT_FINISH,FAIL + } + private FinishFlag finishFlag = FinishFlag.NOT_FINISH; + + + public T start() { + clearFinishFlag(); + return this.start0(); + } + + protected abstract T start0(); + protected void fail(Exception e){ + finishFlag = FinishFlag.FAIL; + HotModuleLogger.logger.error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage()); + } + + protected void success(){ + finishFlag = FinishFlag.FINISH; + } + + protected T getData(Spider spider,String url){ + T data = ((ResultItems) spider.get(url)).get("data"); + spider.close(); + return data; + } + + public FinishFlag isFinish(){ + return finishFlag; + } + + public void clearFinishFlag(){finishFlag = FinishFlag.NOT_FINISH;} +} diff --git a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java new file mode 100644 index 0000000..10386cc --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java @@ -0,0 +1,63 @@ +package org.example.core.control.hotmodule; + +import org.example.bean.HotLive; +import org.example.core.control.HotModuleLoadTask; +import org.example.core.control.LoadTask; +import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.Spider; + +import java.util.List; + +import static org.example.constpool.ApiPool.*; + +/** + * @author Genius + * @date 2023/07/19 02:42 + **/ +public class DouyuHotLiveLoadTask extends HotModuleLoadTask> { + + + + private final DouyuHotLiveProcessor douyuHotLiveProcessor; + + public DouyuHotLiveLoadTask(){ + douyuHotLiveProcessor = new DouyuHotLiveProcessor(); + } + + /** + * 获取Douyu某个模块下的热门直播 + * @param moduleId + */ + public List start(int moduleId){ + clearFinishFlag(); + douyuHotLiveProcessor.setModuleId(moduleId); + return this.start(String.format(DOUYU_HOT_MODULE_LIVES_API,moduleId)); + } + + /** + * 获取Douyu当前最热直播 + */ + @Override + protected List start0() { + return this.start(DOUYU_HOT_LIVES_API); + } + + private List start(String url){ + List lives; + try { + lives = getData(Spider.create(douyuHotLiveProcessor),url); + }catch (Exception e){ + fail(e); + return null; + } + success(); + return lives; + } + + public static void main(String[] args) { + List start = new DouyuHotLiveLoadTask().start(); + System.out.println(start); + } + +} diff --git a/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java new file mode 100644 index 0000000..21449f1 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java @@ -0,0 +1,32 @@ +package org.example.core.control.hotmodule; + +import org.example.bean.hotmodule.HotModuleList; +import org.example.core.control.HotModuleLoadTask; +import org.example.core.control.LoadTask; +import org.example.core.processor.hotmodule.DouyuHotModuleProcessor; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.ResultItems; +import us.codecraft.webmagic.Spider; + +import static org.example.constpool.ApiPool.DOUYU_HOT_MODULE_API; + +/** + * @author Genius + * @date 2023/07/15 21:04 + **/ +public class DouyuHotModuleLoadTask extends HotModuleLoadTask { + + + @Override + protected HotModuleList start0() { + HotModuleList data; + try { + data = getData(Spider.create(new DouyuHotModuleProcessor()),DOUYU_HOT_MODULE_API); + }catch (Exception e){ + fail(e); + return null; + } + success(); + return data; + } +} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java index ed5203e..1a071fc 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java @@ -1,6 +1,6 @@ package org.example.core.control.impl; -import org.example.constpool.ConstPool; +import org.example.constpool.CreeperModuleConstPool; import org.example.core.control.LoadTask; import org.example.core.factory.ProcessorFactory; import org.example.core.pipeline.PipelineWriteJson; @@ -34,7 +34,7 @@ public class BilibiliLiveLoadTask implements LoadTask { public void start() { Spider.create(bilibiliLiveProcessor) // 设置起始Request - .addRequest(new Request(ConstPool.OCCUURL)) + .addRequest(new Request(CreeperModuleConstPool.OCCUURL)) // 设置结果处理类 .addPipeline(pipelineWriteJson) // 设置抓取线程数（可根据需要调整） diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java deleted file mode 100644 index e0dfc27..0000000 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.example.core.control.impl; - -import org.example.constpool.HotModulePool; -import org.example.core.control.LoadTask; -import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; -import us.codecraft.webmagic.Request; -import us.codecraft.webmagic.Spider; - -/** - * @author Genius - * @date 2023/07/19 02:42 - **/ -public class DouyuHotLiveLoadTask implements LoadTask { - - private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api - private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api - - private final DouyuHotLiveProcessor douyuHotLiveProcessor; - - public DouyuHotLiveLoadTask(){ - douyuHotLiveProcessor = new DouyuHotLiveProcessor(); - } - - /** - * 获取Douyu某个模块下的热门直播 - * @param moduleId - */ - public void start(int moduleId){ - douyuHotLiveProcessor.setModuleId(moduleId); - this.start(String.format(HOT_MODULE_LIVES_API,moduleId)); - } - - /** - * 获取Douyu当前最热直播 - */ - @Override - public void start() { - this.start(HOT_LIVES_API); - } - - private void start(String url){ - try { - Spider.create(douyuHotLiveProcessor) - .addRequest(new Request(url)) - .thread(1) - .run(); - }catch (Exception e){ - e.printStackTrace(); - } - } - - @Override - public void end() { - - } - - @Override - public boolean isRunning() { - return false; - } - - @Override - public int getCacheSize() { - return 0; - } - - @Override - public int flushCacheAndSave(String key) { - return 0; - } - - public static void main(String[] args) { - new DouyuHotModuleLoadTask().start(); - new DouyuHotLiveLoadTask().start(1); - System.out.println(HotModulePool.hotModuleListPool); - } -} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java deleted file mode 100644 index a337d56..0000000 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.example.core.control.impl; - -import org.example.constpool.HotModulePool; -import org.example.core.control.LoadTask; -import org.example.core.processor.hotmodule.DouyuHotModuleProcessor; -import us.codecraft.webmagic.Request; -import us.codecraft.webmagic.Spider; - -/** - * @author Genius - * @date 2023/07/15 21:04 - **/ -public class DouyuHotModuleLoadTask implements LoadTask { - - private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate"; - @Override - public void start() { - DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor(); - Spider.create(douyuHotModuleProcessor) - .addRequest(new Request(url)) - .thread(1) - .run(); - } - - @Override - public void end() { - - } - - @Override - public boolean isRunning() { - return false; - } - - @Override - public int getCacheSize() { - return 0; - } - - @Override - public int flushCacheAndSave(String key) { - return 0; - } - - public static void main(String[] args) { - new DouyuHotModuleLoadTask().start(); - } -} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java index 528f1f5..50097f4 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java @@ -1,6 +1,6 @@ package org.example.core.control.impl; -import org.example.constpool.ConstPool; +import org.example.constpool.CreeperModuleConstPool; import org.example.core.control.LoadTask; import org.example.core.factory.ProcessorFactory; import org.example.core.pipeline.PipelineWriteJson; @@ -34,7 +34,7 @@ public class DouyuRecordLoadTask implements LoadTask { public void start() { Spider.create(douyuRecordProcessor) // 设置起始Request - .addRequest(new Request(ConstPool.OCCUURL)) + .addRequest(new Request(CreeperModuleConstPool.OCCUURL)) // 设置结果处理类 .addPipeline(pipelineWriteJson) // 设置抓取线程数（可根据需要调整） diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java index f942b98..b8aca9c 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java @@ -4,20 +4,12 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.bean.HotLive; -import org.example.bean.HotModule; import org.example.bean.hotmodule.DouyuHotLive; -import org.example.bean.hotmodule.HotModuleList; -import org.example.constpool.HotModulePool; -import org.example.log.HotModuleLogger; -import org.example.util.ChineseConvertUtil; -import org.springframework.util.StringUtils; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.processor.PageProcessor; -import us.codecraft.webmagic.selector.Json; import java.util.ArrayList; import java.util.List; -import java.util.Objects; /** * @author Genius @@ -60,25 +52,9 @@ public class DouyuHotLiveProcessor implements PageProcessor { } } }catch (Exception e){ - HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage()); + throw e; } - HotModuleLogger.logger.info(hotLiveList.toString()); - updateHotLiveList(hotLiveList); + page.putField("data",hotLiveList); } - private void updateHotLiveList(List hotLiveList){ - if(moduleId!=-1){ - if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){ - HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules); - HotModule hotModule = hotModuleList.findHotModule(this.moduleId); - if(hotModule!=null){ - hotModule.setHotLives(hotLiveList); - HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName()); - } - } - }else{ - HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated"); - HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList); - } - } } diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java index d4868c4..01c08bf 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java @@ -5,12 +5,9 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.bean.hotmodule.DouyuHotModule; import org.example.bean.hotmodule.HotModuleList; -import org.example.constpool.HotModulePool; -import org.example.log.HotModuleLogger; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.processor.PageProcessor; -import java.util.concurrent.ExecutorService; /** * @author Genius @@ -38,9 +35,9 @@ public class DouyuHotModuleProcessor implements PageProcessor { } } }catch (Exception e){ - HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage()); + throw e; } - HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList); + page.putField("data",douyuHotModuleList); } } diff --git a/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java b/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java index fd60d8d..3059445 100644 --- a/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java +++ b/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java @@ -13,7 +13,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.ConcurrentLinkedQueue; -import static org.example.constpool.ConstPool.BARRAGE_ROOT; +import static org.example.constpool.CreeperModuleConstPool.BARRAGE_ROOT; /** * @author Genius diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java index f29c8d8..f681b5b 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java @@ -1,7 +1,7 @@ package org.example.pojo.download.assign; import lombok.Data; -import org.example.constpool.ConstPool; +import org.example.constpool.CreeperModuleConstPool; import org.example.pojo.download.LoadBarrageConfig; /** @@ -16,7 +16,7 @@ public class BilibiliLiveLoadBarrageConfig extends LoadBarrageConfig { private String roomId; public BilibiliLiveLoadBarrageConfig(String anchorName, String roomId) { - super(ConstPool.BILIBILI, ConstPool.ACTION_LIVE, anchorName); + super(CreeperModuleConstPool.BILIBILI, CreeperModuleConstPool.ACTION_LIVE, anchorName); this.roomId = roomId; } } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java index fac591f..33243fd 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java @@ -1,7 +1,7 @@ package org.example.pojo.download.assign; import lombok.Data; -import org.example.constpool.ConstPool; +import org.example.constpool.CreeperModuleConstPool; import org.example.pojo.download.LoadBarrageConfig; /** @@ -16,7 +16,7 @@ public class DouyuRecordLoadBarrageConfig extends LoadBarrageConfig { private String vid; public DouyuRecordLoadBarrageConfig(String anchorName, String vid) { - super(ConstPool.DOUYU, ConstPool.ACTION_RECORD, anchorName); + super(CreeperModuleConstPool.DOUYU, CreeperModuleConstPool.ACTION_RECORD, anchorName); this.vid = vid; } } diff --git a/FileModule/src/main/java/org/example/cache/FileCache.java b/FileModule/src/main/java/org/example/cache/FileCache.java index 393fca7..973f479 100644 --- a/FileModule/src/main/java/org/example/cache/FileCache.java +++ b/FileModule/src/main/java/org/example/cache/FileCache.java @@ -213,7 +213,7 @@ public class FileCache { * @param key * @return */ - public Object get(String key){ + private Object get(String key){ return jsonFile.get(key); } @@ -238,7 +238,7 @@ public class FileCache { */ public void forceSync(){ if(writeByte.get()==0){ - logger.info("未发生版本变化"); + logger.debug("未发生版本变化"); return; } clearWriteBytes(); diff --git a/FileModule/src/main/java/org/example/cache/FileCacheManager.java b/FileModule/src/main/java/org/example/cache/FileCacheManager.java index fee165e..1127717 100644 --- a/FileModule/src/main/java/org/example/cache/FileCacheManager.java +++ b/FileModule/src/main/java/org/example/cache/FileCacheManager.java @@ -76,7 +76,7 @@ public class FileCacheManager { fileCaches.add(fileCache); fileCacheMap.put(fileCache.getFullFilePath(),fileCache); initSleepTime(); - FileModuleLogger.logger.info("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath()); + FileModuleLogger.logger.debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath()); } return false; } @@ -99,7 +99,7 @@ public class FileCacheManager { BlockingQueue fileChannel = cache.getFileChannel(); if(fileChannel.isEmpty()){ if(cache.needAutoSync()){ - logger.info("检测到需要强制刷新的文件 {}",cache.getFileName()); + FileModuleLogger.logger.debug("检测到需要强制刷新的文件 {}",cache.getFileName()); autoSyncer.submit(new AutoSyncer(cache)); } } diff --git a/FileModule/src/main/java/org/example/util/JsonFileUtil.java b/FileModule/src/main/java/org/example/util/JsonFileUtil.java index 03cd054..59f63cf 100644 --- a/FileModule/src/main/java/org/example/util/JsonFileUtil.java +++ b/FileModule/src/main/java/org/example/util/JsonFileUtil.java @@ -71,7 +71,6 @@ public class JsonFileUtil { T t = null; Path dir = Paths.get(fullPath); try{ - System.out.println(dir); if (FileUtil.isFileExist(dir.toString())) { String res = Files.readString(dir, StandardCharsets.UTF_8); // logger.debug("读取json文件成功, 文件内容为: {}", res); diff --git a/HotModule/src/main/java/org/example/api/HotModuleApi.java b/HotModule/src/main/java/org/example/api/HotModuleApi.java new file mode 100644 index 0000000..3cf5c4d --- /dev/null +++ b/HotModule/src/main/java/org/example/api/HotModuleApi.java @@ -0,0 +1,29 @@ +package org.example.api; + +import org.example.bean.HotLive; +import org.example.bean.hotmodule.HotModuleList; +import org.example.core.control.hotmodule.DouyuHotLiveLoadTask; +import org.example.core.control.hotmodule.DouyuHotModuleLoadTask; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/21 17:53 + **/ +public class HotModuleApi { + private static DouyuHotModuleLoadTask douyuHotModuleLoadTask = new DouyuHotModuleLoadTask(); + private static DouyuHotLiveLoadTask douyuHotLiveLoadTask = new DouyuHotLiveLoadTask(); + + public static HotModuleList getDouyuAllHotModule(){ + return douyuHotModuleLoadTask.start(); + } + + public static List getDouyuHotLive(){ + return douyuHotLiveLoadTask.start(); + } + + public static List getDouyuHotLive(int moduleId){ + return douyuHotLiveLoadTask.start(moduleId); + } +} diff --git a/HotModule/src/main/java/org/example/config/HotModuleConfig.java b/HotModule/src/main/java/org/example/config/HotModuleConfig.java index 39b458c..94b149a 100644 --- a/HotModule/src/main/java/org/example/config/HotModuleConfig.java +++ b/HotModule/src/main/java/org/example/config/HotModuleConfig.java @@ -1,7 +1,7 @@ package org.example.config; import org.example.bean.ConfigFile; -import org.example.constpool.ConstPool; +import org.example.constpool.CreeperModuleConstPool; import org.example.constpool.HotModuleConstPool; import java.nio.file.Paths; @@ -15,22 +15,20 @@ import java.util.Map; **/ public class HotModuleConfig extends ConfigFile> { private static final int FiveMinute = 0x493E0; - - + private static final long OneDay = 0x5265C00; private static final String fileName = "hotModuleConfig.json"; public HotModuleConfig(){ super(HotModuleConstPool.HOT_MODULE_CONFIG_ROOT,fileName, - Map.of("Enable", 1, - "Module", List.of( - new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(), false, - List.of(allLiveDog()), FiveMinute, FiveMinute), - new ModuleSetting(ConstPool.BILIBILI, true, new ArrayList<>(), false, - List.of(allLiveDog()), FiveMinute, FiveMinute), - new ModuleSetting(ConstPool.HUYA, true, new ArrayList<>(), false, - List.of(allLiveDog()), FiveMinute, FiveMinute), - new ModuleSetting(ConstPool.DOUYING, true, new ArrayList<>(), false, - List.of(allLiveDog()), FiveMinute, FiveMinute) + Map.of("Module", List.of( + new HotModuleSetting(CreeperModuleConstPool.DOUYU,2,true, true,true, new ArrayList<>(), false, + List.of(allLiveDog()), OneDay, FiveMinute), + new HotModuleSetting(CreeperModuleConstPool.BILIBILI, 2,true,true,true, new ArrayList<>(), false, + List.of(allLiveDog()), OneDay, FiveMinute), + new HotModuleSetting(CreeperModuleConstPool.HUYA,2, true,true,true, new ArrayList<>(), false, + List.of(allLiveDog()), OneDay, FiveMinute), + new HotModuleSetting(CreeperModuleConstPool.DOUYING, 2,true,true,true, new ArrayList<>(), false, + List.of(allLiveDog()), OneDay, FiveMinute) ), "GuardNum",10 ) diff --git a/HotModule/src/main/java/org/example/config/ModuleSetting.java b/HotModule/src/main/java/org/example/config/HotModuleSetting.java similarity index 75% rename from HotModule/src/main/java/org/example/config/ModuleSetting.java rename to HotModule/src/main/java/org/example/config/HotModuleSetting.java index 18dfd09..d39568d 100644 --- a/HotModule/src/main/java/org/example/config/ModuleSetting.java +++ b/HotModule/src/main/java/org/example/config/HotModuleSetting.java @@ -11,10 +11,16 @@ import java.util.List; **/ @Data @AllArgsConstructor -public class ModuleSetting { +public class HotModuleSetting { private String platform; //平台 + private int failRetryTimes; //失败重试次数 + + private boolean enableHotModule; //是否开启热门模块爬取 + + private boolean enableHotLive; //是否开启热门直播爬取 + private boolean autoWork; //是否自动进行主播直播下载任务推送 private List focusLiver; //关注的主播 diff --git a/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java b/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java index ea53923..60a9ff9 100644 --- a/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java +++ b/HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java @@ -8,6 +8,9 @@ import org.example.config.HotModuleConfig; * @date 2023/07/21 00:21 **/ public class HotModuleConstPool { - public static final String HOT_MODULE_CONFIG_ROOT = (String) GlobalFileCache.ModuleSrcConfigFile.get("hot","src"); + + public static final String HOT_MODULE_CONFIG_ROOT = (String) GlobalFileCache.ModuleSrcConfigFile.get("hot","src"); //热门模块配置文件路径 + + public static final String LOAD_TASK_CLASS_ROOT = "org.example.core.control.hotmodule"; //各个平台爬虫任务包路径 } diff --git a/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java b/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java new file mode 100644 index 0000000..422be63 --- /dev/null +++ b/HotModule/src/main/java/org/example/core/HotModuleDataCenter.java @@ -0,0 +1,119 @@ +package org.example.core; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.example.api.HotModuleApi; +import org.example.bean.HotLive; +import org.example.bean.HotModule; +import org.example.bean.hotmodule.HotModuleList; +import org.example.constpool.ConstPool; +import org.example.log.HotModuleLogger; +import org.example.thread.oddjob.OddJobBoy; +import org.example.util.TimeUtil; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Genius + * @date 2023/07/21 18:44 + **/ + +/** + * 热门模块的数据中心 + */ +public class HotModuleDataCenter { + + /** + * 设计这个参数的原因是，虽然HotModule可以存放HotLive数组，但是每天模块更新后会将整个hotModuleListPool替换，导致模块下的热门直播清空， + * 于是便用hotModuleLivePool来存放每个模块下的热门直播，为了保证访问效率和数据的新鲜度，这里采用 Lazy Delete的方式， + * 不用每次爬虫获取模块下的热门直播，而是查看是否过期，如果过期了才会去爬虫 + */ + private static final long HOT_MODULE_LIVE_TTL = 60; //热门模块直播过期时间 + + private static volatile HotModuleDataCenter dataCenter; + + + public static HotModuleDataCenter DataCenter(){ + if(dataCenter==null){ + synchronized (HotModuleDataCenter.class){ + if (dataCenter==null){ + dataCenter = new HotModuleDataCenter(); + } + } + } + return dataCenter; + } + + private HotModuleDataCenter(){ + hotModuleListPool = new ConcurrentHashMap<>(); + + hotModuleLivePool = new ConcurrentHashMap<>(); + for (ConstPool.PLATFORM value : ConstPool.PLATFORM.values()) { + hotModuleLivePool.put(value.getName(),new ConcurrentHashMap<>()); + } + hotLiveListPool = new ConcurrentHashMap<>(); + } + + private ConcurrentHashMap hotModuleListPool; //各个平台热门模块列表 + + public ConcurrentHashMap> hotModuleLivePool; //各个平台热门模块直播列表 + + private ConcurrentHashMap> hotLiveListPool; //各个平台热门直播列表 + + public void addModuleList(String platform,HotModuleList hotModuleList){ + hotModuleListPool.put(platform,hotModuleList); + } + + public void addLiveList(String platform,List hotLiveList){ + hotLiveListPool.put(platform,hotLiveList); + } + + public void addModuleLiveList(String platform,HotModule hotModule,List hotLiveList){ + ModuleLives moduleLives = new ModuleLives(hotLiveList, LocalDateTime.now()); + hotModule.setHotLives(hotLiveList); + hotModuleLivePool.get(platform).put(hotModule,moduleLives); + } + + public HotModuleList getModuleList(String platform){ + return hotModuleListPool.get(platform); + } + + public List getLiveList(String platform){ + return hotLiveListPool.get(platform); + } + + public List getModuleLiveList(String platform,HotModule hotModule) throws InterruptedException { + ModuleLives moduleLives = hotModuleLivePool.get(platform).get(hotModule); + if(moduleLives!=null){ + if (moduleLives.isExpire()) { + OddJobBoy.Boy().addWork(()->{ + addModuleLiveList(platform,hotModule, + HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId()))); + HotModuleLogger.logger.info("platform:{} module:{} hot lives refresh~",platform,hotModule.getTagName()); + }); + } + return moduleLives.hotLives; + }else{ + //TODO多次访问改方法时会导致链接中断 + + + List hotLives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId())); + addModuleLiveList(platform,hotModule,hotLives); + return hotLives; + } + } + + @Data + @AllArgsConstructor + class ModuleLives{ + private List hotLives; + private LocalDateTime updateTime; + + public boolean isExpire(){ + long now = TimeUtil.getCurrentSecond(); + return now - TimeUtil.getSecond(updateTime)>HOT_MODULE_LIVE_TTL; + } + } +} diff --git a/HotModule/src/main/java/org/example/guard/Guard.java b/HotModule/src/main/java/org/example/guard/Guard.java new file mode 100644 index 0000000..4812860 --- /dev/null +++ b/HotModule/src/main/java/org/example/guard/Guard.java @@ -0,0 +1,90 @@ +package org.example.guard; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.example.bean.HotLive; +import org.example.bean.hotmodule.HotModuleList; +import org.example.constpool.ConstPool; +import org.example.core.HotModuleDataCenter; +import org.example.core.control.HotModuleLoadTask; +import org.example.core.control.LoadTask; +import org.example.guard.HotModuleGuardInstance; +import org.example.log.HotModuleLogger; +import org.example.log.ResultLogger; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.*; + +/** + * @author Genius + * @date 2023/07/21 02:13 + **/ + +/** + * 热度监控守卫，负责执行各个平台的热度监控工作 + */ +@Data +@AllArgsConstructor +public class Guard implements Runnable, ResultLogger { + + private String guardName; + private HotModuleLoadTask task; + + private long delayTime; + + private int failRetryTimes; + @Override + public void run() { + + int retryTimes = 0; + Object data = task.start(); + if(task.isFinish() == HotModuleLoadTask.FinishFlag.FINISH){ + successLog(); + }else{ + while(task.isFinish()== HotModuleLoadTask.FinishFlag.FAIL){ + if(retryTimes) data); + } + } + + @Override + public void successLog() { + HotModuleLogger.logger.info("{} successfully finish!",guardName); + } + + @Override + public void successLog(String str) { + + } + + @Override + public void failLog() { + HotModuleLogger.logger.error("{} finish error,cancel this task!",guardName); + } + + @Override + public void failLog(String str) { + HotModuleLogger.logger.error("{} fail try to redo,retry times:{}!",guardName,str); + } + +} diff --git a/HotModule/src/main/java/org/example/guard/HotModuleGuard.java b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java index c9a0a61..9a7d7bc 100644 --- a/HotModule/src/main/java/org/example/guard/HotModuleGuard.java +++ b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java @@ -1,18 +1,18 @@ package org.example.guard; +import org.example.bean.HotLive; +import org.example.bean.hotmodule.HotModuleList; import org.example.cache.FileCache; import org.example.cache.FileCacheManager; import org.example.cache.FileCacheManagerInstance; import org.example.config.HotModuleConfig; import org.example.constpool.HotModuleConstPool; -import org.example.core.control.impl.DouyuHotLiveLoadTask; -import org.example.core.control.impl.DouyuHotModuleLoadTask; -import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; +import org.example.core.control.HotModuleLoadTask; +import org.example.thread.NamedThreadFactory; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; /** * @author Genius @@ -20,24 +20,76 @@ import java.util.concurrent.TimeUnit; **/ public class HotModuleGuard { - private static final long delayTime = 10*1000; + private List guards; //热度监控守卫列表，用于初始化一开始的热度监控列表 + private ScheduledExecutorService hotModuleGuardPool; //热度监控守卫 定时线程池 + private Map runningGuards; // 运行的热度监控守卫 - private FileCache HotModuleFileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()); - private ScheduledExecutorService hotModuleGuardPool = Executors.newScheduledThreadPool( - (Integer)FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()).get("GuardNum") - ); + + + protected HotModuleGuard(List guards,int guardNum){ + this.guards = guards; + this.hotModuleGuardPool = Executors.newScheduledThreadPool(guardNum, new NamedThreadFactory("HotModuleGuard")); + runningGuards = new ConcurrentHashMap<>(); + } + + + private void guardStart(Guard guard){ + ScheduledFuture scheduledFuture = hotModuleGuardPool.scheduleWithFixedDelay( + guard, 0, guard.getDelayTime(), TimeUnit.MILLISECONDS + ); + runningGuards.put(guard.getGuardName(),scheduledFuture); + } public void start(){ - hotModuleGuardPool.scheduleWithFixedDelay(()-> { - new DouyuHotLiveLoadTask().start(); - },0,delayTime, TimeUnit.MILLISECONDS); - - hotModuleGuardPool.scheduleWithFixedDelay(()->{ - new DouyuHotModuleLoadTask().start(); - },0,delayTime,TimeUnit.MILLISECONDS); + if(runningGuards.size()==0){ + try { + for (Guard guard : guards) { + guardStart(guard); + } + }catch (Exception e){ + throw e; + } + } } - public static void main(String[] args) { - new HotModuleGuard().start(); + public void end(){ + hotModuleGuardPool.shutdown(); + runningGuards.clear(); } + + public boolean addGuard(String platform,boolean isHotModule){ + FileCache fileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()); + platform = platform.substring(0,1).toUpperCase() + platform.substring(1); + String clazzName = platform+"Hot"+(isHotModule?"Module":"Live")+"LoadTask"; + String clazz = HotModuleConstPool.LOAD_TASK_CLASS_ROOT+"."+clazzName; + String timeName = isHotModule?"updateHotModuleTimes":"updateHotLivesTimes"; + try { + addGuard(new Guard( + clazzName.toLowerCase(), + (HotModuleLoadTask)Class.forName(clazz).getDeclaredConstructor().newInstance(), + (long)fileCache.get(timeName), + (int)fileCache.get("failRetryTimes") + )); + }catch (Exception e){ + return false; + } + return true; + } + public boolean addGuard(Guard guard){ + if(!runningGuards.containsKey(guard.getGuardName())){ + guardStart(guard); + return true; + } + return false; + } + + public boolean unActiveGuard(String guardName){ + if (runningGuards.containsKey(guardName)) { + runningGuards.get(guardName).cancel(false); + runningGuards.remove(guardName); + return true; + } + return false; + } + } diff --git a/HotModule/src/main/java/org/example/guard/HotModuleGuardInstance.java b/HotModule/src/main/java/org/example/guard/HotModuleGuardInstance.java new file mode 100644 index 0000000..ef464d7 --- /dev/null +++ b/HotModule/src/main/java/org/example/guard/HotModuleGuardInstance.java @@ -0,0 +1,34 @@ +package org.example.guard; + + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Genius + * @date 2023/07/21 11:39 + **/ +public class HotModuleGuardInstance { + + private static List guardList = new ArrayList<>(); + + private static int guardNum; + private static volatile HotModuleGuard Instance; + + public static HotModuleGuard getInstance(){ + if(Instance==null){ + synchronized (HotModuleGuardInstance.class){ + if(Instance==null){ + Instance = new HotModuleGuard(guardList,guardNum); + } + } + } + return Instance; + } + + public static void InitInstance(List guards,int num){ + guardList = guards; + guardNum = num; + } + +} diff --git a/HotModule/src/main/java/org/example/guard/task/Guard.java b/HotModule/src/main/java/org/example/guard/task/Guard.java deleted file mode 100644 index c2cb954..0000000 --- a/HotModule/src/main/java/org/example/guard/task/Guard.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.example.guard.task; - -/** - * @author Genius - * @date 2023/07/21 02:13 - **/ -public class Guard { -} diff --git a/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java b/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java new file mode 100644 index 0000000..2403124 --- /dev/null +++ b/HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java @@ -0,0 +1,80 @@ +package org.example.init; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.example.cache.FileCache; +import org.example.cache.FileCacheManagerInstance; +import org.example.config.HotModuleConfig; +import org.example.config.HotModuleSetting; +import org.example.core.control.HotModuleLoadTask; +import org.example.guard.HotModuleGuardInstance; +import org.example.guard.Guard; +import org.example.log.HotModuleLogger; +import org.example.util.ClassUtil; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.example.constpool.HotModuleConstPool.LOAD_TASK_CLASS_ROOT; + +/** + * @author Genius + * @date 2023/07/21 09:58 + **/ +public class HotModuleGuardInitMachine extends CommonInitMachine{ + + + public HotModuleGuardInitMachine() { + super(HotModuleLogger.logger); + } + + private void envInit() throws Exception { + FileCache HotModuleFileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()); + + List guards = new ArrayList<>(); + int guardNum = (Integer)HotModuleFileCache.get("GuardNum"); + Map map = new HashMap<>(); + JSONArray modules = (JSONArray)HotModuleFileCache.get("Module"); + for (Object module : modules) { + HotModuleSetting hotModuleSetting = JSONObject.parseObject(module.toString(), HotModuleSetting.class); + map.put(hotModuleSetting.getPlatform(),hotModuleSetting); + } + + for (String clazz : ClassUtil.getClassesInPackage(LOAD_TASK_CLASS_ROOT)) { + String[] split = clazz.split("\\."); + String clazzName = split[split.length-1].toLowerCase(); + if(clazzName.endsWith("loadtask")&& clazzName.contains("hot")){ + String platformName = clazzName.split("hot")[0]; + boolean isHotModule = clazzName.contains("module"); + if(map.containsKey(platformName)){ + HotModuleSetting hotModuleSetting = map.get(platformName); + Class loadClazz = Class.forName(clazz); + if(isHotModule&&hotModuleSetting.isEnableHotModule()){ + HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance(); + guards.add(new Guard(clazzName,task, + hotModuleSetting.getUpdateHotModuleTimes(),hotModuleSetting.getFailRetryTimes())); + }else if(hotModuleSetting.isEnableHotLive()){ + HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance(); + guards.add(new Guard(clazzName,task, + hotModuleSetting.getUpdateHotLivesTimes(),hotModuleSetting.getFailRetryTimes())); + } + } + } + } + HotModuleGuardInstance.InitInstance(guards,guardNum); + HotModuleGuardInstance.getInstance().start(); + } + + @Override + public boolean init() { + try { + envInit(); + } catch (Exception e) { + return fail(e.getMessage()); + } + return success(); + } + +} diff --git a/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java b/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java index 94c3949..cbc3777 100644 --- a/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java +++ b/HotModule/src/main/java/org/example/init/HotModuleInitMachine.java @@ -18,7 +18,7 @@ public class HotModuleInitMachine extends ModuleInitMachine{ public HotModuleInitMachine() { super( - List.of(new HotModuleConfigInitMachine()), + List.of(new HotModuleConfigInitMachine(),new HotModuleGuardInitMachine()), "HotModule", HotModuleLogger.logger ); diff --git a/common/src/main/java/org/example/constpool/ConstPool.java b/common/src/main/java/org/example/constpool/ConstPool.java index 74c6402..04164f5 100644 --- a/common/src/main/java/org/example/constpool/ConstPool.java +++ b/common/src/main/java/org/example/constpool/ConstPool.java @@ -22,6 +22,24 @@ public class ConstPool { public static final List PIC_TYPES = List.of("jpg","jpeg","png","svg"); /**直播平台**/ + + public enum PLATFORM{ + DOUYU("douyu"), + HUYA("huya"), + BILIBILI("bilibili"), + DOUYING("douyin"), + TIKTOK("tiktok"), + TWITCH("twitch"); + private final String name; + PLATFORM(String name){ + this.name = name; + } + + public String getName(){ + return name; + } + + } public static final String DOUYU = "douyu"; public static final String HUYA = "huya"; diff --git a/common/src/main/java/org/example/constpool/HotModulePool.java b/common/src/main/java/org/example/constpool/HotModulePool.java deleted file mode 100644 index 272d564..0000000 --- a/common/src/main/java/org/example/constpool/HotModulePool.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.example.constpool; - -import org.example.bean.HotLive; -import org.example.bean.hotmodule.HotModuleList; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author Genius - * @date 2023/07/18 22:16 - **/ -public class HotModulePool { - - static { - hotModuleListPool = new ConcurrentHashMap<>(); - hotLiveListPool = new ConcurrentHashMap<>(); - } - - public static final String DouYuAllHotModules = "DouyuAllHotModules"; - - public static final String DouYuAllHotLives = "DouyuAllHotLives"; - public static ConcurrentHashMap hotModuleListPool; - - public static ConcurrentHashMap> hotLiveListPool; -} diff --git a/common/src/main/java/org/example/init/CommonInitMachine.java b/common/src/main/java/org/example/init/CommonInitMachine.java index 02f5884..76174c2 100644 --- a/common/src/main/java/org/example/init/CommonInitMachine.java +++ b/common/src/main/java/org/example/init/CommonInitMachine.java @@ -46,7 +46,7 @@ public abstract class CommonInitMachine implements ComponentInitMachine{ @Override public boolean fail(String failCause) { - failLog(String.format("[❌] {%s} init error! Execption:{}",this.getClass().toString(),failCause)); + failLog(String.format("[❌] {%s} init error! Execption:{%s}",this.getClass().toString(),failCause)); return false; } diff --git a/common/src/main/java/org/example/init/InitMachine.java b/common/src/main/java/org/example/init/InitMachine.java index c259530..7f5cb17 100644 --- a/common/src/main/java/org/example/init/InitMachine.java +++ b/common/src/main/java/org/example/init/InitMachine.java @@ -1,7 +1,7 @@ package org.example.init; -import org.example.log.InitMachineLogger; +import org.example.log.ResultLogger; /** * @author Genius @@ -9,7 +9,7 @@ import org.example.log.InitMachineLogger; **/ //模块初始化接口 -public interface InitMachine extends InitMachineLogger { +public interface InitMachine extends ResultLogger { boolean init(); } diff --git a/common/src/main/java/org/example/log/InitMachineLogger.java b/common/src/main/java/org/example/log/ResultLogger.java similarity index 82% rename from common/src/main/java/org/example/log/InitMachineLogger.java rename to common/src/main/java/org/example/log/ResultLogger.java index 435aa9f..d16f0cb 100644 --- a/common/src/main/java/org/example/log/InitMachineLogger.java +++ b/common/src/main/java/org/example/log/ResultLogger.java @@ -4,7 +4,7 @@ package org.example.log; /** * 启动日志接口 */ -public interface InitMachineLogger { +public interface ResultLogger { void successLog(); diff --git a/common/src/main/java/org/example/thread/NamedThreadFactory.java b/common/src/main/java/org/example/thread/NamedThreadFactory.java new file mode 100644 index 0000000..478495d --- /dev/null +++ b/common/src/main/java/org/example/thread/NamedThreadFactory.java @@ -0,0 +1,28 @@ +package org.example.thread; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Genius + * @date 2023/07/21 16:58 + **/ + +/** + * 更改线程池名字 + */ +public class NamedThreadFactory implements ThreadFactory { + private final String poolName; + private final AtomicInteger threadNumber = new AtomicInteger(1); + + public NamedThreadFactory(String poolName) { + this.poolName = poolName; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("ChopperBot-"+poolName + "-" + threadNumber.getAndIncrement()); + return t; + } +} diff --git a/common/src/main/java/org/example/thread/oddjob/OddJob.java b/common/src/main/java/org/example/thread/oddjob/OddJob.java new file mode 100644 index 0000000..10ee049 --- /dev/null +++ b/common/src/main/java/org/example/thread/oddjob/OddJob.java @@ -0,0 +1,9 @@ +package org.example.thread.oddjob; + +/** + * @author Genius + * @date 2023/07/21 19:22 + **/ +public interface OddJob{ + void doJob(); +} diff --git a/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java b/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java new file mode 100644 index 0000000..9b0976b --- /dev/null +++ b/common/src/main/java/org/example/thread/oddjob/OddJobBoy.java @@ -0,0 +1,68 @@ +package org.example.thread.oddjob; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author Genius + * @date 2023/07/21 19:18 + **/ + +/** + * ChopperBot系统中专门用来处理异步事件的类 + */ +public class OddJobBoy { + + private static volatile OddJobBoy Instance; + + private BlockingQueue oddjobs; + + private ExecutorService home; + + private OddJobBoy(){ + home = Executors.newSingleThreadExecutor(); + oddjobs = new ArrayBlockingQueue<>(1024); + } + + public static OddJobBoy Boy(){ + if(Instance==null){ + synchronized (OddJobBoy.class){ + if(Instance==null){ + Instance = new OddJobBoy(); + Instance.work(); + } + } + } + return Instance; + } + + public void addWork(OddJob job) throws InterruptedException { + oddjobs.put(job); + } + + private void work(){ + home.submit(new Boy()); + } + + public boolean relax(){ + home.shutdown(); + return home.isShutdown(); + } + + class Boy implements Runnable{ + + @Override + public void run() { + while(true){ + try { + OddJob job = oddjobs.take(); + job.doJob(); + }catch (InterruptedException e){ + + } + } + } + } +} diff --git a/common/src/main/java/org/example/util/ClassUtil.java b/common/src/main/java/org/example/util/ClassUtil.java new file mode 100644 index 0000000..d114c6b --- /dev/null +++ b/common/src/main/java/org/example/util/ClassUtil.java @@ -0,0 +1,41 @@ +package org.example.util; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @author Genius + * @date 2023/07/21 09:56 + **/ +public class ClassUtil { + public static List getClassesInPackage(String packageName)throws IOException { + List classNames = new ArrayList<>(); + + String packagePath = packageName.replace(".", "/"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + var resources = classLoader.getResources(packagePath); + while (resources.hasMoreElements()) { + var resource = resources.nextElement(); + File file = new File(resource.getFile()); + if (file.isDirectory()) { + scanClassesInDirectory(packageName, file, classNames); + } + } + + return classNames; + } + + private static void scanClassesInDirectory(String packageName, File directory, List classNames) { + for (File file : directory.listFiles()) { + if (file.isFile()) { + String className = packageName + "." + file.getName().replace(".class", ""); + classNames.add(className); + } else if (file.isDirectory()) { + scanClassesInDirectory(packageName + "." + file.getName(), file, classNames); + } + } + } +} diff --git a/console/src/main/java/org/example/controller/HotController.java b/console/src/main/java/org/example/controller/HotController.java new file mode 100644 index 0000000..43fd75a --- /dev/null +++ b/console/src/main/java/org/example/controller/HotController.java @@ -0,0 +1,57 @@ +package org.example.controller; + +import com.genius.assistant.common.Result; +import org.example.api.HotModuleApi; +import org.example.bean.HotLive; +import org.example.bean.HotModule; +import org.example.bean.hotmodule.HotModuleList; +import org.example.constpool.ConstPool; +import org.example.core.HotModuleDataCenter; +import org.example.service.HotModuleService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/21 17:13 + **/ +@RestController +@RequestMapping("/hot") +public class HotController { + + @Autowired + HotModuleService hotModuleService; + + @GetMapping("/douyu/allHotLive") + public Result getDouyuAllHotLive(@RequestParam(defaultValue = "0") int latest){ + List hotLives; + if(latest==1){ + hotLives = HotModuleApi.getDouyuHotLive(); + }else{ + hotLives = HotModuleDataCenter.DataCenter().getLiveList(ConstPool.PLATFORM.DOUYU.getName()); + } + return Result.success(hotLives); + } + + @GetMapping("/douyu/allHotModule") + public Result getDouyuAllHotModule(@RequestParam(defaultValue = "0") int latest){ + HotModuleList hotModuleList; + if(latest==1){ + hotModuleList = HotModuleApi.getDouyuAllHotModule(); + }else{ + hotModuleList = HotModuleDataCenter.DataCenter().getModuleList(ConstPool.PLATFORM.DOUYU.getName()); + } + return Result.success(hotModuleList.getHotModuleList()); + } + + @GetMapping("/douyu/getHotModuleLives") + public Result getDouyuHotModuleLives(@RequestParam int moduleId){ + HotModule moduleHotLives = hotModuleService.getModuleHotLives(ConstPool.PLATFORM.DOUYU.getName(), moduleId); + return Result.success(moduleHotLives); + } +} diff --git a/FileModule/src/main/java/org/example/service/FileService.java b/console/src/main/java/org/example/service/FileService.java similarity index 100% rename from FileModule/src/main/java/org/example/service/FileService.java rename to console/src/main/java/org/example/service/FileService.java diff --git a/console/src/main/java/org/example/service/HotModuleService.java b/console/src/main/java/org/example/service/HotModuleService.java new file mode 100644 index 0000000..e27daac --- /dev/null +++ b/console/src/main/java/org/example/service/HotModuleService.java @@ -0,0 +1,14 @@ +package org.example.service; + +import org.example.bean.HotModule; +import org.example.bean.hotmodule.HotModuleList; +import org.springframework.stereotype.Service; + +import java.util.List; + + +public interface HotModuleService { + + + HotModule getModuleHotLives(String platform, int moduleId); +} diff --git a/FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java b/console/src/main/java/org/example/service/impl/FileServiceImpl.java similarity index 100% rename from FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java rename to console/src/main/java/org/example/service/impl/FileServiceImpl.java diff --git a/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java b/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java new file mode 100644 index 0000000..2ec7ac2 --- /dev/null +++ b/console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java @@ -0,0 +1,45 @@ +package org.example.service.impl; + +import org.example.bean.HotLive; +import org.example.bean.HotModule; +import org.example.bean.hotmodule.HotModuleList; +import org.example.constpool.HotModuleConstPool; +import org.example.core.HotModuleDataCenter; +import org.example.service.HotModuleService; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/21 17:23 + **/ + +@Service +public class HotModuleServiceImpl implements HotModuleService { + + /** + * 获得热门模块下的热门直播 + * @param moduleId + * @return + */ + @Override + public HotModule getModuleHotLives(String platform,int moduleId) { + HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform); + if(moduleList==null){ + return null; + } + HotModule hotModule = moduleList.findHotModule(moduleId); + if(hotModule!=null){ + try { + List moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule); + hotModule.setHotLives(moduleLiveList); + return hotModule; + }catch (Exception e){ + //TODO 交给Spring全局异常处理器 + return null; + } + } + return null; + } +} diff --git a/console/src/main/resources/application.yaml b/console/src/main/resources/application.yaml index 24468f3..1730362 100644 --- a/console/src/main/resources/application.yaml +++ b/console/src/main/resources/application.yaml @@ -1,5 +1,6 @@ server: port: 8888 + spring: datasource: url: jdbc:mysql://localhost:3306/illuminator?useSSL=false&serverTimezone=UTC