1，完善斗鱼的热度模块获取以及热度直播获取

2，更新HotLive模块
3，完善HotModule配置类
4，更新FileCacheManager

CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java

@@ -12,20 +12,41 @@ import us.codecraft.webmagic.Spider;
  **/
 public class DouyuHotLiveLoadTask implements LoadTask {
 
-    private String url = "https://www.douyu.com/japi/search/api/getHotList";
+    private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1";             //全部热门直播api
+    private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
+
+    private final DouyuHotLiveProcessor douyuHotLiveProcessor;
+
+    public DouyuHotLiveLoadTask(){
+        douyuHotLiveProcessor = new DouyuHotLiveProcessor();
+    }
+
+    /**
+     * 获取Douyu某个模块下的热门直播
+     * @param moduleId
+     */
+    public void start(int moduleId){
+        douyuHotLiveProcessor.setModuleId(moduleId);
+        this.start(String.format(HOT_MODULE_LIVES_API,moduleId));
+    }
+
+    /**
+     * 获取Douyu当前最热直播
+     */
     @Override
     public void start() {
+       this.start(HOT_LIVES_API);
+    }
+
+    private void start(String url){
         try {
-            DouyuHotLiveProcessor douyuHotLiveProcessor = new DouyuHotLiveProcessor();
             Spider.create(douyuHotLiveProcessor)
                     .addRequest(new Request(url))
-                    .setEmptySleepTime(10)
                     .thread(1)
                     .run();
         }catch (Exception e){
             e.printStackTrace();
         }
-
     }
 
     @Override
@@ -49,7 +70,8 @@ public class DouyuHotLiveLoadTask implements LoadTask {
     }
 
     public static void main(String[] args) {
-        new DouyuHotLiveLoadTask().start();
-        System.out.println(HotModulePool.hotLiveListPool);
+        new DouyuHotModuleLoadTask().start();
+        new DouyuHotLiveLoadTask().start(1);
+        System.out.println(HotModulePool.hotModuleListPool);
     }
 }

CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java

@@ -12,13 +12,12 @@ import us.codecraft.webmagic.Spider;
  **/
 public class DouyuHotModuleLoadTask implements LoadTask {
 
-    private String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
+    private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
     @Override
     public void start() {
         DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor();
         Spider.create(douyuHotModuleProcessor)
                 .addRequest(new Request(url))
-                .setEmptySleepTime(10)
                 .thread(1)
                 .run();
     }

CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java

@@ -4,49 +4,81 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import org.example.bean.HotLive;
+import org.example.bean.HotModule;
 import org.example.bean.hotmodule.DouyuHotLive;
+import org.example.bean.hotmodule.HotModuleList;
 import org.example.constpool.HotModulePool;
 import org.example.log.HotModuleLogger;
 import org.example.util.ChineseConvertUtil;
+import org.springframework.util.StringUtils;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.processor.PageProcessor;
 import us.codecraft.webmagic.selector.Json;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * @author Genius
  * @date 2023/07/19 00:30
  **/
 public class DouyuHotLiveProcessor implements PageProcessor {
+    private int moduleId;         // 模块Id
 
+    public DouyuHotLiveProcessor(){
+        this.moduleId = -1;
+    }
+
+    public void setModuleId(int moduleId) {
+        this.moduleId = moduleId;
+    }
 
     @Override
     public void process(Page page) {
 
         List<HotLive> hotLiveList = new ArrayList<>();
         try{
-            JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("data");
+            JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONObject("data").getJSONArray("rl");
+
             for (Object live : Lives) {
                 if(live instanceof JSONObject){
                     JSONObject jsonLive = (JSONObject) live;
                     hotLiveList.add(new DouyuHotLive(
-                            ChineseConvertUtil.cnNumericUnitsToInt(jsonLive.getString("hot")),
-                            Integer.parseInt(jsonLive.getString("relId")),
-                            jsonLive.getString("sk"),
-                            jsonLive.containsKey("description")?jsonLive.getString("description"):"",
-                            jsonLive.getInteger("hotTag"),
-                            jsonLive.getInteger("isAd"),
+                            jsonLive.getInteger("ol"),
+                            jsonLive.getInteger("rid"),
+                            jsonLive.getString("rn"),
+                            jsonLive.getString("nn"),
+                            jsonLive.getString("od"),
+                            jsonLive.getString("c2name"),
+                            jsonLive.getString("url"),
+                            jsonLive.getString("rs16"),
                             jsonLive.getInteger("type"),
-                            jsonLive.getString("url")
+                            jsonLive.getInteger("uid"),
+                            jsonLive.getInteger("cid2")
                     ));
                 }
             }
         }catch (Exception e){
-            HotModuleLogger.logger.error("DouYu Hot Live List require fail! Exception:{}",e.getMessage());
+            HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage());
         }
-        HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
+        HotModuleLogger.logger.info(hotLiveList.toString());
+        updateHotLiveList(hotLiveList);
 
     }
+    private void updateHotLiveList(List<HotLive> hotLiveList){
+        if(moduleId!=-1){
+            if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){
+                HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules);
+                HotModule hotModule = hotModuleList.findHotModule(this.moduleId);
+                if(hotModule!=null){
+                    hotModule.setHotLives(hotLiveList);
+                    HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName());
+                }
+            }
+        }else{
+            HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated");
+            HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
+        }
+    }
 }

CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java

@@ -6,16 +6,18 @@ import com.alibaba.fastjson.JSONObject;
 import org.example.bean.hotmodule.DouyuHotModule;
 import org.example.bean.hotmodule.HotModuleList;
 import org.example.constpool.HotModulePool;
+import org.example.log.HotModuleLogger;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.processor.PageProcessor;
 
+import java.util.concurrent.ExecutorService;
+
 /**
  * @author Genius
  * @date 2023/06/01 22:11
  **/
 public class DouyuHotModuleProcessor implements PageProcessor {
 
-
     @Override
     public void process(Page page) {
         HotModuleList douyuHotModuleList = new HotModuleList();
@@ -36,7 +38,7 @@ public class DouyuHotModuleProcessor implements PageProcessor {
                 }
             }
         }catch (Exception e){
-
+            HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage());
         }
         HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList);
     }