diff --git a/.idea/compiler.xml b/.idea/compiler.xml index b9b5754..b6c4da6 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -21,16 +21,11 @@ - - - - \ No newline at end of file diff --git a/CreeperModule/src/main/java/org/example/Test.java b/CreeperModule/src/main/java/org/example/Test.java index a2fae73..23ddd74 100644 --- a/CreeperModule/src/main/java/org/example/Test.java +++ b/CreeperModule/src/main/java/org/example/Test.java @@ -1,7 +1,7 @@ package org.example; import org.example.core.manager.LoadTaskManager; -import org.example.pojo.download.assign.BilibiliLiveLoadConfig; +import org.example.pojo.download.assign.BilibiliLiveLoadBarrageConfig; public class Test { @@ -9,7 +9,7 @@ public class Test { // 创建一个任务管理器 LoadTaskManager manager = new LoadTaskManager(); // 创建一个B站直播的配置类 - BilibiliLiveLoadConfig zzgz = new BilibiliLiveLoadConfig("猪猪公主", "6154037"); + BilibiliLiveLoadBarrageConfig zzgz = new BilibiliLiveLoadBarrageConfig("猪猪公主", "6154037"); // 创建一个任务,返回一个唯一的key String task = manager.creatTask(zzgz); // 开启此次下载任务 diff --git a/CreeperModule/src/main/java/org/example/constpool/ApiPool.java b/CreeperModule/src/main/java/org/example/constpool/ApiPool.java new file mode 100644 index 0000000..7ca268f --- /dev/null +++ b/CreeperModule/src/main/java/org/example/constpool/ApiPool.java @@ -0,0 +1,15 @@ +package org.example.constpool; + +/** + * @author Genius + * @date 2023/06/01 22:29 + **/ + + +/** + * 存放每一个爬虫api的池子 + */ +public class ApiPool { + public final static String HOT_MODULE_LIST_API = "/japi/search/api/getHotList"; + +} diff --git a/CreeperModule/src/main/java/org/example/constpool/ConstPool.java b/CreeperModule/src/main/java/org/example/constpool/ConstPool.java index 509e8ce..14637f0 100644 --- a/CreeperModule/src/main/java/org/example/constpool/ConstPool.java +++ b/CreeperModule/src/main/java/org/example/constpool/ConstPool.java @@ -38,4 +38,6 @@ public class ConstPool { */ // 爬虫启动占位url public static final String OCCUURL = "https://v.douyu.com/"; + + public static final String DOUYU_URL = "www.douyu.com"; } diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java index d270876..ed5203e 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java @@ -5,7 +5,7 @@ import org.example.core.control.LoadTask; import org.example.core.factory.ProcessorFactory; import org.example.core.pipeline.PipelineWriteJson; import org.example.core.processor.BilibiliLiveProcessor; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; import org.example.utils.CreeperConfig; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Spider; @@ -25,9 +25,9 @@ public class BilibiliLiveLoadTask implements LoadTask { private final PipelineWriteJson pipelineWriteJson; - public BilibiliLiveLoadTask(LoadConfig loadConfig) { - bilibiliLiveProcessor = (BilibiliLiveProcessor) new ProcessorFactory().getProcessor(loadConfig); - pipelineWriteJson = new PipelineWriteJson(loadConfig); + public BilibiliLiveLoadTask(LoadBarrageConfig loadBarrageConfig) { + bilibiliLiveProcessor = (BilibiliLiveProcessor) new ProcessorFactory().getProcessor(loadBarrageConfig); + pipelineWriteJson = new PipelineWriteJson(loadBarrageConfig); } @Override diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java new file mode 100644 index 0000000..292238d --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java @@ -0,0 +1,55 @@ +package org.example.core.control.impl; + +import org.example.constpool.HotModulePool; +import org.example.core.control.LoadTask; +import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.Spider; + +/** + * @author Genius + * @date 2023/07/19 02:42 + **/ +public class DouyuHotLiveLoadTask implements LoadTask { + + private String url = "https://www.douyu.com/japi/search/api/getHotList"; + @Override + public void start() { + try { + DouyuHotLiveProcessor douyuHotLiveProcessor = new DouyuHotLiveProcessor(); + Spider.create(douyuHotLiveProcessor) + .addRequest(new Request(url)) + .setEmptySleepTime(10) + .thread(1) + .run(); + }catch (Exception e){ + e.printStackTrace(); + } + + } + + @Override + public void end() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public int getCacheSize() { + return 0; + } + + @Override + public int flushCacheAndSave(String key) { + return 0; + } + + public static void main(String[] args) { + new DouyuHotLiveLoadTask().start(); + System.out.println(HotModulePool.hotLiveListPool); + } +} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java new file mode 100644 index 0000000..0b87725 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java @@ -0,0 +1,49 @@ +package org.example.core.control.impl; + +import org.example.constpool.HotModulePool; +import org.example.core.control.LoadTask; +import org.example.core.processor.hotmodule.DouyuHotModuleProcessor; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.Spider; + +/** + * @author Genius + * @date 2023/07/15 21:04 + **/ +public class DouyuHotModuleLoadTask implements LoadTask { + + private String url = "https://www.douyu.com/japi/weblist/apinc/header/cate"; + @Override + public void start() { + DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor(); + Spider.create(douyuHotModuleProcessor) + .addRequest(new Request(url)) + .setEmptySleepTime(10) + .thread(1) + .run(); + } + + @Override + public void end() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public int getCacheSize() { + return 0; + } + + @Override + public int flushCacheAndSave(String key) { + return 0; + } + + public static void main(String[] args) { + new DouyuHotModuleLoadTask().start(); + } +} diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java index 0c811a2..528f1f5 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java @@ -5,7 +5,7 @@ import org.example.core.control.LoadTask; import org.example.core.factory.ProcessorFactory; import org.example.core.pipeline.PipelineWriteJson; import org.example.core.processor.DouyuRecordProcessor; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; import org.example.utils.CreeperConfig; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Spider; @@ -25,9 +25,9 @@ public class DouyuRecordLoadTask implements LoadTask { private final PipelineWriteJson pipelineWriteJson; - public DouyuRecordLoadTask(LoadConfig loadConfig) { - douyuRecordProcessor = (DouyuRecordProcessor) new ProcessorFactory().getProcessor(loadConfig); - pipelineWriteJson = new PipelineWriteJson(loadConfig); + public DouyuRecordLoadTask(LoadBarrageConfig loadBarrageConfig) { + douyuRecordProcessor = (DouyuRecordProcessor) new ProcessorFactory().getProcessor(loadBarrageConfig); + pipelineWriteJson = new PipelineWriteJson(loadBarrageConfig); } @Override diff --git a/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java b/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java index 2ecd635..c967ece 100644 --- a/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java +++ b/CreeperModule/src/main/java/org/example/core/factory/LoadTaskFactory.java @@ -4,9 +4,9 @@ import org.example.core.control.LoadTask; import org.example.core.control.impl.BilibiliLiveLoadTask; import org.example.core.control.impl.DouyuRecordLoadTask; import org.example.exception.FileCacheException; -import org.example.pojo.download.LoadConfig; -import org.example.pojo.download.assign.BilibiliLiveLoadConfig; -import org.example.pojo.download.assign.DouyuRecordLoadConfig; +import org.example.pojo.download.LoadBarrageConfig; +import org.example.pojo.download.assign.BilibiliLiveLoadBarrageConfig; +import org.example.pojo.download.assign.DouyuRecordLoadBarrageConfig; /** * 弹幕下载任务工厂 @@ -17,22 +17,22 @@ public class LoadTaskFactory { /** * 通过配置信息来获取一个任务 - * @param loadConfig + * @param loadBarrageConfig * @return LoadTask */ - public LoadTask getLoadTask(LoadConfig loadConfig) throws FileCacheException { + public LoadTask getLoadTask(LoadBarrageConfig loadBarrageConfig) throws FileCacheException { - if (loadConfig == null) { + if (loadBarrageConfig == null) { return null; } // 斗鱼录播 - if (loadConfig instanceof DouyuRecordLoadConfig) { - return new DouyuRecordLoadTask((DouyuRecordLoadConfig) loadConfig); + if (loadBarrageConfig instanceof DouyuRecordLoadBarrageConfig) { + return new DouyuRecordLoadTask((DouyuRecordLoadBarrageConfig) loadBarrageConfig); } // B站直播 - else if (loadConfig instanceof BilibiliLiveLoadConfig) { - return new BilibiliLiveLoadTask((BilibiliLiveLoadConfig) loadConfig); + else if (loadBarrageConfig instanceof BilibiliLiveLoadBarrageConfig) { + return new BilibiliLiveLoadTask((BilibiliLiveLoadBarrageConfig) loadBarrageConfig); } return null; diff --git a/CreeperModule/src/main/java/org/example/core/factory/ProcessorFactory.java b/CreeperModule/src/main/java/org/example/core/factory/ProcessorFactory.java index fd4ae81..79f7151 100644 --- a/CreeperModule/src/main/java/org/example/core/factory/ProcessorFactory.java +++ b/CreeperModule/src/main/java/org/example/core/factory/ProcessorFactory.java @@ -3,9 +3,9 @@ package org.example.core.factory; import org.example.core.processor.AbstractProcessor; import org.example.core.processor.BilibiliLiveProcessor; import org.example.core.processor.DouyuRecordProcessor; -import org.example.pojo.download.LoadConfig; -import org.example.pojo.download.assign.BilibiliLiveLoadConfig; -import org.example.pojo.download.assign.DouyuRecordLoadConfig; +import org.example.pojo.download.LoadBarrageConfig; +import org.example.pojo.download.assign.BilibiliLiveLoadBarrageConfig; +import org.example.pojo.download.assign.DouyuRecordLoadBarrageConfig; import org.example.utils.CreeperConfig; /** @@ -29,23 +29,23 @@ public class ProcessorFactory { /** * 通过配置信息来获取一个处理器 * - * @param loadConfig + * @param loadBarrageConfig * @return AbstractProcessor */ - public AbstractProcessor getProcessor(LoadConfig loadConfig) { + public AbstractProcessor getProcessor(LoadBarrageConfig loadBarrageConfig) { - if (loadConfig == null) { + if (loadBarrageConfig == null) { return null; } // 斗鱼录播 - if (loadConfig instanceof DouyuRecordLoadConfig) { - return new DouyuRecordProcessor((DouyuRecordLoadConfig) loadConfig, dy_retryTimes, + if (loadBarrageConfig instanceof DouyuRecordLoadBarrageConfig) { + return new DouyuRecordProcessor((DouyuRecordLoadBarrageConfig) loadBarrageConfig, dy_retryTimes, dy_retrySleepTime, dy_userAgent, dy_sleepTime); } // B站直播 - else if (loadConfig instanceof BilibiliLiveLoadConfig) { - return new BilibiliLiveProcessor((BilibiliLiveLoadConfig) loadConfig, bi_retryTimes, + else if (loadBarrageConfig instanceof BilibiliLiveLoadBarrageConfig) { + return new BilibiliLiveProcessor((BilibiliLiveLoadBarrageConfig) loadBarrageConfig, bi_retryTimes, bi_retrySleepTime, bi_userAgent, bi_sleepTime); } diff --git a/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java b/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java index 220f77d..73de346 100644 --- a/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java +++ b/CreeperModule/src/main/java/org/example/core/manager/LoadTaskManager.java @@ -3,7 +3,7 @@ package org.example.core.manager; import org.example.core.control.LoadTask; import org.example.core.factory.LoadTaskFactory; import org.example.exception.FileCacheException; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; import java.util.Collections; import java.util.Map; @@ -34,12 +34,12 @@ public class LoadTaskManager { } // 创建一个任务并返回唯一的 key - public String creatTask(LoadConfig loadConfig) throws FileCacheException { + public String creatTask(LoadBarrageConfig loadBarrageConfig) throws FileCacheException { // 根据主播名和时间戳生成唯一的 key - String key = generateKey(loadConfig); + String key = generateKey(loadBarrageConfig); // 创建任务并将其添加到任务映射中 - LoadTask loadTask = loadTaskFactory.getLoadTask(loadConfig); + LoadTask loadTask = loadTaskFactory.getLoadTask(loadBarrageConfig); taskMap.put(key, loadTask); // 返回生成的 key @@ -47,9 +47,9 @@ public class LoadTaskManager { } // 生成唯一的key - private String generateKey(LoadConfig loadConfig) { + private String generateKey(LoadBarrageConfig loadBarrageConfig) { // 以主播名和当前时间戳作为 key 的基础 - String baseKey = loadConfig.getAnchorName() + "_" + loadConfig.getStartTime(); + String baseKey = loadBarrageConfig.getAnchorName() + "_" + loadBarrageConfig.getStartTime(); // 确保 key 是唯一的，如果已存在，则添加一个数字后缀 String uniqueKey = baseKey; diff --git a/CreeperModule/src/main/java/org/example/core/pipeline/JsonPipline.java b/CreeperModule/src/main/java/org/example/core/pipeline/JsonPipline.java new file mode 100644 index 0000000..321a316 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/pipeline/JsonPipline.java @@ -0,0 +1,31 @@ +package org.example.core.pipeline; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.example.bean.ConfigFile; +import org.example.cache.FileCache; +import us.codecraft.webmagic.ResultItems; +import us.codecraft.webmagic.Task; +import us.codecraft.webmagic.pipeline.Pipeline; + +import java.util.ArrayList; + +/** + * @author Genius + * @date 2023/07/15 21:18 + **/ +public class JsonPipline implements Pipeline { + + private String handlerName; // 处理的对象名 + + public JsonPipline(String handlerName){ + this.handlerName = handlerName; + } + + @Override + public void process(ResultItems resultItems, Task task) { + Object o = resultItems.get(handlerName); + System.out.println(o); + } +} diff --git a/CreeperModule/src/main/java/org/example/core/pipeline/PipelineWriteJson.java b/CreeperModule/src/main/java/org/example/core/pipeline/PipelineWriteJson.java index eb5c490..127cb37 100644 --- a/CreeperModule/src/main/java/org/example/core/pipeline/PipelineWriteJson.java +++ b/CreeperModule/src/main/java/org/example/core/pipeline/PipelineWriteJson.java @@ -4,7 +4,7 @@ import org.example.cache.FileCache; import org.example.exception.FileCacheException; import org.example.pojo.Barrage; import org.example.pojo.configfile.BarrageSaveFile; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; import us.codecraft.webmagic.ResultItems; import us.codecraft.webmagic.Task; import us.codecraft.webmagic.pipeline.Pipeline; @@ -22,15 +22,15 @@ public class PipelineWriteJson implements Pipeline { private FileCache filecache; private final ConcurrentLinkedQueue cache; - LoadConfig loadConfig; + LoadBarrageConfig loadBarrageConfig; private BarrageSaveFile barrageSaveFile; - public PipelineWriteJson(LoadConfig loadConfig) { + public PipelineWriteJson(LoadBarrageConfig loadBarrageConfig) { try { - this.loadConfig = loadConfig; + this.loadBarrageConfig = loadBarrageConfig; this.cache = new ConcurrentLinkedQueue<>(); - this.barrageSaveFile = new BarrageSaveFile(loadConfig, cache); + this.barrageSaveFile = new BarrageSaveFile(loadBarrageConfig, cache); this.filecache = new FileCache(barrageSaveFile, 0, 10 * 1024); } catch (FileCacheException e) { throw new RuntimeException(e); diff --git a/CreeperModule/src/main/java/org/example/core/processor/BilibiliLiveProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/BilibiliLiveProcessor.java index d3d73ca..f669f49 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/BilibiliLiveProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/BilibiliLiveProcessor.java @@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.pojo.Barrage; -import org.example.pojo.download.assign.BilibiliLiveLoadConfig; +import org.example.pojo.download.assign.BilibiliLiveLoadBarrageConfig; import us.codecraft.webmagic.Page; import java.util.ArrayList; @@ -19,7 +19,7 @@ import java.util.Set; */ public class BilibiliLiveProcessor extends AbstractProcessor { - BilibiliLiveLoadConfig loadConfig; + BilibiliLiveLoadBarrageConfig loadConfig; // 前缀url private String urlPrefix = "https://api.live.bilibili.com/ajax/msg?roomid="; @@ -32,7 +32,7 @@ public class BilibiliLiveProcessor extends AbstractProcessor { // 开始时间戳 private Long startTime; - public BilibiliLiveProcessor(BilibiliLiveLoadConfig loadConfig, int retryTimes, int retrySleepTime, String userAgent, int sleepTime) { + public BilibiliLiveProcessor(BilibiliLiveLoadBarrageConfig loadConfig, int retryTimes, int retrySleepTime, String userAgent, int sleepTime) { super(retryTimes, retrySleepTime, userAgent, sleepTime); this.loadConfig = loadConfig; this.processedMids = new HashSet<>(); diff --git a/CreeperModule/src/main/java/org/example/core/processor/DouyuRecordProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/DouyuRecordProcessor.java index f192b8a..033491c 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/DouyuRecordProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/DouyuRecordProcessor.java @@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.pojo.Barrage; -import org.example.pojo.download.assign.DouyuRecordLoadConfig; +import org.example.pojo.download.assign.DouyuRecordLoadBarrageConfig; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.utils.HttpConstant; @@ -19,12 +19,12 @@ import java.util.List; */ public class DouyuRecordProcessor extends AbstractProcessor { - DouyuRecordLoadConfig loadConfig; + DouyuRecordLoadBarrageConfig loadConfig; // 前缀url private String urlPrefix = "https://v.douyu.com/wgapi/vod/center/getBarrageListByPage?vid="; - public DouyuRecordProcessor(DouyuRecordLoadConfig loadConfig, int retryTimes, int retrySleepTime, String userAgent, int sleepTime) { + public DouyuRecordProcessor(DouyuRecordLoadBarrageConfig loadConfig, int retryTimes, int retrySleepTime, String userAgent, int sleepTime) { super(retryTimes, retrySleepTime, userAgent, sleepTime); this.loadConfig = loadConfig; } diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/BilibiliHotModuleProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/BilibiliHotModuleProcessor.java new file mode 100644 index 0000000..c95e3da --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/BilibiliHotModuleProcessor.java @@ -0,0 +1,20 @@ +package org.example.core.processor.hotmodule; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.processor.PageProcessor; + +/** + * @author Genius + * @date 2023/06/01 22:10 + **/ +public class BilibiliHotModuleProcessor implements PageProcessor { + + private String url = "https://www.douyu.com/japi/weblist/apinc/header/cate"; + + + @Override + public void process(Page page) { + + } + +} diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java new file mode 100644 index 0000000..f122d36 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java @@ -0,0 +1,52 @@ +package org.example.core.processor.hotmodule; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.example.bean.HotLive; +import org.example.bean.hotmodule.DouyuHotLive; +import org.example.constpool.HotModulePool; +import org.example.log.HotModuleLogger; +import org.example.util.ChineseConvertUtil; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.processor.PageProcessor; +import us.codecraft.webmagic.selector.Json; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Genius + * @date 2023/07/19 00:30 + **/ +public class DouyuHotLiveProcessor implements PageProcessor { + + + @Override + public void process(Page page) { + + List hotLiveList = new ArrayList<>(); + try{ + JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("data"); + for (Object live : Lives) { + if(live instanceof JSONObject){ + JSONObject jsonLive = (JSONObject) live; + hotLiveList.add(new DouyuHotLive( + ChineseConvertUtil.cnNumericUnitsToInt(jsonLive.getString("hot")), + Integer.parseInt(jsonLive.getString("relId")), + jsonLive.getString("sk"), + jsonLive.containsKey("description")?jsonLive.getString("description"):"", + jsonLive.getInteger("hotTag"), + jsonLive.getInteger("isAd"), + jsonLive.getInteger("type"), + jsonLive.getString("url") + )); + } + } + }catch (Exception e){ + HotModuleLogger.logger.error("DouYu Hot Live List require fail! Exception:{}",e.getMessage()); + } + HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList); + + } +} diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java new file mode 100644 index 0000000..e9430d8 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java @@ -0,0 +1,44 @@ +package org.example.core.processor.hotmodule; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.example.bean.hotmodule.DouyuHotModule; +import org.example.bean.hotmodule.HotModuleList; +import org.example.constpool.HotModulePool; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.processor.PageProcessor; + +/** + * @author Genius + * @date 2023/06/01 22:11 + **/ +public class DouyuHotModuleProcessor implements PageProcessor { + + + @Override + public void process(Page page) { + HotModuleList douyuHotModuleList = new HotModuleList(); + try { + JSONObject data = JSON.parseObject(page.getRawText()).getJSONObject("data"); + JSONArray cateList = data.getJSONArray("cateList"); + JSONObject recommendAndHotObj = cateList.getJSONObject(1); + JSONArray recommendAndHostList = recommendAndHotObj.getJSONArray("list"); + + for (Object elem : recommendAndHostList) { + if(elem instanceof JSONObject){ + DouyuHotModule hotModule = new DouyuHotModule( + ((JSONObject) elem).getString("tagId"), + ((JSONObject) elem).getString("name"), + ((JSONObject) elem).getString("url") + ); + douyuHotModuleList.getHotModuleList().add(hotModule); + } + } + }catch (Exception e){ + + } + HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList); + } + +} diff --git a/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java b/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java index 012486b..fd60d8d 100644 --- a/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java +++ b/CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java @@ -1,7 +1,7 @@ package org.example.pojo.configfile; -import org.example.common.ConfigFile; -import org.example.pojo.download.LoadConfig; +import org.example.bean.ConfigFile; +import org.example.pojo.download.LoadBarrageConfig; import org.example.exception.FileCacheException; import org.example.pojo.Barrage; import org.example.util.FileUtil; @@ -25,11 +25,11 @@ import static org.example.constpool.ConstPool.BARRAGE_ROOT; */ public class BarrageSaveFile extends ConfigFile> { - private LoadConfig loadConfig; + private LoadBarrageConfig loadBarrageConfig; - public BarrageSaveFile(LoadConfig loadConfig, ConcurrentLinkedQueue data) throws FileCacheException { + public BarrageSaveFile(LoadBarrageConfig loadBarrageConfig, ConcurrentLinkedQueue data) throws FileCacheException { super(); - this.loadConfig = loadConfig; + this.loadBarrageConfig = loadBarrageConfig; if (!init(data)) { throw new FileCacheException("File init Error"); } @@ -44,7 +44,7 @@ public class BarrageSaveFile extends ConfigFile> private boolean init(ConcurrentLinkedQueue data) { String fileName = this.filaName(); setFileName(fileName); - String rootPath = Paths.get(BARRAGE_ROOT, loadConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 + String rootPath = Paths.get(BARRAGE_ROOT, loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 setFilePath(rootPath); Path path = Path.of(rootPath); //TODO 待移除 建立主播文件夹 @@ -67,7 +67,7 @@ public class BarrageSaveFile extends ConfigFile> private String filaName() { String format = "%s_%s_%s.json"; - return String.format(format, loadConfig.getPlatform(), loadConfig.getAnchorName(), loadConfig.getStartTime()); + return String.format(format, loadBarrageConfig.getPlatform(), loadBarrageConfig.getAnchorName(), loadBarrageConfig.getStartTime()); } } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java similarity index 83% rename from CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java rename to CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java index bb1c688..0bc2b74 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/LoadConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadBarrageConfig.java @@ -9,7 +9,7 @@ import org.example.utils.FormatUtil; * @date 2023/4/23 16:03 */ @Data -public class LoadConfig { +public class LoadBarrageConfig { // 爬取的平台 protected String platform; @@ -23,7 +23,7 @@ public class LoadConfig { // 开始时间 protected String startTime; - public LoadConfig(String platform, String action, String anchorName) { + public LoadBarrageConfig(String platform, String action, String anchorName) { this.platform = platform; this.action = action; this.anchorName = anchorName; diff --git a/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java new file mode 100644 index 0000000..5042732 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/LoadHotModuleConfig.java @@ -0,0 +1,8 @@ +package org.example.pojo.download; + +/** + * @author Genius + * @date 2023/06/01 22:18 + **/ +public class LoadHotModuleConfig { +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliHotModuleConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliHotModuleConfig.java new file mode 100644 index 0000000..3fdabb1 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliHotModuleConfig.java @@ -0,0 +1,8 @@ +package org.example.pojo.download.assign; + +/** + * @author Genius + * @date 2023/06/01 22:16 + **/ +public class BilibiliHotModuleConfig { +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java similarity index 63% rename from CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadConfig.java rename to CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java index 81812e0..f29c8d8 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java @@ -2,7 +2,7 @@ package org.example.pojo.download.assign; import lombok.Data; import org.example.constpool.ConstPool; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; /** * (B站直播)配置信息 @@ -10,12 +10,12 @@ import org.example.pojo.download.LoadConfig; * @date 2023/4/23 17:59 */ @Data -public class BilibiliLiveLoadConfig extends LoadConfig { +public class BilibiliLiveLoadBarrageConfig extends LoadBarrageConfig { // 房间号 private String roomId; - public BilibiliLiveLoadConfig(String anchorName, String roomId) { + public BilibiliLiveLoadBarrageConfig(String anchorName, String roomId) { super(ConstPool.BILIBILI, ConstPool.ACTION_LIVE, anchorName); this.roomId = roomId; } diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java new file mode 100644 index 0000000..c77b9a5 --- /dev/null +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuHotModuleConfig.java @@ -0,0 +1,12 @@ +package org.example.pojo.download.assign; + +import lombok.Data; + +/** + * @author Genius + * @date 2023/06/01 22:16 + **/ + +@Data +public class DouyuHotModuleConfig { +} diff --git a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadConfig.java b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java similarity index 63% rename from CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadConfig.java rename to CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java index e8227f4..fac591f 100644 --- a/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadConfig.java +++ b/CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java @@ -2,7 +2,7 @@ package org.example.pojo.download.assign; import lombok.Data; import org.example.constpool.ConstPool; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; /** * (斗鱼录播)配置信息 @@ -10,12 +10,12 @@ import org.example.pojo.download.LoadConfig; * @date 2023/4/23 16:30 */ @Data -public class DouyuRecordLoadConfig extends LoadConfig { +public class DouyuRecordLoadBarrageConfig extends LoadBarrageConfig { // 录播vid private String vid; - public DouyuRecordLoadConfig(String anchorName, String vid) { + public DouyuRecordLoadBarrageConfig(String anchorName, String vid) { super(ConstPool.DOUYU, ConstPool.ACTION_RECORD, anchorName); this.vid = vid; } diff --git a/CreeperModule/src/test/java/org/example/BarrageSaveFileAutoCreateTest.java b/CreeperModule/src/test/java/org/example/BarrageSaveFileAutoCreateTest.java index 87e5acf..d1cee2a 100644 --- a/CreeperModule/src/test/java/org/example/BarrageSaveFileAutoCreateTest.java +++ b/CreeperModule/src/test/java/org/example/BarrageSaveFileAutoCreateTest.java @@ -2,7 +2,7 @@ package org.example; import org.example.exception.FileCacheException; import org.example.pojo.configfile.BarrageSaveFile; -import org.example.pojo.download.LoadConfig; +import org.example.pojo.download.LoadBarrageConfig; import org.junit.jupiter.api.Test; import java.util.concurrent.ConcurrentLinkedQueue; @@ -15,7 +15,7 @@ public class BarrageSaveFileAutoCreateTest { @Test public void test() throws FileCacheException { - LoadConfig loadConfig = new LoadConfig("斗鱼","Post","大司马"); - BarrageSaveFile barrageSaveFile = new BarrageSaveFile(loadConfig,new ConcurrentLinkedQueue<>()); + LoadBarrageConfig loadBarrageConfig = new LoadBarrageConfig("斗鱼","Post","大司马"); + BarrageSaveFile barrageSaveFile = new BarrageSaveFile(loadBarrageConfig,new ConcurrentLinkedQueue<>()); } } diff --git a/FileModule/pom.xml b/FileModule/pom.xml index 725639a..7bfd12e 100644 --- a/FileModule/pom.xml +++ b/FileModule/pom.xml @@ -30,11 +30,6 @@ aliyun-sdk-oss 3.10.2 - - org.example - common - 1.0-SNAPSHOT - com.alibaba @@ -50,5 +45,11 @@ org.projectlombok lombok + + org.example + common + 1.0-SNAPSHOT + compile + diff --git a/FileModule/src/main/java/org/example/cache/FileCache.java b/FileModule/src/main/java/org/example/cache/FileCache.java index c98f00a..393fca7 100644 --- a/FileModule/src/main/java/org/example/cache/FileCache.java +++ b/FileModule/src/main/java/org/example/cache/FileCache.java @@ -3,8 +3,8 @@ package org.example.cache; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import org.example.common.ConfigFile; -import org.example.common.FileType; +import org.example.bean.ConfigFile; +import org.example.bean.FileType; import org.example.exception.FileCacheException; import org.example.util.JsonFileUtil; import org.example.util.TimeUtil; diff --git a/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java b/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java index eedc307..8fa6f1c 100644 --- a/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java +++ b/FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java @@ -5,7 +5,6 @@ import org.example.util.FileUtil; import org.example.util.JsonFileUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/FileModule/src/main/java/org/example/pojo/CommonConfigFile.java b/FileModule/src/main/java/org/example/pojo/CommonConfigFile.java index 54a633c..b78c932 100644 --- a/FileModule/src/main/java/org/example/pojo/CommonConfigFile.java +++ b/FileModule/src/main/java/org/example/pojo/CommonConfigFile.java @@ -1,6 +1,6 @@ package org.example.pojo; -import org.example.common.ConfigFile; +import org.example.bean.ConfigFile; /** * @author Genius diff --git a/FileModule/src/main/java/org/example/pojo/configfile/ModuleSrcConfigFile.java b/FileModule/src/main/java/org/example/pojo/configfile/ModuleSrcConfigFile.java index 5351976..af407ce 100644 --- a/FileModule/src/main/java/org/example/pojo/configfile/ModuleSrcConfigFile.java +++ b/FileModule/src/main/java/org/example/pojo/configfile/ModuleSrcConfigFile.java @@ -1,7 +1,7 @@ package org.example.pojo.configfile; -import org.example.common.ConfigFile; -import org.example.common.FileType; +import org.example.bean.ConfigFile; +import org.example.bean.FileType; import org.example.constpool.ConstPool; import java.util.Map; diff --git a/FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java b/FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java index d65053b..a0ff105 100644 --- a/FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java +++ b/FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java @@ -1,15 +1,12 @@ package org.example.service.impl; import org.example.cache.FileCache; -import org.example.cache.FileCacheManager; import org.example.cache.FileCacheManagerInstance; -import org.example.common.ConfigFile; -import org.example.common.FileType; +import org.example.bean.FileType; import org.example.pojo.vo.ConfigVO; import org.example.service.FileService; import org.springframework.stereotype.Service; -import java.io.File; import java.util.ArrayList; import java.util.List; diff --git a/FileModule/src/test/java/org/example/cache/FileCacheTest.java b/FileModule/src/test/java/org/example/cache/FileCacheTest.java index cded4bc..33f87a3 100644 --- a/FileModule/src/test/java/org/example/cache/FileCacheTest.java +++ b/FileModule/src/test/java/org/example/cache/FileCacheTest.java @@ -1,6 +1,5 @@ package org.example.cache; -import org.example.common.ConfigFile; import org.example.constpool.GlobalFileCache; import org.example.pojo.CommonConfigFile; import org.example.pojo.Student; @@ -8,7 +7,6 @@ import org.example.pojo.configfile.ModuleSrcConfigFile; import org.example.exception.FileCacheException; import org.junit.jupiter.api.Test; -import java.io.File; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; diff --git a/common/pom.xml b/common/pom.xml index f8bd4c7..c1dc023 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -34,6 +34,10 @@ spring-boot-starter-test test + + org.projectlombok + lombok + diff --git a/common/src/main/java/org/example/common/ConfigFile.java b/common/src/main/java/org/example/bean/ConfigFile.java similarity index 98% rename from common/src/main/java/org/example/common/ConfigFile.java rename to common/src/main/java/org/example/bean/ConfigFile.java index 72f6f1b..d1d2143 100644 --- a/common/src/main/java/org/example/common/ConfigFile.java +++ b/common/src/main/java/org/example/bean/ConfigFile.java @@ -1,4 +1,4 @@ -package org.example.common; +package org.example.bean; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; diff --git a/common/src/main/java/org/example/common/FileType.java b/common/src/main/java/org/example/bean/FileType.java similarity index 93% rename from common/src/main/java/org/example/common/FileType.java rename to common/src/main/java/org/example/bean/FileType.java index 8397fb0..f1c9453 100644 --- a/common/src/main/java/org/example/common/FileType.java +++ b/common/src/main/java/org/example/bean/FileType.java @@ -1,4 +1,4 @@ -package org.example.common; +package org.example.bean; /** * @author Genius diff --git a/common/src/main/java/org/example/bean/HotLive.java b/common/src/main/java/org/example/bean/HotLive.java new file mode 100644 index 0000000..8f15386 --- /dev/null +++ b/common/src/main/java/org/example/bean/HotLive.java @@ -0,0 +1,55 @@ +package org.example.bean; + +/** + * @author Genius + * @date 2023/07/18 22:24 + **/ + +/** + * 热门直播抽象类 + */ +public abstract class HotLive { + private int watcherNum; + private int liveId; + private String liveName; + private String description; + + public HotLive(int watcherNum, int liveId, String liveName, String description) { + this.watcherNum = watcherNum; + this.liveId = liveId; + this.liveName = liveName; + this.description = description; + } + + public int getWatcherNum() { + return watcherNum; + } + + public void setWatcherNum(int watcherNum) { + this.watcherNum = watcherNum; + } + + public int getLiveId() { + return liveId; + } + + public void setLiveId(int liveId) { + this.liveId = liveId; + } + + public String getLiveName() { + return liveName; + } + + public void setLiveName(String liveName) { + this.liveName = liveName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/common/src/main/java/org/example/bean/HotModule.java b/common/src/main/java/org/example/bean/HotModule.java new file mode 100644 index 0000000..5cfc1ca --- /dev/null +++ b/common/src/main/java/org/example/bean/HotModule.java @@ -0,0 +1,48 @@ +package org.example.bean; + +import lombok.Data; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/18 20:39 + **/ + + +public abstract class HotModule { + private String tagId; + private String tagName; + + private List hotLives; + + public String getTagId() { + return tagId; + } + + public String getTagName() { + return tagName; + } + + public void setTagId(String tagId) { + this.tagId = tagId; + } + + public void setTagName(String tagName) { + this.tagName = tagName; + } + + public List getHotLives() { + return hotLives; + } + + public void setHotLives(List hotLives) { + this.hotLives = hotLives; + } + + public HotModule(String tagId, String tagName){ + this.tagId = tagId; + this.tagName = tagName; + } + +} diff --git a/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java b/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java new file mode 100644 index 0000000..67784f5 --- /dev/null +++ b/common/src/main/java/org/example/bean/hotmodule/BilibiliHotModule.java @@ -0,0 +1,91 @@ +package org.example.bean.hotmodule; + +import lombok.Data; +import org.example.bean.HotModule; + +/** + * @author Genius + * @date 2023/07/18 20:49 + **/ + +public class BilibiliHotModule extends HotModule { + + private BilibiliParentHotModule parentHotModule; + + private String pinyin; + + private String act_id; + + private int hot_status; + + private int pk_status; + + private int lock_status; + + private String pic; + + private int area_type; + + public BilibiliHotModule(String tagId, String tagName, BilibiliParentHotModule parentHotModule, String pinyin, + String act_id, int hot_status, int pk_status, int lock_status, String pic, int area_type) { + super(tagId, tagName); + this.parentHotModule = parentHotModule; + this.pinyin = pinyin; + this.act_id = act_id; + this.hot_status = hot_status; + this.pk_status = pk_status; + this.lock_status = lock_status; + this.pic = pic; + this.area_type = area_type; + } + + public BilibiliParentHotModule getParentHotModule() { + return parentHotModule; + } + public String getPinyin() { + return pinyin; + } + public String getAct_id() { + return act_id; + } + public int getHot_status() { + return hot_status; + } + public int getPk_status() { + return pk_status; + } + public int getLock_status() { + return lock_status; + } + public String getPic() { + return pic; + } + public int getArea_type() { + return area_type; + } + + public void setParentHotModule(BilibiliParentHotModule parentHotModule) { + this.parentHotModule = parentHotModule; + } + public void setPinyin(String pinyin) { + this.pinyin = pinyin; + } + public void setAct_id(String act_id) { + this.act_id = act_id; + } + public void setHot_status(int hot_status) { + this.hot_status = hot_status; + } + public void setPk_status(int pk_status) { + this.pk_status = pk_status; + } + public void setLock_status(int lock_status) { + this.lock_status = lock_status; + } + public void setPic(String pic) { + this.pic = pic; + } + public void setArea_type(int area_type) { + this.area_type = area_type; + } +} diff --git a/common/src/main/java/org/example/bean/hotmodule/BilibiliParentHotModule.java b/common/src/main/java/org/example/bean/hotmodule/BilibiliParentHotModule.java new file mode 100644 index 0000000..2d26200 --- /dev/null +++ b/common/src/main/java/org/example/bean/hotmodule/BilibiliParentHotModule.java @@ -0,0 +1,15 @@ +package org.example.bean.hotmodule; + +import org.example.bean.HotModule; + +/** + * @author Genius + * @date 2023/07/18 20:50 + **/ +public class BilibiliParentHotModule extends HotModule { + + public BilibiliParentHotModule(String tagId, String tagName) { + super(tagId, tagName); + } + +} diff --git a/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java b/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java new file mode 100644 index 0000000..1e734bf --- /dev/null +++ b/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java @@ -0,0 +1,61 @@ +package org.example.bean.hotmodule; + +import org.example.bean.HotLive; + +/** + * @author Genius + * @date 2023/07/19 01:16 + **/ +public class DouyuHotLive extends HotLive { + + private int hotTag; + + private int isAd; + + + private int type; + + private String url; + + public DouyuHotLive(int watcherNum, int liveId, String liveName, String description, + int hotTag, int isAd, int type, String url) { + super(watcherNum, liveId, liveName, description); + this.hotTag = hotTag; + this.isAd = isAd; + this.type = type; + this.url = url; + } + + public int getHotTag() { + return hotTag; + } + + public void setHotTag(int hotTag) { + this.hotTag = hotTag; + } + + public int getIsAd() { + return isAd; + } + + public void setIsAd(int isAd) { + this.isAd = isAd; + } + + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java b/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java new file mode 100644 index 0000000..5274fd4 --- /dev/null +++ b/common/src/main/java/org/example/bean/hotmodule/DouyuHotModule.java @@ -0,0 +1,25 @@ +package org.example.bean.hotmodule; + +import lombok.Data; +import org.example.bean.HotModule; + +/** + * @author Genius + * @date 2023/07/18 20:46 + **/ +public class DouyuHotModule extends HotModule { + private String url; + + public DouyuHotModule(String tagId, String tagName, String url) { + super(tagId, tagName); + this.url = url; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java b/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java new file mode 100644 index 0000000..88c547f --- /dev/null +++ b/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java @@ -0,0 +1,22 @@ +package org.example.bean.hotmodule; + +import lombok.Data; +import org.example.bean.HotModule; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Genius + * @date 2023/07/18 21:09 + **/ + +@Data +public class HotModuleList { + private List hotModuleList; + + public HotModuleList(){ + hotModuleList = new ArrayList<>(); + } + +} diff --git a/common/src/main/java/org/example/constpool/HotModulePool.java b/common/src/main/java/org/example/constpool/HotModulePool.java new file mode 100644 index 0000000..272d564 --- /dev/null +++ b/common/src/main/java/org/example/constpool/HotModulePool.java @@ -0,0 +1,26 @@ +package org.example.constpool; + +import org.example.bean.HotLive; +import org.example.bean.hotmodule.HotModuleList; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author Genius + * @date 2023/07/18 22:16 + **/ +public class HotModulePool { + + static { + hotModuleListPool = new ConcurrentHashMap<>(); + hotLiveListPool = new ConcurrentHashMap<>(); + } + + public static final String DouYuAllHotModules = "DouyuAllHotModules"; + + public static final String DouYuAllHotLives = "DouyuAllHotLives"; + public static ConcurrentHashMap hotModuleListPool; + + public static ConcurrentHashMap> hotLiveListPool; +} diff --git a/common/src/main/java/org/example/log/HotModuleLogger.java b/common/src/main/java/org/example/log/HotModuleLogger.java new file mode 100644 index 0000000..0fd431d --- /dev/null +++ b/common/src/main/java/org/example/log/HotModuleLogger.java @@ -0,0 +1,14 @@ +package org.example.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + +/** + * @author Genius + * @date 2023/07/19 02:55 + **/ +public class HotModuleLogger { + public static final Logger logger = LoggerFactory.getLogger("HotModuleLogger"); +} diff --git a/common/src/main/java/org/example/util/ChineseConvertUtil.java b/common/src/main/java/org/example/util/ChineseConvertUtil.java new file mode 100644 index 0000000..b76503d --- /dev/null +++ b/common/src/main/java/org/example/util/ChineseConvertUtil.java @@ -0,0 +1,38 @@ +package org.example.util; + +/** + * @author Genius + * @date 2023/07/19 02:17 + **/ + +/** + * 中文转化工具 + */ +public class ChineseConvertUtil { + + /** + * 中文数量单位转Int + * @param str 中文字符串 如：“350.1万” + * @return + */ + public static int cnNumericUnitsToInt(String str){ + String[] parts = str.split("(?<=\\d\\.\\d)(?=\\D)"); + if(parts.length==2){ + float num = Float.parseFloat(parts[0]); + switch (parts[1]){ + case "亿": + return (int)num*100000; + case "万": + return (int)num*10000; + case "千": + return (int)num*1000; + case "百": + return (int)num*100; + case "个": + return (int)num; + } + } + return Integer.parseInt(str); + } + +} diff --git a/console/src/main/java/org/example/ConsoleApplication.java b/console/src/main/java/org/example/ConsoleApplication.java index 598a411..6178ad6 100644 --- a/console/src/main/java/org/example/ConsoleApplication.java +++ b/console/src/main/java/org/example/ConsoleApplication.java @@ -5,8 +5,6 @@ import org.example.exception.FileCacheException; import org.example.init.FileCacheManagerInit; import org.example.init.InitWorld; import org.example.init.ModuleSrcConfigFileInit; -import org.example.danmaku.pojo.configfile.BarrageSaveFile; -import org.example.danmaku.pojo.download.LoadConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;