lywsvip/ChopperBot
1
0
Fork 0
mirror of https://github.com/Geniusay/ChopperBot.git synced 2026-06-01 03:20:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

1，完成热门模块的数据中心

Browse Source 
2，完成api接口
This commit is contained in:
userA
2023-07-22 16:50:58 +08:00
parent a55490c184
commit e5e4cb41de
43 changed files with 895 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CreeperModule/src/main/java/org/example/constpool/ApiPool.java
View File

@@ -10,6 +10,10 @@ package org.example.constpool;
* 存放每一个爬虫api的池子
*/
public class ApiPool {
public final static String HOT_MODULE_LIST_API = "/japi/search/api/getHotList";
public static final String DOUYU_HOT_MODULE_API = "https://www.douyu.com/japi/weblist/apinc/header/cate";
public static final String DOUYU_HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api
public static final String DOUYU_HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
}

2
CreeperModule/src/main/java/org/example/constpool/ConstPool.java → CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java
View File

@@ -5,7 +5,7 @@ package org.example.constpool;
* @author 燧枫
* @date 2023/4/23 16:12
*/
public class ConstPool {
public class CreeperModuleConstPool {
/**
* 文件路径

44
CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java Normal file
View File

@@ -0,0 +1,44 @@
package org.example.core.control;
import org.example.log.HotModuleLogger;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/21 10:22
**/
public abstract class HotModuleLoadTask<T>{
public enum FinishFlag{
FINISH,NOT_FINISH,FAIL
}
private FinishFlag finishFlag = FinishFlag.NOT_FINISH;
public T start() {
clearFinishFlag();
return this.start0();
}
protected abstract T start0();
protected void fail(Exception e){
finishFlag = FinishFlag.FAIL;
HotModuleLogger.logger.error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage());
}
protected void success(){
finishFlag = FinishFlag.FINISH;
}
protected T getData(Spider spider,String url){
T data = ((ResultItems) spider.get(url)).get("data");
spider.close();
return data;
}
public FinishFlag isFinish(){
return finishFlag;
}
public void clearFinishFlag(){finishFlag = FinishFlag.NOT_FINISH;}
}

63
CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java Normal file
View File

@@ -0,0 +1,63 @@
package org.example.core.control.hotmodule;
import org.example.bean.HotLive;
import org.example.core.control.HotModuleLoadTask;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import java.util.List;
import static org.example.constpool.ApiPool.*;
/**
* @author Genius
* @date 2023/07/19 02:42
**/
public class DouyuHotLiveLoadTask extends HotModuleLoadTask<List<HotLive>> {
private final DouyuHotLiveProcessor douyuHotLiveProcessor;
public DouyuHotLiveLoadTask(){
douyuHotLiveProcessor = new DouyuHotLiveProcessor();
}
/**
* 获取Douyu某个模块下的热门直播
* @param moduleId
*/
public List<HotLive> start(int moduleId){
clearFinishFlag();
douyuHotLiveProcessor.setModuleId(moduleId);
return this.start(String.format(DOUYU_HOT_MODULE_LIVES_API,moduleId));
}
/**
* 获取Douyu当前最热直播
*/
@Override
protected List<HotLive> start0() {
return this.start(DOUYU_HOT_LIVES_API);
}
private List<HotLive> start(String url){
List<HotLive> lives;
try {
lives = getData(Spider.create(douyuHotLiveProcessor),url);
}catch (Exception e){
fail(e);
return null;
}
success();
return lives;
}
public static void main(String[] args) {
List<HotLive> start = new DouyuHotLiveLoadTask().start();
System.out.println(start);
}
}

32
CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java Normal file
View File

@@ -0,0 +1,32 @@
package org.example.core.control.hotmodule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.core.control.HotModuleLoadTask;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotModuleProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
import static org.example.constpool.ApiPool.DOUYU_HOT_MODULE_API;
/**
* @author Genius
* @date 2023/07/15 21:04
**/
public class DouyuHotModuleLoadTask extends HotModuleLoadTask<HotModuleList> {
@Override
protected HotModuleList start0() {
HotModuleList data;
try {
data = getData(Spider.create(new DouyuHotModuleProcessor()),DOUYU_HOT_MODULE_API);
}catch (Exception e){
fail(e);
return null;
}
success();
return data;
}
}

4
CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java
View File

@@ -1,6 +1,6 @@
package org.example.core.control.impl;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.core.control.LoadTask;
import org.example.core.factory.ProcessorFactory;
import org.example.core.pipeline.PipelineWriteJson;
@@ -34,7 +34,7 @@ public class BilibiliLiveLoadTask implements LoadTask {
public void start() {
Spider.create(bilibiliLiveProcessor)
// 设置起始Request
.addRequest(new Request(ConstPool.OCCUURL))
.addRequest(new Request(CreeperModuleConstPool.OCCUURL))
// 设置结果处理类
.addPipeline(pipelineWriteJson)
// 设置抓取线程数（可根据需要调整）

77
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java
View File

@@ -1,77 +0,0 @@
package org.example.core.control.impl;
import org.example.constpool.HotModulePool;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/19 02:42
**/
public class DouyuHotLiveLoadTask implements LoadTask {
private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api
private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
private final DouyuHotLiveProcessor douyuHotLiveProcessor;
public DouyuHotLiveLoadTask(){
douyuHotLiveProcessor = new DouyuHotLiveProcessor();
}
/**
* 获取Douyu某个模块下的热门直播
* @param moduleId
*/
public void start(int moduleId){
douyuHotLiveProcessor.setModuleId(moduleId);
this.start(String.format(HOT_MODULE_LIVES_API,moduleId));
}
/**
* 获取Douyu当前最热直播
*/
@Override
public void start() {
this.start(HOT_LIVES_API);
}
private void start(String url){
try {
Spider.create(douyuHotLiveProcessor)
.addRequest(new Request(url))
.thread(1)
.run();
}catch (Exception e){
e.printStackTrace();
}
}
@Override
public void end() {
}
@Override
public boolean isRunning() {
return false;
}
@Override
public int getCacheSize() {
return 0;
}
@Override
public int flushCacheAndSave(String key) {
return 0;
}
public static void main(String[] args) {
new DouyuHotModuleLoadTask().start();
new DouyuHotLiveLoadTask().start(1);
System.out.println(HotModulePool.hotModuleListPool);
}
}

48
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java
View File

@@ -1,48 +0,0 @@
package org.example.core.control.impl;
import org.example.constpool.HotModulePool;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotModuleProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/15 21:04
**/
public class DouyuHotModuleLoadTask implements LoadTask {
private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
@Override
public void start() {
DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor();
Spider.create(douyuHotModuleProcessor)
.addRequest(new Request(url))
.thread(1)
.run();
}
@Override
public void end() {
}
@Override
public boolean isRunning() {
return false;
}
@Override
public int getCacheSize() {
return 0;
}
@Override
public int flushCacheAndSave(String key) {
return 0;
}
public static void main(String[] args) {
new DouyuHotModuleLoadTask().start();
}
}

4
CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java
View File

@@ -1,6 +1,6 @@
package org.example.core.control.impl;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.core.control.LoadTask;
import org.example.core.factory.ProcessorFactory;
import org.example.core.pipeline.PipelineWriteJson;
@@ -34,7 +34,7 @@ public class DouyuRecordLoadTask implements LoadTask {
public void start() {
Spider.create(douyuRecordProcessor)
// 设置起始Request
.addRequest(new Request(ConstPool.OCCUURL))
.addRequest(new Request(CreeperModuleConstPool.OCCUURL))
// 设置结果处理类
.addPipeline(pipelineWriteJson)
// 设置抓取线程数（可根据需要调整）

28
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java
View File

@@ -4,20 +4,12 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.DouyuHotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import org.example.util.ChineseConvertUtil;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Json;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* @author Genius
@@ -60,25 +52,9 @@ public class DouyuHotLiveProcessor implements PageProcessor {
}
}
}catch (Exception e){
HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage());
throw e;
}
HotModuleLogger.logger.info(hotLiveList.toString());
updateHotLiveList(hotLiveList);
page.putField("data",hotLiveList);
}
private void updateHotLiveList(List<HotLive> hotLiveList){
if(moduleId!=-1){
if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){
HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules);
HotModule hotModule = hotModuleList.findHotModule(this.moduleId);
if(hotModule!=null){
hotModule.setHotLives(hotLiveList);
HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName());
}
}
}else{
HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated");
HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
}
}
}

7
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java
View File

@@ -5,12 +5,9 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.hotmodule.DouyuHotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.concurrent.ExecutorService;
/**
* @author Genius
@@ -38,9 +35,9 @@ public class DouyuHotModuleProcessor implements PageProcessor {
}
}
}catch (Exception e){
HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage());
throw e;
}
HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList);
page.putField("data",douyuHotModuleList);
}
}

2
CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java
View File

@@ -13,7 +13,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentLinkedQueue;
import static org.example.constpool.ConstPool.BARRAGE_ROOT;
import static org.example.constpool.CreeperModuleConstPool.BARRAGE_ROOT;
/**
* @author Genius

4
CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java
View File

@@ -1,7 +1,7 @@
package org.example.pojo.download.assign;
import lombok.Data;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.pojo.download.LoadBarrageConfig;
/**
@@ -16,7 +16,7 @@ public class BilibiliLiveLoadBarrageConfig extends LoadBarrageConfig {
private String roomId;
public BilibiliLiveLoadBarrageConfig(String anchorName, String roomId) {
super(ConstPool.BILIBILI, ConstPool.ACTION_LIVE, anchorName);
super(CreeperModuleConstPool.BILIBILI, CreeperModuleConstPool.ACTION_LIVE, anchorName);
this.roomId = roomId;
}
}

4
CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java
View File

@@ -1,7 +1,7 @@
package org.example.pojo.download.assign;
import lombok.Data;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.pojo.download.LoadBarrageConfig;
/**
@@ -16,7 +16,7 @@ public class DouyuRecordLoadBarrageConfig extends LoadBarrageConfig {
private String vid;
public DouyuRecordLoadBarrageConfig(String anchorName, String vid) {
super(ConstPool.DOUYU, ConstPool.ACTION_RECORD, anchorName);
super(CreeperModuleConstPool.DOUYU, CreeperModuleConstPool.ACTION_RECORD, anchorName);
this.vid = vid;
}
}

4
FileModule/src/main/java/org/example/cache/FileCache.java vendored
View File

@@ -213,7 +213,7 @@ public class FileCache <T extends ConfigFile>{
* @param key
* @return
*/
public Object get(String key){
private Object get(String key){
return jsonFile.get(key);
}
@@ -238,7 +238,7 @@ public class FileCache <T extends ConfigFile>{
*/
public void forceSync(){
if(writeByte.get()==0){
logger.info("未发生版本变化");
logger.debug("未发生版本变化");
return;
}
clearWriteBytes();

4
FileModule/src/main/java/org/example/cache/FileCacheManager.java vendored
View File

@@ -76,7 +76,7 @@ public class FileCacheManager {
fileCaches.add(fileCache);
fileCacheMap.put(fileCache.getFullFilePath(),fileCache);
initSleepTime();
FileModuleLogger.logger.info("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath());
FileModuleLogger.logger.debug("FileCacheManager add a new FileCache:{}",fileCache.getFullFilePath());
}
return false;
}
@@ -99,7 +99,7 @@ public class FileCacheManager {
BlockingQueue fileChannel = cache.getFileChannel();
if(fileChannel.isEmpty()){
if(cache.needAutoSync()){
logger.info("检测到需要强制刷新的文件 {}",cache.getFileName());
FileModuleLogger.logger.debug("检测到需要强制刷新的文件 {}",cache.getFileName());
autoSyncer.submit(new AutoSyncer(cache));
}
}

1
FileModule/src/main/java/org/example/util/JsonFileUtil.java
View File

@@ -71,7 +71,6 @@ public class JsonFileUtil {
T t = null;
Path dir = Paths.get(fullPath);
try{
System.out.println(dir);
if (FileUtil.isFileExist(dir.toString())) {
String res = Files.readString(dir, StandardCharsets.UTF_8);
// logger.debug("读取json文件成功, 文件内容为: {}", res);

29
HotModule/src/main/java/org/example/api/HotModuleApi.java Normal file
View File

@@ -0,0 +1,29 @@
package org.example.api;
import org.example.bean.HotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.core.control.hotmodule.DouyuHotLiveLoadTask;
import org.example.core.control.hotmodule.DouyuHotModuleLoadTask;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 17:53
**/
public class HotModuleApi {
private static DouyuHotModuleLoadTask douyuHotModuleLoadTask = new DouyuHotModuleLoadTask();
private static DouyuHotLiveLoadTask douyuHotLiveLoadTask = new DouyuHotLiveLoadTask();
public static HotModuleList getDouyuAllHotModule(){
return douyuHotModuleLoadTask.start();
}
public static List<HotLive> getDouyuHotLive(){
return douyuHotLiveLoadTask.start();
}
public static List<HotLive> getDouyuHotLive(int moduleId){
return douyuHotLiveLoadTask.start(moduleId);
}
}

24
HotModule/src/main/java/org/example/config/HotModuleConfig.java
View File

@@ -1,7 +1,7 @@
package org.example.config;
import org.example.bean.ConfigFile;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.constpool.HotModuleConstPool;
import java.nio.file.Paths;
@@ -15,22 +15,20 @@ import java.util.Map;
**/
public class HotModuleConfig extends ConfigFile<Map<String,Object>> {
private static final int FiveMinute = 0x493E0;
private static final long OneDay = 0x5265C00;
private static final String fileName = "hotModuleConfig.json";
public HotModuleConfig(){
super(HotModuleConstPool.HOT_MODULE_CONFIG_ROOT,fileName,
Map.of("Enable", 1,
"Module", List.of(
new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(), false,
List.of(allLiveDog()), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.BILIBILI, true, new ArrayList<>(), false,
List.of(allLiveDog()), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.HUYA, true, new ArrayList<>(), false,
List.of(allLiveDog()), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.DOUYING, true, new ArrayList<>(), false,
List.of(allLiveDog()), FiveMinute, FiveMinute)
Map.of("Module", List.of(
new HotModuleSetting(CreeperModuleConstPool.DOUYU,2,true, true,true, new ArrayList<>(), false,
List.of(allLiveDog()), OneDay, FiveMinute),
new HotModuleSetting(CreeperModuleConstPool.BILIBILI, 2,true,true,true, new ArrayList<>(), false,
List.of(allLiveDog()), OneDay, FiveMinute),
new HotModuleSetting(CreeperModuleConstPool.HUYA,2, true,true,true, new ArrayList<>(), false,
List.of(allLiveDog()), OneDay, FiveMinute),
new HotModuleSetting(CreeperModuleConstPool.DOUYING, 2,true,true,true, new ArrayList<>(), false,
List.of(allLiveDog()), OneDay, FiveMinute)
),
"GuardNum",10
)

8
HotModule/src/main/java/org/example/config/ModuleSetting.java → HotModule/src/main/java/org/example/config/HotModuleSetting.java
View File

@@ -11,10 +11,16 @@ import java.util.List;
**/
@Data
@AllArgsConstructor
public class ModuleSetting {
public class HotModuleSetting {
private String platform; //平台
private int failRetryTimes; //失败重试次数
private boolean enableHotModule; //是否开启热门模块爬取
private boolean enableHotLive; //是否开启热门直播爬取
private boolean autoWork; //是否自动进行主播直播下载任务推送
private List<String> focusLiver; //关注的主播

5
HotModule/src/main/java/org/example/constpool/HotModuleConstPool.java
View File

@@ -8,6 +8,9 @@ import org.example.config.HotModuleConfig;
* @date 2023/07/21 00:21
**/
public class HotModuleConstPool {
public static final String HOT_MODULE_CONFIG_ROOT = (String) GlobalFileCache.ModuleSrcConfigFile.get("hot","src");
public static final String HOT_MODULE_CONFIG_ROOT = (String) GlobalFileCache.ModuleSrcConfigFile.get("hot","src"); //热门模块配置文件路径
public static final String LOAD_TASK_CLASS_ROOT = "org.example.core.control.hotmodule"; //各个平台爬虫任务包路径
}

119
HotModule/src/main/java/org/example/core/HotModuleDataCenter.java Normal file
View File

@@ -0,0 +1,119 @@
package org.example.core;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.example.api.HotModuleApi;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.ConstPool;
import org.example.log.HotModuleLogger;
import org.example.thread.oddjob.OddJobBoy;
import org.example.util.TimeUtil;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author Genius
* @date 2023/07/21 18:44
**/
/**
* 热门模块的数据中心
*/
public class HotModuleDataCenter {
/**
* 设计这个参数的原因是，虽然HotModule可以存放HotLive数组，但是每天模块更新后会将整个hotModuleListPool替换，导致模块下的热门直播清空，
* 于是便用hotModuleLivePool来存放每个模块下的热门直播，为了保证访问效率和数据的新鲜度，这里采用 Lazy Delete的方式，
* 不用每次爬虫获取模块下的热门直播，而是查看是否过期，如果过期了才会去爬虫
*/
private static final long HOT_MODULE_LIVE_TTL = 60; //热门模块直播过期时间
private static volatile HotModuleDataCenter dataCenter;
public static HotModuleDataCenter DataCenter(){
if(dataCenter==null){
synchronized (HotModuleDataCenter.class){
if (dataCenter==null){
dataCenter = new HotModuleDataCenter();
}
}
}
return dataCenter;
}
private HotModuleDataCenter(){
hotModuleListPool = new ConcurrentHashMap<>();
hotModuleLivePool = new ConcurrentHashMap<>();
for (ConstPool.PLATFORM value : ConstPool.PLATFORM.values()) {
hotModuleLivePool.put(value.getName(),new ConcurrentHashMap<>());
}
hotLiveListPool = new ConcurrentHashMap<>();
}
private ConcurrentHashMap<String, HotModuleList> hotModuleListPool; //各个平台热门模块列表
public ConcurrentHashMap<String,ConcurrentHashMap<HotModule,ModuleLives>> hotModuleLivePool; //各个平台热门模块直播列表
private ConcurrentHashMap<String, List<HotLive>> hotLiveListPool; //各个平台热门直播列表
public void addModuleList(String platform,HotModuleList hotModuleList){
hotModuleListPool.put(platform,hotModuleList);
}
public void addLiveList(String platform,List<HotLive> hotLiveList){
hotLiveListPool.put(platform,hotLiveList);
}
public void addModuleLiveList(String platform,HotModule hotModule,List<HotLive> hotLiveList){
ModuleLives moduleLives = new ModuleLives(hotLiveList, LocalDateTime.now());
hotModule.setHotLives(hotLiveList);
hotModuleLivePool.get(platform).put(hotModule,moduleLives);
}
public HotModuleList getModuleList(String platform){
return hotModuleListPool.get(platform);
}
public List<HotLive> getLiveList(String platform){
return hotLiveListPool.get(platform);
}
public List<HotLive> getModuleLiveList(String platform,HotModule hotModule) throws InterruptedException {
ModuleLives moduleLives = hotModuleLivePool.get(platform).get(hotModule);
if(moduleLives!=null){
if (moduleLives.isExpire()) {
OddJobBoy.Boy().addWork(()->{
addModuleLiveList(platform,hotModule,
HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId())));
HotModuleLogger.logger.info("platform:{} module:{} hot lives refresh~",platform,hotModule.getTagName());
});
}
return moduleLives.hotLives;
}else{
//TODO多次访问改方法时会导致链接中断
List<HotLive> hotLives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId()));
addModuleLiveList(platform,hotModule,hotLives);
return hotLives;
}
}
@Data
@AllArgsConstructor
class ModuleLives{
private List<HotLive> hotLives;
private LocalDateTime updateTime;
public boolean isExpire(){
long now = TimeUtil.getCurrentSecond();
return now - TimeUtil.getSecond(updateTime)>HOT_MODULE_LIVE_TTL;
}
}
}

90
HotModule/src/main/java/org/example/guard/Guard.java Normal file
View File

@@ -0,0 +1,90 @@
package org.example.guard;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.example.bean.HotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.ConstPool;
import org.example.core.HotModuleDataCenter;
import org.example.core.control.HotModuleLoadTask;
import org.example.core.control.LoadTask;
import org.example.guard.HotModuleGuardInstance;
import org.example.log.HotModuleLogger;
import org.example.log.ResultLogger;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.*;
/**
* @author Genius
* @date 2023/07/21 02:13
**/
/**
* 热度监控守卫，负责执行各个平台的热度监控工作
*/
@Data
@AllArgsConstructor
public class Guard implements Runnable, ResultLogger {
private String guardName;
private HotModuleLoadTask task;
private long delayTime;
private int failRetryTimes;
@Override
public void run() {
int retryTimes = 0;
Object data = task.start();
if(task.isFinish() == HotModuleLoadTask.FinishFlag.FINISH){
successLog();
}else{
while(task.isFinish()== HotModuleLoadTask.FinishFlag.FAIL){
if(retryTimes<failRetryTimes){
data = task.start();
failLog(String.valueOf(retryTimes++));
}else{
failLog();
HotModuleGuardInstance.getInstance().unActiveGuard(this.guardName);
}
}
}
updateDataCenter(task.getClass().getName(),data);
}
private void updateDataCenter(String clazz, Object data){
String[] split = clazz.split("\\.");
String clazzName = split[split.length-1].toLowerCase();
String platform = clazzName.split("hot")[0];
if(clazzName.contains("module")){
HotModuleDataCenter.DataCenter().addModuleList(platform,(HotModuleList) data);
}else if(clazzName.contains("live")){
HotModuleDataCenter.DataCenter().addLiveList(platform,(List<HotLive>) data);
}
}
@Override
public void successLog() {
HotModuleLogger.logger.info("{} successfully finish!",guardName);
}
@Override
public void successLog(String str) {
}
@Override
public void failLog() {
HotModuleLogger.logger.error("{} finish error,cancel this task!",guardName);
}
@Override
public void failLog(String str) {
HotModuleLogger.logger.error("{} fail try to redo,retry times:{}!",guardName,str);
}
}

94
HotModule/src/main/java/org/example/guard/HotModuleGuard.java
View File

@@ -1,18 +1,18 @@
package org.example.guard;
import org.example.bean.HotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManager;
import org.example.cache.FileCacheManagerInstance;
import org.example.config.HotModuleConfig;
import org.example.constpool.HotModuleConstPool;
import org.example.core.control.impl.DouyuHotLiveLoadTask;
import org.example.core.control.impl.DouyuHotModuleLoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import org.example.core.control.HotModuleLoadTask;
import org.example.thread.NamedThreadFactory;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
/**
* @author Genius
@@ -20,24 +20,76 @@ import java.util.concurrent.TimeUnit;
**/
public class HotModuleGuard {
private static final long delayTime = 10*1000;
private List<Guard> guards; //热度监控守卫列表，用于初始化一开始的热度监控列表
private ScheduledExecutorService hotModuleGuardPool; //热度监控守卫 定时线程池
private Map<String,ScheduledFuture> runningGuards; // 运行的热度监控守卫
private FileCache HotModuleFileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath());
private ScheduledExecutorService hotModuleGuardPool = Executors.newScheduledThreadPool(
(Integer)FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()).get("GuardNum")
);
protected HotModuleGuard(List<Guard> guards,int guardNum){
this.guards = guards;
this.hotModuleGuardPool = Executors.newScheduledThreadPool(guardNum, new NamedThreadFactory("HotModuleGuard"));
runningGuards = new ConcurrentHashMap<>();
}
private void guardStart(Guard guard){
ScheduledFuture<?> scheduledFuture = hotModuleGuardPool.scheduleWithFixedDelay(
guard, 0, guard.getDelayTime(), TimeUnit.MILLISECONDS
);
runningGuards.put(guard.getGuardName(),scheduledFuture);
}
public void start(){
hotModuleGuardPool.scheduleWithFixedDelay(()-> {
new DouyuHotLiveLoadTask().start();
},0,delayTime, TimeUnit.MILLISECONDS);
hotModuleGuardPool.scheduleWithFixedDelay(()->{
new DouyuHotModuleLoadTask().start();
},0,delayTime,TimeUnit.MILLISECONDS);
if(runningGuards.size()==0){
try {
for (Guard guard : guards) {
guardStart(guard);
}
}catch (Exception e){
throw e;
}
}
}
public static void main(String[] args) {
new HotModuleGuard().start();
public void end(){
hotModuleGuardPool.shutdown();
runningGuards.clear();
}
public boolean addGuard(String platform,boolean isHotModule){
FileCache fileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath());
platform = platform.substring(0,1).toUpperCase() + platform.substring(1);
String clazzName = platform+"Hot"+(isHotModule?"Module":"Live")+"LoadTask";
String clazz = HotModuleConstPool.LOAD_TASK_CLASS_ROOT+"."+clazzName;
String timeName = isHotModule?"updateHotModuleTimes":"updateHotLivesTimes";
try {
addGuard(new Guard(
clazzName.toLowerCase(),
(HotModuleLoadTask)Class.forName(clazz).getDeclaredConstructor().newInstance(),
(long)fileCache.get(timeName),
(int)fileCache.get("failRetryTimes")
));
}catch (Exception e){
return false;
}
return true;
}
public boolean addGuard(Guard guard){
if(!runningGuards.containsKey(guard.getGuardName())){
guardStart(guard);
return true;
}
return false;
}
public boolean unActiveGuard(String guardName){
if (runningGuards.containsKey(guardName)) {
runningGuards.get(guardName).cancel(false);
runningGuards.remove(guardName);
return true;
}
return false;
}
}

34
HotModule/src/main/java/org/example/guard/HotModuleGuardInstance.java Normal file
View File

@@ -0,0 +1,34 @@
package org.example.guard;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 11:39
**/
public class HotModuleGuardInstance {
private static List<Guard> guardList = new ArrayList<>();
private static int guardNum;
private static volatile HotModuleGuard Instance;
public static HotModuleGuard getInstance(){
if(Instance==null){
synchronized (HotModuleGuardInstance.class){
if(Instance==null){
Instance = new HotModuleGuard(guardList,guardNum);
}
}
}
return Instance;
}
public static void InitInstance(List<Guard> guards,int num){
guardList = guards;
guardNum = num;
}
}

8
HotModule/src/main/java/org/example/guard/task/Guard.java
View File

@@ -1,8 +0,0 @@
package org.example.guard.task;
/**
* @author Genius
* @date 2023/07/21 02:13
**/
public class Guard {
}

80
HotModule/src/main/java/org/example/init/HotModuleGuardInitMachine.java Normal file
View File

@@ -0,0 +1,80 @@
package org.example.init;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.config.HotModuleConfig;
import org.example.config.HotModuleSetting;
import org.example.core.control.HotModuleLoadTask;
import org.example.guard.HotModuleGuardInstance;
import org.example.guard.Guard;
import org.example.log.HotModuleLogger;
import org.example.util.ClassUtil;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.example.constpool.HotModuleConstPool.LOAD_TASK_CLASS_ROOT;
/**
* @author Genius
* @date 2023/07/21 09:58
**/
public class HotModuleGuardInitMachine extends CommonInitMachine{
public HotModuleGuardInitMachine() {
super(HotModuleLogger.logger);
}
private void envInit() throws Exception {
FileCache HotModuleFileCache = FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath());
List<Guard> guards = new ArrayList<>();
int guardNum = (Integer)HotModuleFileCache.get("GuardNum");
Map<String,HotModuleSetting> map = new HashMap<>();
JSONArray modules = (JSONArray)HotModuleFileCache.get("Module");
for (Object module : modules) {
HotModuleSetting hotModuleSetting = JSONObject.parseObject(module.toString(), HotModuleSetting.class);
map.put(hotModuleSetting.getPlatform(),hotModuleSetting);
}
for (String clazz : ClassUtil.getClassesInPackage(LOAD_TASK_CLASS_ROOT)) {
String[] split = clazz.split("\\.");
String clazzName = split[split.length-1].toLowerCase();
if(clazzName.endsWith("loadtask")&& clazzName.contains("hot")){
String platformName = clazzName.split("hot")[0];
boolean isHotModule = clazzName.contains("module");
if(map.containsKey(platformName)){
HotModuleSetting hotModuleSetting = map.get(platformName);
Class<?> loadClazz = Class.forName(clazz);
if(isHotModule&&hotModuleSetting.isEnableHotModule()){
HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance();
guards.add(new Guard(clazzName,task,
hotModuleSetting.getUpdateHotModuleTimes(),hotModuleSetting.getFailRetryTimes()));
}else if(hotModuleSetting.isEnableHotLive()){
HotModuleLoadTask task = (HotModuleLoadTask)loadClazz.getDeclaredConstructor().newInstance();
guards.add(new Guard(clazzName,task,
hotModuleSetting.getUpdateHotLivesTimes(),hotModuleSetting.getFailRetryTimes()));
}
}
}
}
HotModuleGuardInstance.InitInstance(guards,guardNum);
HotModuleGuardInstance.getInstance().start();
}
@Override
public boolean init() {
try {
envInit();
} catch (Exception e) {
return fail(e.getMessage());
}
return success();
}
}

2
HotModule/src/main/java/org/example/init/HotModuleInitMachine.java
View File

@@ -18,7 +18,7 @@ public class HotModuleInitMachine extends ModuleInitMachine{
public HotModuleInitMachine() {
super(
List.of(new HotModuleConfigInitMachine()),
List.of(new HotModuleConfigInitMachine(),new HotModuleGuardInitMachine()),
"HotModule",
HotModuleLogger.logger
);

18
common/src/main/java/org/example/constpool/ConstPool.java
View File

@@ -22,6 +22,24 @@ public class ConstPool {
public static final List<String> PIC_TYPES = List.of("jpg","jpeg","png","svg");
/**直播平台**/
public enum PLATFORM{
DOUYU("douyu"),
HUYA("huya"),
BILIBILI("bilibili"),
DOUYING("douyin"),
TIKTOK("tiktok"),
TWITCH("twitch");
private final String name;
PLATFORM(String name){
this.name = name;
}
public String getName(){
return name;
}
}
public static final String DOUYU = "douyu";
public static final String HUYA = "huya";

26
common/src/main/java/org/example/constpool/HotModulePool.java
View File

@@ -1,26 +0,0 @@
package org.example.constpool;
import org.example.bean.HotLive;
import org.example.bean.hotmodule.HotModuleList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author Genius
* @date 2023/07/18 22:16
**/
public class HotModulePool {
static {
hotModuleListPool = new ConcurrentHashMap<>();
hotLiveListPool = new ConcurrentHashMap<>();
}
public static final String DouYuAllHotModules = "DouyuAllHotModules";
public static final String DouYuAllHotLives = "DouyuAllHotLives";
public static ConcurrentHashMap<String, HotModuleList> hotModuleListPool;
public static ConcurrentHashMap<String, List<HotLive>> hotLiveListPool;
}

2
common/src/main/java/org/example/init/CommonInitMachine.java
View File

@@ -46,7 +46,7 @@ public abstract class CommonInitMachine implements ComponentInitMachine{
@Override
public boolean fail(String failCause) {
failLog(String.format("[❌] {%s} init error! Execption:{}",this.getClass().toString(),failCause));
failLog(String.format("[❌] {%s} init error! Execption:{%s}",this.getClass().toString(),failCause));
return false;
}

4
common/src/main/java/org/example/init/InitMachine.java
View File

@@ -1,7 +1,7 @@
package org.example.init;
import org.example.log.InitMachineLogger;
import org.example.log.ResultLogger;
/**
* @author Genius
@@ -9,7 +9,7 @@ import org.example.log.InitMachineLogger;
**/
//模块初始化接口
public interface InitMachine extends InitMachineLogger {
public interface InitMachine extends ResultLogger {
boolean init();
}

2
common/src/main/java/org/example/log/InitMachineLogger.java → common/src/main/java/org/example/log/ResultLogger.java
View File

@@ -4,7 +4,7 @@ package org.example.log;
/**
* 启动日志接口
*/
public interface InitMachineLogger {
public interface ResultLogger {
void successLog();

28
common/src/main/java/org/example/thread/NamedThreadFactory.java Normal file
View File

@@ -0,0 +1,28 @@
package org.example.thread;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author Genius
* @date 2023/07/21 16:58
**/
/**
* 更改线程池名字
*/
public class NamedThreadFactory implements ThreadFactory {
private final String poolName;
private final AtomicInteger threadNumber = new AtomicInteger(1);
public NamedThreadFactory(String poolName) {
this.poolName = poolName;
}
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName("ChopperBot-"+poolName + "-" + threadNumber.getAndIncrement());
return t;
}
}

9
common/src/main/java/org/example/thread/oddjob/OddJob.java Normal file
View File

@@ -0,0 +1,9 @@
package org.example.thread.oddjob;
/**
* @author Genius
* @date 2023/07/21 19:22
**/
public interface OddJob{
void doJob();
}

68
common/src/main/java/org/example/thread/oddjob/OddJobBoy.java Normal file
View File

@@ -0,0 +1,68 @@
package org.example.thread.oddjob;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* @author Genius
* @date 2023/07/21 19:18
**/
/**
* ChopperBot系统中专门用来处理异步事件的类
*/
public class OddJobBoy {
private static volatile OddJobBoy Instance;
private BlockingQueue<OddJob> oddjobs;
private ExecutorService home;
private OddJobBoy(){
home = Executors.newSingleThreadExecutor();
oddjobs = new ArrayBlockingQueue<>(1024);
}
public static OddJobBoy Boy(){
if(Instance==null){
synchronized (OddJobBoy.class){
if(Instance==null){
Instance = new OddJobBoy();
Instance.work();
}
}
}
return Instance;
}
public void addWork(OddJob job) throws InterruptedException {
oddjobs.put(job);
}
private void work(){
home.submit(new Boy());
}
public boolean relax(){
home.shutdown();
return home.isShutdown();
}
class Boy implements Runnable{
@Override
public void run() {
while(true){
try {
OddJob job = oddjobs.take();
job.doJob();
}catch (InterruptedException e){
}
}
}
}
}

41
common/src/main/java/org/example/util/ClassUtil.java Normal file
View File

@@ -0,0 +1,41 @@
package org.example.util;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 09:56
**/
public class ClassUtil {
public static List<String> getClassesInPackage(String packageName)throws IOException {
List<String> classNames = new ArrayList<>();
String packagePath = packageName.replace(".", "/");
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
var resources = classLoader.getResources(packagePath);
while (resources.hasMoreElements()) {
var resource = resources.nextElement();
File file = new File(resource.getFile());
if (file.isDirectory()) {
scanClassesInDirectory(packageName, file, classNames);
}
}
return classNames;
}
private static void scanClassesInDirectory(String packageName, File directory, List<String> classNames) {
for (File file : directory.listFiles()) {
if (file.isFile()) {
String className = packageName + "." + file.getName().replace(".class", "");
classNames.add(className);
} else if (file.isDirectory()) {
scanClassesInDirectory(packageName + "." + file.getName(), file, classNames);
}
}
}
}

57
console/src/main/java/org/example/controller/HotController.java Normal file
View File

@@ -0,0 +1,57 @@
package org.example.controller;
import com.genius.assistant.common.Result;
import org.example.api.HotModuleApi;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.ConstPool;
import org.example.core.HotModuleDataCenter;
import org.example.service.HotModuleService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 17:13
**/
@RestController
@RequestMapping("/hot")
public class HotController {
@Autowired
HotModuleService hotModuleService;
@GetMapping("/douyu/allHotLive")
public Result getDouyuAllHotLive(@RequestParam(defaultValue = "0") int latest){
List<HotLive> hotLives;
if(latest==1){
hotLives = HotModuleApi.getDouyuHotLive();
}else{
hotLives = HotModuleDataCenter.DataCenter().getLiveList(ConstPool.PLATFORM.DOUYU.getName());
}
return Result.success(hotLives);
}
@GetMapping("/douyu/allHotModule")
public Result getDouyuAllHotModule(@RequestParam(defaultValue = "0") int latest){
HotModuleList hotModuleList;
if(latest==1){
hotModuleList = HotModuleApi.getDouyuAllHotModule();
}else{
hotModuleList = HotModuleDataCenter.DataCenter().getModuleList(ConstPool.PLATFORM.DOUYU.getName());
}
return Result.success(hotModuleList.getHotModuleList());
}
@GetMapping("/douyu/getHotModuleLives")
public Result getDouyuHotModuleLives(@RequestParam int moduleId){
HotModule moduleHotLives = hotModuleService.getModuleHotLives(ConstPool.PLATFORM.DOUYU.getName(), moduleId);
return Result.success(moduleHotLives);
}
}

0
FileModule/src/main/java/org/example/service/FileService.java → console/src/main/java/org/example/service/FileService.java
View File

14
console/src/main/java/org/example/service/HotModuleService.java Normal file
View File

@@ -0,0 +1,14 @@
package org.example.service;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.springframework.stereotype.Service;
import java.util.List;
public interface HotModuleService {
HotModule getModuleHotLives(String platform, int moduleId);
}

0
FileModule/src/main/java/org/example/service/impl/FileServiceImpl.java → console/src/main/java/org/example/service/impl/FileServiceImpl.java
View File

45
console/src/main/java/org/example/service/impl/HotModuleServiceImpl.java Normal file
View File

@@ -0,0 +1,45 @@
package org.example.service.impl;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModuleConstPool;
import org.example.core.HotModuleDataCenter;
import org.example.service.HotModuleService;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 17:23
**/
@Service
public class HotModuleServiceImpl implements HotModuleService {
/**
* 获得热门模块下的热门直播
* @param moduleId
* @return
*/
@Override
public HotModule getModuleHotLives(String platform,int moduleId) {
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
if(moduleList==null){
return null;
}
HotModule hotModule = moduleList.findHotModule(moduleId);
if(hotModule!=null){
try {
List<HotLive> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
hotModule.setHotLives(moduleLiveList);
return hotModule;
}catch (Exception e){
//TODO 交给Spring全局异常处理器
return null;
}
}
return null;
}
}

1
console/src/main/resources/application.yaml
View File

@@ -1,5 +1,6 @@
server:
port: 8888
spring:
datasource:
url: jdbc:mysql://localhost:3306/illuminator?useSSL=false&serverTimezone=UTC