lywsvip/ChopperBot
1
0
Fork 0
mirror of https://github.com/Geniusay/ChopperBot.git synced 2026-06-06 06:39:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

1，完善斗鱼的热度模块获取以及热度直播获取

Browse Source 
2，更新HotLive模块
3，完善HotModule配置类
4，更新FileCacheManager
This commit is contained in:
userA
2023-07-20 04:41:42 +08:00
parent 6d8f80b291
commit 98e912682f
14 changed files with 323 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java
View File

@@ -12,20 +12,41 @@ import us.codecraft.webmagic.Spider;
**/
public class DouyuHotLiveLoadTask implements LoadTask {
private String url = "https://www.douyu.com/japi/search/api/getHotList";
private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api
private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
private final DouyuHotLiveProcessor douyuHotLiveProcessor;
public DouyuHotLiveLoadTask(){
douyuHotLiveProcessor = new DouyuHotLiveProcessor();
}
/**
* 获取Douyu某个模块下的热门直播
* @param moduleId
*/
public void start(int moduleId){
douyuHotLiveProcessor.setModuleId(moduleId);
this.start(String.format(HOT_MODULE_LIVES_API,moduleId));
}
/**
* 获取Douyu当前最热直播
*/
@Override
public void start() {
this.start(HOT_LIVES_API);
}
private void start(String url){
try {
DouyuHotLiveProcessor douyuHotLiveProcessor = new DouyuHotLiveProcessor();
Spider.create(douyuHotLiveProcessor)
.addRequest(new Request(url))
.setEmptySleepTime(10)
.thread(1)
.run();
}catch (Exception e){
e.printStackTrace();
}
}
@Override
@@ -49,7 +70,8 @@ public class DouyuHotLiveLoadTask implements LoadTask {
}
public static void main(String[] args) {
new DouyuHotLiveLoadTask().start();
System.out.println(HotModulePool.hotLiveListPool);
new DouyuHotModuleLoadTask().start();
new DouyuHotLiveLoadTask().start(1);
System.out.println(HotModulePool.hotModuleListPool);
}
}

3
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java
View File

@@ -12,13 +12,12 @@ import us.codecraft.webmagic.Spider;
**/
public class DouyuHotModuleLoadTask implements LoadTask {
private String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
@Override
public void start() {
DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor();
Spider.create(douyuHotModuleProcessor)
.addRequest(new Request(url))
.setEmptySleepTime(10)
.thread(1)
.run();
}

52
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java
View File

@@ -4,49 +4,81 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.DouyuHotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import org.example.util.ChineseConvertUtil;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Json;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* @author Genius
* @date 2023/07/19 00:30
**/
public class DouyuHotLiveProcessor implements PageProcessor {
private int moduleId; // 模块Id
public DouyuHotLiveProcessor(){
this.moduleId = -1;
}
public void setModuleId(int moduleId) {
this.moduleId = moduleId;
}
@Override
public void process(Page page) {
List<HotLive> hotLiveList = new ArrayList<>();
try{
JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("data");
JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONObject("data").getJSONArray("rl");
for (Object live : Lives) {
if(live instanceof JSONObject){
JSONObject jsonLive = (JSONObject) live;
hotLiveList.add(new DouyuHotLive(
ChineseConvertUtil.cnNumericUnitsToInt(jsonLive.getString("hot")),
Integer.parseInt(jsonLive.getString("relId")),
jsonLive.getString("sk"),
jsonLive.containsKey("description")?jsonLive.getString("description"):"",
jsonLive.getInteger("hotTag"),
jsonLive.getInteger("isAd"),
jsonLive.getInteger("ol"),
jsonLive.getInteger("rid"),
jsonLive.getString("rn"),
jsonLive.getString("nn"),
jsonLive.getString("od"),
jsonLive.getString("c2name"),
jsonLive.getString("url"),
jsonLive.getString("rs16"),
jsonLive.getInteger("type"),
jsonLive.getString("url")
jsonLive.getInteger("uid"),
jsonLive.getInteger("cid2")
));
}
}
}catch (Exception e){
HotModuleLogger.logger.error("DouYu Hot Live List require fail! Exception:{}",e.getMessage());
HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage());
}
HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
HotModuleLogger.logger.info(hotLiveList.toString());
updateHotLiveList(hotLiveList);
}
private void updateHotLiveList(List<HotLive> hotLiveList){
if(moduleId!=-1){
if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){
HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules);
HotModule hotModule = hotModuleList.findHotModule(this.moduleId);
if(hotModule!=null){
hotModule.setHotLives(hotLiveList);
HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName());
}
}
}else{
HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated");
HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
}
}
}

6
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java
View File

@@ -6,16 +6,18 @@ import com.alibaba.fastjson.JSONObject;
import org.example.bean.hotmodule.DouyuHotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.concurrent.ExecutorService;
/**
* @author Genius
* @date 2023/06/01 22:11
**/
public class DouyuHotModuleProcessor implements PageProcessor {
@Override
public void process(Page page) {
HotModuleList douyuHotModuleList = new HotModuleList();
@@ -36,7 +38,7 @@ public class DouyuHotModuleProcessor implements PageProcessor {
}
}
}catch (Exception e){
HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage());
}
HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList);
}

16
FileModule/src/main/java/org/example/cache/FileCacheManager.java vendored
View File

@@ -6,10 +6,7 @@ import org.slf4j.LoggerFactory;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -28,6 +25,8 @@ public class FileCacheManager {
private Logger logger = LoggerFactory.getLogger(FileCacheManager.class);
private final List<FileCache> fileCaches;
private final ConcurrentHashMap<String,FileCache> fileCacheMap;
private AtomicLong sleepTime; //睡眠时间
private ExecutorService watchPool; //巡逻线程
@@ -38,6 +37,10 @@ public class FileCacheManager {
protected FileCacheManager(List<FileCache> fileCaches){
this.fileCaches = new CopyOnWriteArrayList<>(fileCaches);
fileCacheMap = new ConcurrentHashMap<>();
for (FileCache fileCache : fileCaches) {
fileCacheMap.put(fileCache.getFullFilePath(),fileCache);
}
initSleepTime();
this.watchPool = Executors.newSingleThreadExecutor();
this.autoSyncer = Executors.newFixedThreadPool(fileCaches.size());
@@ -70,11 +73,16 @@ public class FileCacheManager {
public boolean addFileCache(FileCache fileCache){
if (this.fileCaches.indexOf(fileCache)==-1) {
fileCaches.add(fileCache);
fileCacheMap.put(fileCache.getFullFilePath(),fileCache);
initSleepTime();
}
return false;
}
public FileCache getFileCache(String filePath){
return fileCacheMap.get(filePath);
}
public List<FileCache> getRunnableFileCaches(){
return this.fileCaches;
}

5
HotModule/pom.xml
View File

@@ -18,6 +18,11 @@
</properties>
<dependencies>
<dependency>
<groupId>org.example</groupId>
<artifactId>CreeperModule</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

22
HotModule/src/main/java/org/example/config/FollowDog.java Normal file
View File

@@ -0,0 +1,22 @@
package org.example.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
/**
* @author Genius
* @date 2023/07/20 00:40
**/
//config中的跟风狗设置
@Data
@AllArgsConstructor
public class FollowDog {
public final static String ALL_LIVES = "all";
private String moduleName; //模块名称，all代表热门直播
private int top; //前几个
private List<String> banLiver; //去除的主播，可以是正则
}

44
HotModule/src/main/java/org/example/config/HotModuleConfig.java Normal file
View File

@@ -0,0 +1,44 @@
package org.example.config;
import org.example.bean.ConfigFile;
import org.example.constpool.ConstPool;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author Genius
* @date 2023/07/20 00:20
**/
public class HotModuleConfig extends ConfigFile<Map<String,Object>> {
private static final FollowDog allLivesDog = new FollowDog(FollowDog.ALL_LIVES,6,new ArrayList<>());
private static final int FiveMinute = 0x493E0;
private static final String filePath = "./config";
private static final String fileName = "hotmodule.config";
public HotModuleConfig(){
super(filePath,fileName,
Map.of("Enable", 1,
"Module", List.of(
new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(),
false, List.of(allLivesDog), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.BILIBILI, true, new ArrayList<>(),
false, List.of(allLivesDog), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.HUYA, true, new ArrayList<>(),
false, List.of(allLivesDog), FiveMinute, FiveMinute),
new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(),
false, List.of(allLivesDog), FiveMinute, FiveMinute)
),
"GuardNum",10
)
);
}
public static String getFullFilePath(){
return Paths.get(filePath,fileName).toString();
}
}

30
HotModule/src/main/java/org/example/config/ModuleSetting.java Normal file
View File

@@ -0,0 +1,30 @@
package org.example.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
/**
* @author Genius
* @date 2023/07/20 00:25
**/
@Data
@AllArgsConstructor
public class ModuleSetting {
private String platform; //平台
private boolean autoWork; //是否自动进行主播直播下载任务推送
private List<String> focusLiver; //关注的主播
private boolean followDogEnable; //当开启autoWork时，启用跟风狗模式，跟风狗模式会自动爬取热门直播
private List<FollowDog> followDogs;
private long updateHotModuleTimes; //自动更新平台热门模块时间
private long updateHotLivesTimes; //自动更新平台热门直播时间
}

39
HotModule/src/main/java/org/example/guard/HotModuleGuard.java Normal file
View File

@@ -0,0 +1,39 @@
package org.example.guard;
import org.example.cache.FileCacheManager;
import org.example.cache.FileCacheManagerInstance;
import org.example.config.HotModuleConfig;
import org.example.core.control.impl.DouyuHotLiveLoadTask;
import org.example.core.control.impl.DouyuHotModuleLoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* @author Genius
* @date 2023/07/19 03:21
**/
public class HotModuleGuard {
private static final long delayTime = 10*1000;
private ScheduledExecutorService hotModuleGuardPool = Executors.newScheduledThreadPool(
(Integer)FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()).get("GuardNum")
);
public void start(){
hotModuleGuardPool.scheduleWithFixedDelay(()-> {
new DouyuHotLiveLoadTask().start();
},0,delayTime, TimeUnit.MILLISECONDS);
hotModuleGuardPool.scheduleWithFixedDelay(()->{
new DouyuHotModuleLoadTask().start();
},0,delayTime,TimeUnit.MILLISECONDS);
}
public static void main(String[] args) {
new HotModuleGuard().start();
}
}

26
common/src/main/java/org/example/bean/HotLive.java
View File

@@ -9,10 +9,12 @@ package org.example.bean;
* 热门直播抽象类
*/
public abstract class HotLive {
private int watcherNum;
private int liveId;
private String liveName;
private String description;
private int watcherNum; //直播间观众数目
private int liveId; //直播间ID
private String liveName; //直播间名字
private String liver; //主播
private String description; //直播间简介
public HotLive(int watcherNum, int liveId, String liveName, String description) {
this.watcherNum = watcherNum;
@@ -21,6 +23,14 @@ public abstract class HotLive {
this.description = description;
}
public HotLive(int watcherNum, int liveId, String liveName, String liver, String description) {
this.watcherNum = watcherNum;
this.liveId = liveId;
this.liveName = liveName;
this.liver = liver;
this.description = description;
}
public int getWatcherNum() {
return watcherNum;
}
@@ -45,6 +55,14 @@ public abstract class HotLive {
this.liveName = liveName;
}
public String getLiver() {
return liver;
}
public void setLiver(String liver) {
this.liver = liver;
}
public String getDescription() {
return description;
}

70
common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java
View File

@@ -8,38 +8,60 @@ import org.example.bean.HotLive;
**/
public class DouyuHotLive extends HotLive {
private int hotTag;
private String liveModule; //直播板块
private int isAd;
private String url; //直播间地址
private String roomCoverPic; //直播间封面地址
private int type;
private String url;
private int uid;
public DouyuHotLive(int watcherNum, int liveId, String liveName, String description,
int hotTag, int isAd, int type, String url) {
super(watcherNum, liveId, liveName, description);
this.hotTag = hotTag;
this.isAd = isAd;
private int moduleId; //直播板块Id
public DouyuHotLive(int watcherNum, int liveId, String liveName, String liver, String description,
String liveModule, String url, String roomCoverPic, int type, int uid,int moduleId) {
super(watcherNum, liveId, liveName, liver, description);
this.liveModule = liveModule;
this.url = url;
this.roomCoverPic = roomCoverPic;
this.type = type;
this.uid = uid;
this.moduleId = moduleId;
}
public int getModuleId() {
return moduleId;
}
public void setModuleId(int moduleId) {
this.moduleId = moduleId;
}
public String getLiveModule() {
return liveModule;
}
public void setLiveModule(String liveModule) {
this.liveModule = liveModule;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public int getHotTag() {
return hotTag;
public String getRoomCoverPic() {
return roomCoverPic;
}
public void setHotTag(int hotTag) {
this.hotTag = hotTag;
}
public int getIsAd() {
return isAd;
}
public void setIsAd(int isAd) {
this.isAd = isAd;
public void setRoomCoverPic(String roomCoverPic) {
this.roomCoverPic = roomCoverPic;
}
@@ -51,11 +73,11 @@ public class DouyuHotLive extends HotLive {
this.type = type;
}
public String getUrl() {
return url;
public int getUid() {
return uid;
}
public void setUrl(String url) {
this.url = url;
public void setUid(int uid) {
this.uid = uid;
}
}

18
common/src/main/java/org/example/bean/hotmodule/HotModuleList.java
View File

@@ -19,4 +19,22 @@ public class HotModuleList {
hotModuleList = new ArrayList<>();
}
public HotModule findHotModule(String hotModuleName){
for (HotModule hotModule : hotModuleList) {
if(hotModule.getTagName().equals(hotModuleName)){
return hotModule;
}
}
return null;
}
public HotModule findHotModule(int hotModuleId){
for (HotModule hotModule : hotModuleList) {
if(hotModule.getTagId().equals(String.valueOf(hotModuleId))){
return hotModule;
}
}
return null;
}
}

10
common/src/main/java/org/example/constpool/ConstPool.java
View File

@@ -20,4 +20,14 @@ public class ConstPool {
/**图片格式**/
public static final List<String> PIC_TYPES = List.of("jpg","jpeg","png","svg");
/**直播平台**/
public static final String DOUYU = "douyu";
public static final String HUYA = "huya";
public static final String BILIBILI = "bilibili";
public static final String TIKTOK = "douyin";
public static final String TWITCH = "twitch";
}