diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java index 292238d..e0dfc27 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java @@ -12,20 +12,41 @@ import us.codecraft.webmagic.Spider; **/ public class DouyuHotLiveLoadTask implements LoadTask { - private String url = "https://www.douyu.com/japi/search/api/getHotList"; + private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api + private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api + + private final DouyuHotLiveProcessor douyuHotLiveProcessor; + + public DouyuHotLiveLoadTask(){ + douyuHotLiveProcessor = new DouyuHotLiveProcessor(); + } + + /** + * 获取Douyu某个模块下的热门直播 + * @param moduleId + */ + public void start(int moduleId){ + douyuHotLiveProcessor.setModuleId(moduleId); + this.start(String.format(HOT_MODULE_LIVES_API,moduleId)); + } + + /** + * 获取Douyu当前最热直播 + */ @Override public void start() { + this.start(HOT_LIVES_API); + } + + private void start(String url){ try { - DouyuHotLiveProcessor douyuHotLiveProcessor = new DouyuHotLiveProcessor(); Spider.create(douyuHotLiveProcessor) .addRequest(new Request(url)) - .setEmptySleepTime(10) .thread(1) .run(); }catch (Exception e){ e.printStackTrace(); } - } @Override @@ -49,7 +70,8 @@ public class DouyuHotLiveLoadTask implements LoadTask { } public static void main(String[] args) { - new DouyuHotLiveLoadTask().start(); - System.out.println(HotModulePool.hotLiveListPool); + new DouyuHotModuleLoadTask().start(); + new DouyuHotLiveLoadTask().start(1); + System.out.println(HotModulePool.hotModuleListPool); } } diff --git a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java index 0b87725..a337d56 100644 --- a/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java +++ b/CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java @@ -12,13 +12,12 @@ import us.codecraft.webmagic.Spider; **/ public class DouyuHotModuleLoadTask implements LoadTask { - private String url = "https://www.douyu.com/japi/weblist/apinc/header/cate"; + private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate"; @Override public void start() { DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor(); Spider.create(douyuHotModuleProcessor) .addRequest(new Request(url)) - .setEmptySleepTime(10) .thread(1) .run(); } diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java index f122d36..f942b98 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java @@ -4,49 +4,81 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.bean.HotLive; +import org.example.bean.HotModule; import org.example.bean.hotmodule.DouyuHotLive; +import org.example.bean.hotmodule.HotModuleList; import org.example.constpool.HotModulePool; import org.example.log.HotModuleLogger; import org.example.util.ChineseConvertUtil; +import org.springframework.util.StringUtils; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.selector.Json; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * @author Genius * @date 2023/07/19 00:30 **/ public class DouyuHotLiveProcessor implements PageProcessor { + private int moduleId; // 模块Id + public DouyuHotLiveProcessor(){ + this.moduleId = -1; + } + + public void setModuleId(int moduleId) { + this.moduleId = moduleId; + } @Override public void process(Page page) { List hotLiveList = new ArrayList<>(); try{ - JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("data"); + JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONObject("data").getJSONArray("rl"); + for (Object live : Lives) { if(live instanceof JSONObject){ JSONObject jsonLive = (JSONObject) live; hotLiveList.add(new DouyuHotLive( - ChineseConvertUtil.cnNumericUnitsToInt(jsonLive.getString("hot")), - Integer.parseInt(jsonLive.getString("relId")), - jsonLive.getString("sk"), - jsonLive.containsKey("description")?jsonLive.getString("description"):"", - jsonLive.getInteger("hotTag"), - jsonLive.getInteger("isAd"), + jsonLive.getInteger("ol"), + jsonLive.getInteger("rid"), + jsonLive.getString("rn"), + jsonLive.getString("nn"), + jsonLive.getString("od"), + jsonLive.getString("c2name"), + jsonLive.getString("url"), + jsonLive.getString("rs16"), jsonLive.getInteger("type"), - jsonLive.getString("url") + jsonLive.getInteger("uid"), + jsonLive.getInteger("cid2") )); } } }catch (Exception e){ - HotModuleLogger.logger.error("DouYu Hot Live List require fail! Exception:{}",e.getMessage()); + HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage()); } - HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList); + HotModuleLogger.logger.info(hotLiveList.toString()); + updateHotLiveList(hotLiveList); } + private void updateHotLiveList(List hotLiveList){ + if(moduleId!=-1){ + if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){ + HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules); + HotModule hotModule = hotModuleList.findHotModule(this.moduleId); + if(hotModule!=null){ + hotModule.setHotLives(hotLiveList); + HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName()); + } + } + }else{ + HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated"); + HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList); + } + } } diff --git a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java index e9430d8..d4868c4 100644 --- a/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java +++ b/CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java @@ -6,16 +6,18 @@ import com.alibaba.fastjson.JSONObject; import org.example.bean.hotmodule.DouyuHotModule; import org.example.bean.hotmodule.HotModuleList; import org.example.constpool.HotModulePool; +import org.example.log.HotModuleLogger; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.processor.PageProcessor; +import java.util.concurrent.ExecutorService; + /** * @author Genius * @date 2023/06/01 22:11 **/ public class DouyuHotModuleProcessor implements PageProcessor { - @Override public void process(Page page) { HotModuleList douyuHotModuleList = new HotModuleList(); @@ -36,7 +38,7 @@ public class DouyuHotModuleProcessor implements PageProcessor { } } }catch (Exception e){ - + HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage()); } HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList); } diff --git a/FileModule/src/main/java/org/example/cache/FileCacheManager.java b/FileModule/src/main/java/org/example/cache/FileCacheManager.java index cce49f8..77d2891 100644 --- a/FileModule/src/main/java/org/example/cache/FileCacheManager.java +++ b/FileModule/src/main/java/org/example/cache/FileCacheManager.java @@ -6,10 +6,7 @@ import org.slf4j.LoggerFactory; import java.time.LocalDateTime; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -28,6 +25,8 @@ public class FileCacheManager { private Logger logger = LoggerFactory.getLogger(FileCacheManager.class); private final List fileCaches; + private final ConcurrentHashMap fileCacheMap; + private AtomicLong sleepTime; //睡眠时间 private ExecutorService watchPool; //巡逻线程 @@ -38,6 +37,10 @@ public class FileCacheManager { protected FileCacheManager(List fileCaches){ this.fileCaches = new CopyOnWriteArrayList<>(fileCaches); + fileCacheMap = new ConcurrentHashMap<>(); + for (FileCache fileCache : fileCaches) { + fileCacheMap.put(fileCache.getFullFilePath(),fileCache); + } initSleepTime(); this.watchPool = Executors.newSingleThreadExecutor(); this.autoSyncer = Executors.newFixedThreadPool(fileCaches.size()); @@ -70,11 +73,16 @@ public class FileCacheManager { public boolean addFileCache(FileCache fileCache){ if (this.fileCaches.indexOf(fileCache)==-1) { fileCaches.add(fileCache); + fileCacheMap.put(fileCache.getFullFilePath(),fileCache); initSleepTime(); } return false; } + public FileCache getFileCache(String filePath){ + return fileCacheMap.get(filePath); + } + public List getRunnableFileCaches(){ return this.fileCaches; } diff --git a/HotModule/pom.xml b/HotModule/pom.xml index 56ba22a..288e9df 100644 --- a/HotModule/pom.xml +++ b/HotModule/pom.xml @@ -18,6 +18,11 @@ + + org.example + CreeperModule + 1.0-SNAPSHOT + junit junit diff --git a/HotModule/src/main/java/org/example/config/FollowDog.java b/HotModule/src/main/java/org/example/config/FollowDog.java new file mode 100644 index 0000000..cb42290 --- /dev/null +++ b/HotModule/src/main/java/org/example/config/FollowDog.java @@ -0,0 +1,22 @@ +package org.example.config; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/20 00:40 + **/ + +//config中的跟风狗设置 +@Data +@AllArgsConstructor +public class FollowDog { + public final static String ALL_LIVES = "all"; + private String moduleName; //模块名称，all代表热门直播 + private int top; //前几个 + private List banLiver; //去除的主播，可以是正则 + +} diff --git a/HotModule/src/main/java/org/example/config/HotModuleConfig.java b/HotModule/src/main/java/org/example/config/HotModuleConfig.java new file mode 100644 index 0000000..2db456f --- /dev/null +++ b/HotModule/src/main/java/org/example/config/HotModuleConfig.java @@ -0,0 +1,44 @@ +package org.example.config; + +import org.example.bean.ConfigFile; +import org.example.constpool.ConstPool; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author Genius + * @date 2023/07/20 00:20 + **/ +public class HotModuleConfig extends ConfigFile> { + private static final FollowDog allLivesDog = new FollowDog(FollowDog.ALL_LIVES,6,new ArrayList<>()); + private static final int FiveMinute = 0x493E0; + + private static final String filePath = "./config"; + + private static final String fileName = "hotmodule.config"; + + public HotModuleConfig(){ + super(filePath,fileName, + Map.of("Enable", 1, + "Module", List.of( + new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(), + false, List.of(allLivesDog), FiveMinute, FiveMinute), + new ModuleSetting(ConstPool.BILIBILI, true, new ArrayList<>(), + false, List.of(allLivesDog), FiveMinute, FiveMinute), + new ModuleSetting(ConstPool.HUYA, true, new ArrayList<>(), + false, List.of(allLivesDog), FiveMinute, FiveMinute), + new ModuleSetting(ConstPool.DOUYU, true, new ArrayList<>(), + false, List.of(allLivesDog), FiveMinute, FiveMinute) + ), + "GuardNum",10 + ) + ); + } + + public static String getFullFilePath(){ + return Paths.get(filePath,fileName).toString(); + } +} diff --git a/HotModule/src/main/java/org/example/config/ModuleSetting.java b/HotModule/src/main/java/org/example/config/ModuleSetting.java new file mode 100644 index 0000000..18dfd09 --- /dev/null +++ b/HotModule/src/main/java/org/example/config/ModuleSetting.java @@ -0,0 +1,30 @@ +package org.example.config; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +/** + * @author Genius + * @date 2023/07/20 00:25 + **/ +@Data +@AllArgsConstructor +public class ModuleSetting { + + private String platform; //平台 + + private boolean autoWork; //是否自动进行主播直播下载任务推送 + + private List focusLiver; //关注的主播 + + private boolean followDogEnable; //当开启autoWork时，启用跟风狗模式，跟风狗模式会自动爬取热门直播 + + private List followDogs; + + private long updateHotModuleTimes; //自动更新平台热门模块时间 + + private long updateHotLivesTimes; //自动更新平台热门直播时间 + +} diff --git a/HotModule/src/main/java/org/example/guard/HotModuleGuard.java b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java new file mode 100644 index 0000000..37d4c06 --- /dev/null +++ b/HotModule/src/main/java/org/example/guard/HotModuleGuard.java @@ -0,0 +1,39 @@ +package org.example.guard; + +import org.example.cache.FileCacheManager; +import org.example.cache.FileCacheManagerInstance; +import org.example.config.HotModuleConfig; +import org.example.core.control.impl.DouyuHotLiveLoadTask; +import org.example.core.control.impl.DouyuHotModuleLoadTask; +import org.example.core.processor.hotmodule.DouyuHotLiveProcessor; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author Genius + * @date 2023/07/19 03:21 + **/ +public class HotModuleGuard { + + private static final long delayTime = 10*1000; + private ScheduledExecutorService hotModuleGuardPool = Executors.newScheduledThreadPool( + (Integer)FileCacheManagerInstance.getInstance().getFileCache(HotModuleConfig.getFullFilePath()).get("GuardNum") + ); + + public void start(){ + hotModuleGuardPool.scheduleWithFixedDelay(()-> { + new DouyuHotLiveLoadTask().start(); + },0,delayTime, TimeUnit.MILLISECONDS); + + hotModuleGuardPool.scheduleWithFixedDelay(()->{ + new DouyuHotModuleLoadTask().start(); + },0,delayTime,TimeUnit.MILLISECONDS); + } + + public static void main(String[] args) { + new HotModuleGuard().start(); + } +} diff --git a/common/src/main/java/org/example/bean/HotLive.java b/common/src/main/java/org/example/bean/HotLive.java index 8f15386..01f4bc5 100644 --- a/common/src/main/java/org/example/bean/HotLive.java +++ b/common/src/main/java/org/example/bean/HotLive.java @@ -9,10 +9,12 @@ package org.example.bean; * 热门直播抽象类 */ public abstract class HotLive { - private int watcherNum; - private int liveId; - private String liveName; - private String description; + private int watcherNum; //直播间观众数目 + private int liveId; //直播间ID + private String liveName; //直播间名字 + + private String liver; //主播 + private String description; //直播间简介 public HotLive(int watcherNum, int liveId, String liveName, String description) { this.watcherNum = watcherNum; @@ -21,6 +23,14 @@ public abstract class HotLive { this.description = description; } + public HotLive(int watcherNum, int liveId, String liveName, String liver, String description) { + this.watcherNum = watcherNum; + this.liveId = liveId; + this.liveName = liveName; + this.liver = liver; + this.description = description; + } + public int getWatcherNum() { return watcherNum; } @@ -45,6 +55,14 @@ public abstract class HotLive { this.liveName = liveName; } + public String getLiver() { + return liver; + } + + public void setLiver(String liver) { + this.liver = liver; + } + public String getDescription() { return description; } diff --git a/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java b/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java index 1e734bf..0afb48b 100644 --- a/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java +++ b/common/src/main/java/org/example/bean/hotmodule/DouyuHotLive.java @@ -8,38 +8,60 @@ import org.example.bean.HotLive; **/ public class DouyuHotLive extends HotLive { - private int hotTag; + private String liveModule; //直播板块 - private int isAd; + private String url; //直播间地址 + + private String roomCoverPic; //直播间封面地址 private int type; - private String url; + private int uid; - public DouyuHotLive(int watcherNum, int liveId, String liveName, String description, - int hotTag, int isAd, int type, String url) { - super(watcherNum, liveId, liveName, description); - this.hotTag = hotTag; - this.isAd = isAd; + private int moduleId; //直播板块Id + + public DouyuHotLive(int watcherNum, int liveId, String liveName, String liver, String description, + String liveModule, String url, String roomCoverPic, int type, int uid,int moduleId) { + super(watcherNum, liveId, liveName, liver, description); + this.liveModule = liveModule; + this.url = url; + this.roomCoverPic = roomCoverPic; this.type = type; + this.uid = uid; + this.moduleId = moduleId; + } + + public int getModuleId() { + return moduleId; + } + + public void setModuleId(int moduleId) { + this.moduleId = moduleId; + } + + public String getLiveModule() { + return liveModule; + } + + public void setLiveModule(String liveModule) { + this.liveModule = liveModule; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { this.url = url; } - public int getHotTag() { - return hotTag; + public String getRoomCoverPic() { + return roomCoverPic; } - public void setHotTag(int hotTag) { - this.hotTag = hotTag; - } - - public int getIsAd() { - return isAd; - } - - public void setIsAd(int isAd) { - this.isAd = isAd; + public void setRoomCoverPic(String roomCoverPic) { + this.roomCoverPic = roomCoverPic; } @@ -51,11 +73,11 @@ public class DouyuHotLive extends HotLive { this.type = type; } - public String getUrl() { - return url; + public int getUid() { + return uid; } - public void setUrl(String url) { - this.url = url; + public void setUid(int uid) { + this.uid = uid; } } diff --git a/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java b/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java index 88c547f..40e8c43 100644 --- a/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java +++ b/common/src/main/java/org/example/bean/hotmodule/HotModuleList.java @@ -19,4 +19,22 @@ public class HotModuleList { hotModuleList = new ArrayList<>(); } + public HotModule findHotModule(String hotModuleName){ + for (HotModule hotModule : hotModuleList) { + if(hotModule.getTagName().equals(hotModuleName)){ + return hotModule; + } + } + return null; + } + + public HotModule findHotModule(int hotModuleId){ + for (HotModule hotModule : hotModuleList) { + if(hotModule.getTagId().equals(String.valueOf(hotModuleId))){ + return hotModule; + } + } + return null; + } + } diff --git a/common/src/main/java/org/example/constpool/ConstPool.java b/common/src/main/java/org/example/constpool/ConstPool.java index 4e44bca..74c6402 100644 --- a/common/src/main/java/org/example/constpool/ConstPool.java +++ b/common/src/main/java/org/example/constpool/ConstPool.java @@ -20,4 +20,14 @@ public class ConstPool { /**图片格式**/ public static final List PIC_TYPES = List.of("jpg","jpeg","png","svg"); + + /**直播平台**/ + public static final String DOUYU = "douyu"; + + public static final String HUYA = "huya"; + public static final String BILIBILI = "bilibili"; + + public static final String TIKTOK = "douyin"; + + public static final String TWITCH = "twitch"; }