lywsvip/ChopperBot
1
0
Fork 0
mirror of https://github.com/Geniusay/ChopperBot.git synced 2026-05-31 19:12:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

1，完成热门模块的数据中心

Browse Source 
2，完成api接口
This commit is contained in:
userA
2023-07-22 16:50:58 +08:00
parent a55490c184
commit e5e4cb41de
43 changed files with 895 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CreeperModule/src/main/java/org/example/constpool/ApiPool.java
View File

@@ -10,6 +10,10 @@ package org.example.constpool;
* 存放每一个爬虫api的池子
*/
public class ApiPool {
public final static String HOT_MODULE_LIST_API = "/japi/search/api/getHotList";
public static final String DOUYU_HOT_MODULE_API = "https://www.douyu.com/japi/weblist/apinc/header/cate";
public static final String DOUYU_HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api
public static final String DOUYU_HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
}

2
CreeperModule/src/main/java/org/example/constpool/ConstPool.java → CreeperModule/src/main/java/org/example/constpool/CreeperModuleConstPool.java
View File

@@ -5,7 +5,7 @@ package org.example.constpool;
* @author 燧枫
* @date 2023/4/23 16:12
*/
public class ConstPool {
public class CreeperModuleConstPool {
/**
* 文件路径

44
CreeperModule/src/main/java/org/example/core/control/HotModuleLoadTask.java Normal file
View File

@@ -0,0 +1,44 @@
package org.example.core.control;
import org.example.log.HotModuleLogger;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/21 10:22
**/
public abstract class HotModuleLoadTask<T>{
public enum FinishFlag{
FINISH,NOT_FINISH,FAIL
}
private FinishFlag finishFlag = FinishFlag.NOT_FINISH;
public T start() {
clearFinishFlag();
return this.start0();
}
protected abstract T start0();
protected void fail(Exception e){
finishFlag = FinishFlag.FAIL;
HotModuleLogger.logger.error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage());
}
protected void success(){
finishFlag = FinishFlag.FINISH;
}
protected T getData(Spider spider,String url){
T data = ((ResultItems) spider.get(url)).get("data");
spider.close();
return data;
}
public FinishFlag isFinish(){
return finishFlag;
}
public void clearFinishFlag(){finishFlag = FinishFlag.NOT_FINISH;}
}

63
CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotLiveLoadTask.java Normal file
View File

@@ -0,0 +1,63 @@
package org.example.core.control.hotmodule;
import org.example.bean.HotLive;
import org.example.core.control.HotModuleLoadTask;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import java.util.List;
import static org.example.constpool.ApiPool.*;
/**
* @author Genius
* @date 2023/07/19 02:42
**/
public class DouyuHotLiveLoadTask extends HotModuleLoadTask<List<HotLive>> {
private final DouyuHotLiveProcessor douyuHotLiveProcessor;
public DouyuHotLiveLoadTask(){
douyuHotLiveProcessor = new DouyuHotLiveProcessor();
}
/**
* 获取Douyu某个模块下的热门直播
* @param moduleId
*/
public List<HotLive> start(int moduleId){
clearFinishFlag();
douyuHotLiveProcessor.setModuleId(moduleId);
return this.start(String.format(DOUYU_HOT_MODULE_LIVES_API,moduleId));
}
/**
* 获取Douyu当前最热直播
*/
@Override
protected List<HotLive> start0() {
return this.start(DOUYU_HOT_LIVES_API);
}
private List<HotLive> start(String url){
List<HotLive> lives;
try {
lives = getData(Spider.create(douyuHotLiveProcessor),url);
}catch (Exception e){
fail(e);
return null;
}
success();
return lives;
}
public static void main(String[] args) {
List<HotLive> start = new DouyuHotLiveLoadTask().start();
System.out.println(start);
}
}

32
CreeperModule/src/main/java/org/example/core/control/hotmodule/DouyuHotModuleLoadTask.java Normal file
View File

@@ -0,0 +1,32 @@
package org.example.core.control.hotmodule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.core.control.HotModuleLoadTask;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotModuleProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
import static org.example.constpool.ApiPool.DOUYU_HOT_MODULE_API;
/**
* @author Genius
* @date 2023/07/15 21:04
**/
public class DouyuHotModuleLoadTask extends HotModuleLoadTask<HotModuleList> {
@Override
protected HotModuleList start0() {
HotModuleList data;
try {
data = getData(Spider.create(new DouyuHotModuleProcessor()),DOUYU_HOT_MODULE_API);
}catch (Exception e){
fail(e);
return null;
}
success();
return data;
}
}

4
CreeperModule/src/main/java/org/example/core/control/impl/BilibiliLiveLoadTask.java
View File

@@ -1,6 +1,6 @@
package org.example.core.control.impl;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.core.control.LoadTask;
import org.example.core.factory.ProcessorFactory;
import org.example.core.pipeline.PipelineWriteJson;
@@ -34,7 +34,7 @@ public class BilibiliLiveLoadTask implements LoadTask {
public void start() {
Spider.create(bilibiliLiveProcessor)
// 设置起始Request
.addRequest(new Request(ConstPool.OCCUURL))
.addRequest(new Request(CreeperModuleConstPool.OCCUURL))
// 设置结果处理类
.addPipeline(pipelineWriteJson)
// 设置抓取线程数（可根据需要调整）

77
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotLiveLoadTask.java
View File

@@ -1,77 +0,0 @@
package org.example.core.control.impl;
import org.example.constpool.HotModulePool;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotLiveProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/19 02:42
**/
public class DouyuHotLiveLoadTask implements LoadTask {
private final String HOT_LIVES_API = "https://www.douyu.com/japi/weblist/apinc/allpage/6/1"; //全部热门直播api
private final String HOT_MODULE_LIVES_API = "https://www.douyu.com/gapi/rkc/directory/mixList/2_%s/1"; //某个模块热门直播api
private final DouyuHotLiveProcessor douyuHotLiveProcessor;
public DouyuHotLiveLoadTask(){
douyuHotLiveProcessor = new DouyuHotLiveProcessor();
}
/**
* 获取Douyu某个模块下的热门直播
* @param moduleId
*/
public void start(int moduleId){
douyuHotLiveProcessor.setModuleId(moduleId);
this.start(String.format(HOT_MODULE_LIVES_API,moduleId));
}
/**
* 获取Douyu当前最热直播
*/
@Override
public void start() {
this.start(HOT_LIVES_API);
}
private void start(String url){
try {
Spider.create(douyuHotLiveProcessor)
.addRequest(new Request(url))
.thread(1)
.run();
}catch (Exception e){
e.printStackTrace();
}
}
@Override
public void end() {
}
@Override
public boolean isRunning() {
return false;
}
@Override
public int getCacheSize() {
return 0;
}
@Override
public int flushCacheAndSave(String key) {
return 0;
}
public static void main(String[] args) {
new DouyuHotModuleLoadTask().start();
new DouyuHotLiveLoadTask().start(1);
System.out.println(HotModulePool.hotModuleListPool);
}
}

48
CreeperModule/src/main/java/org/example/core/control/impl/DouyuHotModuleLoadTask.java
View File

@@ -1,48 +0,0 @@
package org.example.core.control.impl;
import org.example.constpool.HotModulePool;
import org.example.core.control.LoadTask;
import org.example.core.processor.hotmodule.DouyuHotModuleProcessor;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/07/15 21:04
**/
public class DouyuHotModuleLoadTask implements LoadTask {
private final String url = "https://www.douyu.com/japi/weblist/apinc/header/cate";
@Override
public void start() {
DouyuHotModuleProcessor douyuHotModuleProcessor = new DouyuHotModuleProcessor();
Spider.create(douyuHotModuleProcessor)
.addRequest(new Request(url))
.thread(1)
.run();
}
@Override
public void end() {
}
@Override
public boolean isRunning() {
return false;
}
@Override
public int getCacheSize() {
return 0;
}
@Override
public int flushCacheAndSave(String key) {
return 0;
}
public static void main(String[] args) {
new DouyuHotModuleLoadTask().start();
}
}

4
CreeperModule/src/main/java/org/example/core/control/impl/DouyuRecordLoadTask.java
View File

@@ -1,6 +1,6 @@
package org.example.core.control.impl;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.core.control.LoadTask;
import org.example.core.factory.ProcessorFactory;
import org.example.core.pipeline.PipelineWriteJson;
@@ -34,7 +34,7 @@ public class DouyuRecordLoadTask implements LoadTask {
public void start() {
Spider.create(douyuRecordProcessor)
// 设置起始Request
.addRequest(new Request(ConstPool.OCCUURL))
.addRequest(new Request(CreeperModuleConstPool.OCCUURL))
// 设置结果处理类
.addPipeline(pipelineWriteJson)
// 设置抓取线程数（可根据需要调整）

28
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotLiveProcessor.java
View File

@@ -4,20 +4,12 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.HotLive;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.DouyuHotLive;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import org.example.util.ChineseConvertUtil;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Json;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* @author Genius
@@ -60,25 +52,9 @@ public class DouyuHotLiveProcessor implements PageProcessor {
}
}
}catch (Exception e){
HotModuleLogger.logger.error("Douyu Hot Live List require fail! Exception:{}",e.getMessage());
throw e;
}
HotModuleLogger.logger.info(hotLiveList.toString());
updateHotLiveList(hotLiveList);
page.putField("data",hotLiveList);
}
private void updateHotLiveList(List<HotLive> hotLiveList){
if(moduleId!=-1){
if(HotModulePool.hotModuleListPool.containsKey(HotModulePool.DouYuAllHotModules)){
HotModuleList hotModuleList = HotModulePool.hotModuleListPool.get(HotModulePool.DouYuAllHotModules);
HotModule hotModule = hotModuleList.findHotModule(this.moduleId);
if(hotModule!=null){
hotModule.setHotLives(hotLiveList);
HotModuleLogger.logger.info("Douyu module {} hotLiveListPool successfully updated",hotModule.getTagName());
}
}
}else{
HotModuleLogger.logger.info("Douyu hotLiveListPool successfully updated");
HotModulePool.hotLiveListPool.put(HotModulePool.DouYuAllHotLives,hotLiveList);
}
}
}

7
CreeperModule/src/main/java/org/example/core/processor/hotmodule/DouyuHotModuleProcessor.java
View File

@@ -5,12 +5,9 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.hotmodule.DouyuHotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.HotModulePool;
import org.example.log.HotModuleLogger;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.concurrent.ExecutorService;
/**
* @author Genius
@@ -38,9 +35,9 @@ public class DouyuHotModuleProcessor implements PageProcessor {
}
}
}catch (Exception e){
HotModuleLogger.logger.error("Douyu Hot module list require fail! Exception:{}",e.getMessage());
throw e;
}
HotModulePool.hotModuleListPool.put(HotModulePool.DouYuAllHotModules,douyuHotModuleList);
page.putField("data",douyuHotModuleList);
}
}

2
CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java
View File

@@ -13,7 +13,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.ConcurrentLinkedQueue;
import static org.example.constpool.ConstPool.BARRAGE_ROOT;
import static org.example.constpool.CreeperModuleConstPool.BARRAGE_ROOT;
/**
* @author Genius

4
CreeperModule/src/main/java/org/example/pojo/download/assign/BilibiliLiveLoadBarrageConfig.java
View File

@@ -1,7 +1,7 @@
package org.example.pojo.download.assign;
import lombok.Data;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.pojo.download.LoadBarrageConfig;
/**
@@ -16,7 +16,7 @@ public class BilibiliLiveLoadBarrageConfig extends LoadBarrageConfig {
private String roomId;
public BilibiliLiveLoadBarrageConfig(String anchorName, String roomId) {
super(ConstPool.BILIBILI, ConstPool.ACTION_LIVE, anchorName);
super(CreeperModuleConstPool.BILIBILI, CreeperModuleConstPool.ACTION_LIVE, anchorName);
this.roomId = roomId;
}
}

4
CreeperModule/src/main/java/org/example/pojo/download/assign/DouyuRecordLoadBarrageConfig.java
View File

@@ -1,7 +1,7 @@
package org.example.pojo.download.assign;
import lombok.Data;
import org.example.constpool.ConstPool;
import org.example.constpool.CreeperModuleConstPool;
import org.example.pojo.download.LoadBarrageConfig;
/**
@@ -16,7 +16,7 @@ public class DouyuRecordLoadBarrageConfig extends LoadBarrageConfig {
private String vid;
public DouyuRecordLoadBarrageConfig(String anchorName, String vid) {
super(ConstPool.DOUYU, ConstPool.ACTION_RECORD, anchorName);
super(CreeperModuleConstPool.DOUYU, CreeperModuleConstPool.ACTION_RECORD, anchorName);
this.vid = vid;
}
}