虎牙平台爬虫 融入ChopperBot #33

This commit is contained in:
userA
2023-10-17 02:18:40 +08:00
parent e491639b9c
commit 1ddd9712e8
77 changed files with 779 additions and 483 deletions

2
.idea/compiler.xml generated
View File

@@ -38,7 +38,7 @@
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="ChopperBot" options="-parameters" />
<module name="ChopperBot" options="" />
<module name="chopperbot-account" options="-parameters" />
<module name="chopperbot-barrage" options="-parameters" />
<module name="chopperbot-common" options="-parameters" />

View File

@@ -2,6 +2,7 @@ package org.example.api;
import org.example.constpool.ConstPool;
import org.example.constpool.GlobalFileCache;
import org.example.util.ExceptionUtil;
import java.io.File;
import java.util.*;
@@ -45,7 +46,7 @@ public class VideoApi {
return false;
}
}catch (SecurityException e) {
System.out.println("删除文件时发生安全异常: " + e.getMessage());
System.out.println("删除文件时发生安全异常: " + ExceptionUtil.getCause(e));
return false;
}
}

View File

@@ -9,6 +9,7 @@ import org.example.core.bgevnet.BarrageEventCenter;
import org.example.core.bgevnet.instantslicing.InstantSlicingPlugin;
import org.example.core.section.SectionRequest;
import org.example.core.section.VideoSectionWorkShop;
import org.example.core.taskcenter.TaskCenter;
import org.example.core.taskcenter.observer.AbstractTaskCenterObserver;
import org.example.core.taskcenter.task.ReptileTask;
import org.example.init.InitPluginRegister;
@@ -36,6 +37,7 @@ public class LiveAndBarrageHandlerObserver extends AbstractTaskCenterObserver {
@Override
public void onFinish(ReptileTask task) {
taskCenter.stopTask(task.getTaskId().replace("live","barrage"));
if(!InitPluginRegister.isRegister(PluginName.INSTANT_SLICING_PLUGIN)){
PluginCheckAndDo.CheckAndDo((plugin)->{
Object param = task.getRequest().getParam();

View File

@@ -15,9 +15,7 @@ import org.example.init.InitPluginRegister;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.thread.NamedThreadFactory;
import org.example.util.BarrageUtil;
import org.example.util.TimeUtil;
import org.example.util.VideoUtil;
import org.example.util.*;
import java.nio.file.Path;
import java.util.HashMap;
@@ -103,18 +101,24 @@ public class ScheduleTimeHandler implements InstantSlicingHandler {
}
}
}catch (Exception e){
ChopperLogFactory.getLogger(LoggerType.Barrage).error("Error:{}",e.toString());
ChopperLogFactory.getLogger(LoggerType.Barrage).error("Error:{}", ExceptionUtil.getCause(e));
}
}
}
private boolean splitBarrageFile(String oldPath,String newPath,long startTime,long endTime){
return BarrageUtil.cutBarrageFile(oldPath,newPath,startTime,endTime);
if(FileUtil.isFileExist(oldPath)){
return BarrageUtil.cutBarrageFile(oldPath,newPath,startTime,endTime);
}
return false;
}
private boolean spiltVideoFile(String oldPath,String newPath,long startSecond,long endSecond){
return VideoUtil.cutVideoByFFMpeg(oldPath,newPath,startSecond,endSecond);
if(FileUtil.isFileExist(oldPath)){
return VideoUtil.cutVideoByFFMpeg(oldPath,newPath,startSecond,endSecond);
}
return false;
}
@Override

View File

@@ -0,0 +1,24 @@
package org.example.core.creeper.builder;
import org.example.bean.live.DouyuLive;
import org.example.bean.live.HuyaLive;
import org.example.core.creeper.loadconfig.DouyuLiveBarrageLoadConfig;
import org.example.core.creeper.loadconfig.HuyaLiveBarrageLoadConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/16 22:03
**/
@Component
public class HuyaLiveBarrageLoadConfigBuilder extends CommonLoadConfigBuilder<HuyaLiveBarrageLoadConfig> {
@Override
public HuyaLiveBarrageLoadConfig build(Object obj) {
if(obj instanceof HuyaLive){
return new HuyaLiveBarrageLoadConfig(((HuyaLive) obj).getLiver(),((HuyaLive) obj).getLiveId());
}
return null;
}
}

View File

@@ -1,23 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:22
**/
@Component
public class BilibiliLiveBarrageGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.BILIBILI;
}
@Override
public String getFunctionName() {
return ConstGroup.BARRAGE_ONLINE;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/12 23:02
**/
@Component
public class DouyuLiveBarrageGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.DOUYU;
}
@Override
public String getFunctionName() {
return ConstGroup.BARRAGE_ONLINE;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.example.ws.handler.AbstractMessageHandler;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:28
**/
@Component
public class DouyuRecordBarrageGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.DOUYU;
}
@Override
public String getFunctionName() {
return ConstGroup.BARRAGE_RECORD;
}
}

View File

@@ -1,21 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
/**
* @author dhx
* @date 2023/10/16 15:59
*/
public class HuyaLiveBarrageGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.HUYA;
}
@Override
public String getFunctionName() {
return ConstGroup.BARRAGE_ONLINE;
}
}

View File

@@ -23,7 +23,7 @@ public class HuyaLiveBarrageLoadConfig extends LoadBarrageConfig{
private String roomId;
public HuyaLiveBarrageLoadConfig(String anchorName,String roomId) {
super(ConstPool.DOUYU, CreeperModuleConstPool.ACTION_LIVE, anchorName);
super(ConstPool.HUYA, CreeperModuleConstPool.ACTION_LIVE, anchorName);
this.roomId = roomId;
this.url = "wss://cdnws.api.huya.com/";
}

View File

@@ -13,6 +13,7 @@ import org.example.core.taskmonitor.Monitor;
import org.example.core.taskmonitor.MonitorCenter;
import org.example.exception.FileCacheException;
import org.example.plugin.PluginCheckAndDo;
import org.example.util.ExceptionUtil;
import org.java_websocket.handshake.ServerHandshake;
import java.nio.ByteBuffer;
@@ -113,7 +114,7 @@ public class DouyuLiveBarrageLoadTask extends WebSocketLoadTask<List<DouyuBarrag
@Override
public void onError(Exception e) {
client.close();
this.logger.error("Douyu Live Barrage Error:{}",e.getMessage());
this.logger.error("Douyu Live Barrage Error:{}", ExceptionUtil.getCause(e));
}
public DouyuBarrage wrapperBarrage(ByteBuffer buffer, long startTime){

View File

@@ -17,6 +17,7 @@ import org.example.core.taskmonitor.Monitor;
import org.example.core.taskmonitor.MonitorCenter;
import org.example.exception.FileCacheException;
import org.example.plugin.PluginCheckAndDo;
import org.example.util.ExceptionUtil;
import org.java_websocket.handshake.ServerHandshake;
import java.io.UnsupportedEncodingException;
@@ -46,14 +47,6 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
fileCache = new BarrageFileCache(new BarrageSaveFile<>(loadConfig,new ConcurrentLinkedQueue<>()));
}
public static String stringToHex(String input) {
StringBuilder hexString = new StringBuilder();
for (char c : input.toCharArray()) {
hexString.append(String.format("%02X ", (int) c));
}
return hexString.toString();
}
@Override
public List<HuyaBarrage> handler() {
return list;
@@ -100,7 +93,15 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
@Override
public void onMessage(ByteBuffer buffer) {
HuyaBarrage huyaBarrage = wrapperBarrage(buffer,0);
HuyaBarrage huyaBarrage = wrapperBarrage(buffer);
if(huyaBarrage!=null){
list.add(huyaBarrage);
try {
fileCache.append(huyaBarrage,"-1");
}catch (FileCacheException | InterruptedException e){
throw new RuntimeException(e);
}
}
}
@Override
@@ -111,13 +112,13 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
@Override
public void onError(Exception e) {
client.close();
this.logger.error("Huya Live Barrage Error:{}",e.getMessage());
this.logger.error("Huya Live Barrage Error:{}", ExceptionUtil.getCause(e));
}
public HuyaBarrage wrapperBarrage(ByteBuffer buffer, long startTime){
String name = "";
public HuyaBarrage wrapperBarrage(ByteBuffer buffer){
String content = "";
TarsInputStream tarsInputStream = new TarsInputStream(buffer);
long nowTime = System.currentTimeMillis();
int int32 = 0;
long int64 = 0L;
byte[] bytes = null;
@@ -125,18 +126,13 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
tarsInputStream = new TarsInputStream(tarsInputStream.read(bytes,1,false));
if(tarsInputStream.read(int64,1,false)==1400){
tarsInputStream = new TarsInputStream(tarsInputStream.read(bytes,2,false));
// name = tarsInputStream.read("",0,false);
tarsInputStream.setServerEncoding("utf-8");
content = tarsInputStream.read("", 3, true);
try {
System.out.println(new String(content.getBytes("GBK")));
System.out.println(stringToHex(content));
System.out.println(new String(content.replaceAll("\uFFFD", "").getBytes("GBK")));
System.out.println(stringToHex(new String(content.getBytes("GBK"))));
} catch (UnsupportedEncodingException e) {
return new HuyaBarrage("",nowTime,nowTime-startTime,content);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println(name);
}
}
return null;

View File

@@ -75,16 +75,11 @@ public class HuyaProtocol {
regDatas = wscmd.toByteArray();
return regDatas;
}catch (Exception e){
e.printStackTrace();
throw new RuntimeException("解析失败");
}
return null;
}
public static byte[] getHeartbeat(){
return heartbeat;
}
public static void main(String[] args) throws IOException {
String roomId = "dank1ng";
System.out.println(HexUtil.encodeHex(getWsInfo(roomId)));
}
}

View File

@@ -16,6 +16,7 @@ import java.time.LocalDateTime;
*/
@Data
public abstract class Live implements Serializable {
private int watcherNum; //直播间观众数目
private String liveId; //直播间ID
private String liveName; //直播间名字
@@ -33,6 +34,9 @@ public abstract class Live implements Serializable {
private String roomPic;
public Live() {
}
public Live(int watcherNum, String liveId, String liveName, String description) {
this.watcherNum = watcherNum;
this.liveId = liveId;
@@ -68,4 +72,5 @@ public abstract class Live implements Serializable {
this.moduleName = moduleName;
}
}

View File

@@ -0,0 +1,15 @@
package org.example.bean.hotmodule;
import org.example.bean.HotModule;
import org.example.bean.live.HuyaLive;
/**
* @author Genius
* @date 2023/10/17 00:42
**/
public class HuyaHotModule extends HotModule<HuyaLive> {
public HuyaHotModule(String tagId, String tagName) {
super(tagId, tagName);
}
}

View File

@@ -0,0 +1,34 @@
package org.example.bean.live;
import lombok.Data;
import org.example.bean.Live;
/**
* @author Genius
* @date 2023/10/16 22:04
**/
@Data
public class HuyaLive extends Live {
private String uid;
public HuyaLive(String uid) {
this.uid = uid;
}
public HuyaLive(int watcherNum, String liveId, String liveName, String description) {
super(watcherNum, liveId, liveName, description);
}
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description) {
super(watcherNum, liveId, liveName, liver, description);
}
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description, String platform) {
super(watcherNum, liveId, liveName, liver, description, platform);
}
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description, String platform, String moduleId, String moduleName) {
super(watcherNum, liveId, liveName, liver, description, platform, moduleId, moduleName);
}
}

View File

@@ -0,0 +1,24 @@
package org.example.core;
/**
* @author Genius
* @date 2023/10/17 01:30
**/
public class SafeBag<T> {
T data;
public SafeBag(T data) {
this.data = data;
}
public SafeBag() {
}
public T getData() {
return data;
}
public void setData(T data){
this.data = data;
}
}

View File

@@ -8,6 +8,7 @@ package org.example.init;
import org.example.log.ResultLogger;
import org.example.plugin.CommonPlugin;
import org.example.plugin.annotation.Plugin;
import org.example.util.ExceptionUtil;
import org.slf4j.Logger;
import java.util.ArrayList;
@@ -89,7 +90,7 @@ public abstract class CommonInitMachine implements ComponentInitMachine, ResultL
return fail();
}
}catch (Exception e){
return fail(e.getMessage());
return fail(ExceptionUtil.getCause(e));
}
}

View File

@@ -13,6 +13,7 @@ import org.example.plugin.CommonPlugin;
import org.example.plugin.annotation.Plugin;
import org.example.thread.ChopperBotGuardPool;
import org.example.util.ClassUtil;
import org.example.util.ExceptionUtil;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
@@ -161,7 +162,7 @@ public class InitPluginRegister {
}
}catch (Exception e){
ChopperLogFactory.getLogger(LoggerType.System)
.error(String.format("❌ %s plugin start failed ! error:%s", pluginName,e.getMessage()));
.error(String.format("❌ %s plugin start failed ! error:%s", pluginName, ExceptionUtil.getCause(e)));
}
}else{
return false;

View File

@@ -1,5 +1,6 @@
package org.example.init;
import org.example.util.ExceptionUtil;
import org.example.util.PluginUtil;
import org.slf4j.Logger;
@@ -50,7 +51,7 @@ public abstract class ModuleInitMachine extends CommonInitMachine{
}
}
} catch (Exception e) {
return fail(e.getMessage());
return fail( ExceptionUtil.getCause(e));
}
return initLogger(()->{

View File

@@ -0,0 +1,12 @@
package org.example.util;
/**
* @author Genius
* @date 2023/10/17 01:51
**/
public class ExceptionUtil {
public static <T extends Exception> String getCause(T e){
return e.getMessage()!=null?e.getMessage():e.getCause()!=null?e.getCause().toString():e.getLocalizedMessage();
}
}

View File

@@ -32,12 +32,7 @@ public class HotController {
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/hotLive/live")
public Result getAllHotLive(@RequestParam(defaultValue = "0") int latest,@RequestParam String platform){
List<? extends Live> lives;
if(latest==1){
lives = HotModuleApi.getDouyuHotLive();
}else{
lives = HotModuleDataCenter.DataCenter().getLiveList(platform);
}
List<? extends Live> lives = hotModuleService.hotModuleApi().getHotLiveList(platform);
if(lives==null){
return Result.error("403","暂无该数据");
}
@@ -47,12 +42,7 @@ public class HotController {
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/hotLive/module")
public Result getAllHotModule(@RequestParam(defaultValue = "0") int latest,@RequestParam String platform){
HotModuleList hotModuleList;
if(latest==1){
hotModuleList = HotModuleApi.getDouyuAllHotModule();
}else{
hotModuleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
}
HotModuleList hotModuleList = hotModuleService.hotModuleApi().getAllHotModule(platform);
if(hotModuleList==null||hotModuleList.getHotModuleList()==null){
return Result.error("403","暂无该数据");
}
@@ -62,7 +52,7 @@ public class HotController {
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/hotLive/modelLive")
public Result getHotModuleLives(@RequestParam String moduleId,@RequestParam String platform){
HotModule moduleHotLives = hotModuleService.getModuleHotLives(platform, moduleId);
HotModule moduleHotLives = hotModuleService.hotModuleApi().getModuleList(platform, moduleId);
if(moduleHotLives==null||moduleHotLives.getHotLives()==null){
return Result.error("403","暂无该数据");
}

View File

@@ -11,6 +11,7 @@ import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.plugin.annotation.Plugin;
import org.example.util.ClassUtil;
import org.example.util.ExceptionUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
@@ -73,7 +74,7 @@ public class InitWorld implements CommandLineRunner {
close();
}
}catch (Exception e){
ChopperLogFactory.getLogger(LoggerType.System).error("Init Plugins Error:{}",e.getMessage());
ChopperLogFactory.getLogger(LoggerType.System).error("Init Plugins Error:{}",ExceptionUtil.getCause(e));
close();
}

View File

@@ -4,6 +4,7 @@ import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.thread.ChopperBotGuardPool;
import org.example.thread.oddjob.OddJobBoy;
import org.example.util.ExceptionUtil;
import org.example.util.PluginUtil;
import java.util.ArrayList;
@@ -50,7 +51,7 @@ public class WorldInitMachine extends ModuleInitMachine{
});
} catch (Exception e) {
return fail(e.getMessage());
return fail( ExceptionUtil.getCause(e));
}
}

View File

@@ -1,6 +1,7 @@
package org.example.service;
import org.example.api.HeatRecommendApi;
import org.example.api.HotModuleApi;
import org.example.api.HotModuleGuardApi;
import org.example.api.LiverFollowApi;
import org.example.bean.HotModule;
@@ -13,8 +14,8 @@ import java.util.List;
public interface HotModuleService {
HotModule getModuleHotLives(String platform, String moduleId);
HotModuleApi hotModuleApi();
LiverFollowApi liverFollowApi();
HotModuleGuardApi hotModuleGuardApi();

View File

@@ -1,6 +1,7 @@
package org.example.service.impl;
import org.example.api.HeatRecommendApi;
import org.example.api.HotModuleApi;
import org.example.api.HotModuleGuardApi;
import org.example.api.LiverFollowApi;
import org.example.bean.Live;
@@ -21,6 +22,8 @@ import java.util.List;
@Service
public class HotModuleServiceImpl implements HotModuleService {
@Resource
HotModuleApi hotModuleApi;
@Resource
LiverFollowApi liverFollowApi;
@@ -34,24 +37,10 @@ public class HotModuleServiceImpl implements HotModuleService {
* @param moduleId
* @return
*/
@Override
public HotModule getModuleHotLives(String platform,String moduleId) {
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
if(moduleList==null){
return null;
}
HotModule hotModule = moduleList.findHotModuleById(moduleId);
if(hotModule!=null){
try {
List<? extends Live> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
hotModule.setHotLives(moduleLiveList);
return hotModule;
}catch (Exception e){
//TODO 交给Spring全局异常处理器
return null;
}
}
return null;
public HotModuleApi hotModuleApi() {
return hotModuleApi;
}
@Override

View File

@@ -19,6 +19,12 @@
<dependencies>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

View File

@@ -2,6 +2,7 @@ package org.example.core.loadtask;
import org.example.bean.barrage.DouyuBarrage;
import org.example.core.loadconfig.LoadConfig;
import org.example.util.ExceptionUtil;
import org.example.ws.Draft_6455;
import org.java_websocket.client.WebSocketClient;
import org.java_websocket.handshake.ServerHandshake;
@@ -58,7 +59,7 @@ public abstract class WebSocketLoadTask<T> extends CommonLoadTask<T>{
client.setSocketFactory(sslSocketFactory);
client.connectBlocking(60, TimeUnit.SECONDS);
}catch (URISyntaxException | InterruptedException e){
this.logger.error("Websocket creeper error:{}",e.getMessage());
this.logger.error("Websocket creeper error:{}", ExceptionUtil.getCause(e));
return null;
}
if(client==null){

View File

@@ -17,7 +17,7 @@ import java.util.TreeSet;
* @date 2023/09/07 14:54
**/
@Component
public abstract class AbstractCreeperGroup implements Serializable {
public class AbstractCreeperGroup implements Serializable {
private final Set<CreeperMember> creeperMember;
@@ -51,9 +51,13 @@ public abstract class AbstractCreeperGroup implements Serializable {
return null;
}
public abstract String getPlatform();
public String getPlatform(){
return null;
}
public abstract String getFunctionName();
public String getFunctionName(){
return null;
}
public String getGroupName(){
return getPlatform()+"_"+getFunctionName();

View File

@@ -1,12 +1,14 @@
package org.example.core.manager;
import lombok.Data;
import org.example.constpool.ConstGroup;
import org.example.core.loadconfig.LoadConfig;
import org.example.util.ClassUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import java.io.Serializable;
import java.util.HashMap;
@@ -25,10 +27,25 @@ public class CreeperGroupCenter implements Serializable {
@Autowired
public CreeperGroupCenter(ApplicationContext context){
Map<String, AbstractCreeperGroup> beans = context.getBeansOfType(AbstractCreeperGroup.class);
beans.forEach((k,v)->{
groupMap.put(v.getGroupName(),v);
});
Set<Class<?>> creepers = ClassUtil.getAnnotationClass(PROJECT_PATH + ".core.creeper.loadconfig", Creeper.class);
for (Class<?> creeper : creepers) {
Creeper annotation = creeper.getAnnotation(Creeper.class);
String name = annotation.creeperName();
boolean discard = annotation.discard();
String platform = annotation.platform();
String groupName = StringUtils.hasText(platform)?CreeperGroupCenter.getGroupName(platform,annotation.group()):annotation.group();
int priority = annotation.priority();
AbstractCreeperGroup.CreeperMember member = new AbstractCreeperGroup.CreeperMember(priority, (Class<? extends LoadConfig>) creeper, discard, name);
if (CreeperGroupCenter.GroupMap().containsKey(groupName)) {
AbstractCreeperGroup group = CreeperGroupCenter.GroupMap().get(groupName);
group.addMember(member);
}else{
AbstractCreeperGroup group = new AbstractCreeperGroup();
groupMap.put(groupName,group);
group.addMember(member);
}
}
}
public static Map<String,AbstractCreeperGroup> GroupMap(){

View File

@@ -39,11 +39,6 @@ public class CreeperManager extends CommonPlugin {
String groupName = CreeperGroupCenter.getGroupName(annotation.platform(),annotation.group());
int priority = annotation.priority();
if (CreeperGroupCenter.GroupMap().containsKey(groupName)) {
AbstractCreeperGroup group = CreeperGroupCenter.GroupMap().get(groupName);
group.addMember(new AbstractCreeperGroup.CreeperMember(priority, (Class<? extends LoadConfig>) creeper,discard,name));
}
creeperBeans.add(new CreeperBean(name,description,author,groupName,discard,priority));
}
return true;

View File

@@ -12,6 +12,7 @@ import org.example.init.InitPluginRegister;
import org.example.core.taskcenter.request.ReptileRequest;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.util.ExceptionUtil;
import org.example.util.TimeUtil;
import java.io.Serializable;
@@ -50,8 +51,8 @@ public class ReptileTask implements Serializable {
}
public void reptile(){
TaskCenter plugin = InitPluginRegister.getPlugin(PluginName.TASK_CENTER_PLUGIN,TaskCenter.class);
try {
TaskCenter plugin = InitPluginRegister.getPlugin(PluginName.TASK_CENTER_PLUGIN,TaskCenter.class);
assert plugin != null;
//开始任务
this.startTime = TimeUtil.getNowTime_YMDHMS();
@@ -63,15 +64,17 @@ public class ReptileTask implements Serializable {
Object res = loadTask.start();
request.response(res); //让请求响应结果
}catch (Exception e){
ChopperLogFactory.getLogger(LoggerType.Creeper).info("[{}] {} stop, Error:{}",
PluginName.TASK_CENTER_PLUGIN,taskId, ExceptionUtil.getCause(e));
}finally {
assert plugin != null;
//完成任务
plugin.finishTask(taskId);
plugin.info(String.format("%s end reptile!", taskId));
this.type = TaskStatus.Finish;
this.endTime = TimeUtil.getNowTime_YMDHMS();
plugin.getTaskCenterLogger().setEndTime(taskId,endTime);
}catch (Exception e){
ChopperLogFactory.getLogger(LoggerType.Creeper).info("[{}] {} stop, Error:{}",
PluginName.TASK_CENTER_PLUGIN,taskId,e.getCause());
}
}

View File

@@ -6,6 +6,7 @@ import lombok.Data;
import org.example.core.taskcenter.task.serializer.TaskStatusEnumDeserializer;
import org.example.core.taskcenter.task.serializer.TaskStatusEnumSerializer;
import org.example.exception.taskcenter.TaskSerializationException;
import org.example.util.ExceptionUtil;
import java.io.ByteArrayInputStream;
@@ -51,7 +52,7 @@ public class TaskRecord {
oos.writeObject(reptileTask);
return baos.toByteArray();
}catch (Exception e){
throw new TaskSerializationException(reptileTask.getTaskId(),e.getMessage());
throw new TaskSerializationException(reptileTask.getTaskId(), ExceptionUtil.getCause(e));
}
}

View File

@@ -5,6 +5,7 @@ import org.example.constpool.PluginName;
import org.example.core.loadtask.LoadTask;
import org.example.plugin.CommonPlugin;
import org.example.thread.NamedThreadFactory;
import org.example.util.ExceptionUtil;
import java.util.HashMap;
import java.util.List;
@@ -52,7 +53,7 @@ public class MonitorCenter extends CommonPlugin {
this.info(String.format("%s no setting monitor", taskId));
}
}catch (Exception e){
this.error(String.format("Error:%s",e.getMessage()));
this.error(String.format("Error:%s", ExceptionUtil.getCause(e)));
}
}

View File

@@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSONObject;
import org.example.bean.ConfigFile;
import org.example.bean.FileType;
import org.example.exception.FileCacheException;
import org.example.util.ExceptionUtil;
import org.example.util.JsonFileUtil;
import org.example.util.TimeUtil;
import org.slf4j.Logger;
@@ -259,7 +260,7 @@ public class FileCache <T extends ConfigFile>{
try {
syncChannel.put(temp);
} catch (InterruptedException e) {
logger.error("自动刷入失败,Error:{}",e.getMessage());
logger.error("自动刷入失败,Error:{}", ExceptionUtil.getCause(e));
}
}

View File

@@ -7,6 +7,7 @@ import org.example.plugin.CommonPlugin;
import org.example.plugin.PluginCheckAndDo;
import org.example.plugin.annotation.Plugin;
import org.example.util.ConfigFileUtil;
import org.example.util.ExceptionUtil;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -40,7 +41,7 @@ public abstract class ConfigInitMachine extends CommonInitMachine{
return false;
}
}catch (Exception e){
return fail(e.getMessage());
return fail( ExceptionUtil.getCause(e));
}
}

View File

@@ -7,6 +7,7 @@ import org.example.constpool.ModuleName;
import org.example.constpool.PluginName;
import org.example.plugin.CommonPlugin;
import org.example.plugin.annotation.Plugin;
import org.example.util.ExceptionUtil;
import java.util.List;
@@ -43,7 +44,7 @@ public class FileCacheManagerInitMachine extends CommonInitMachine{
return fail();
}
}catch (Exception e){
return fail(e.getMessage());
return fail(ExceptionUtil.getCause(e));
}
}

View File

@@ -2,7 +2,12 @@ package org.example;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.TreeSet;
/**
@@ -11,7 +16,7 @@ import java.util.TreeSet;
*/
public class App
{
public static void main(String[] args) {
public static void main(String[] args) throws IOException {
}
}

View File

@@ -1,6 +1,7 @@
package org.example.api;
import org.example.bean.HotModule;
import org.example.bean.Live;
import org.example.bean.live.BiliBiliLive;
import org.example.bean.live.DouyuLive;
@@ -35,19 +36,23 @@ public class HotModuleApi {
return HotModuleDataCenter.DataCenter().getLiveList(platform);
}
public static HotModuleList getDouyuAllHotModule(){
return new DouyuHotModuleLoadTask(new DouyuHotModuleConfig()).start();
public HotModule getModuleList(String platform,String moduleId){
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
if(moduleList==null){
return null;
}
HotModule hotModule = moduleList.findHotModuleById(moduleId);
if(hotModule!=null){
try {
List<? extends Live> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
hotModule.setHotLives(moduleLiveList);
return hotModule;
}catch (Exception e){
//TODO 交给Spring全局异常处理器
return null;
}
}
return null;
}
public static List<DouyuLive> getDouyuHotLive(){
return new DouyuHotLiveLoadTask(new DouyuHotLiveConfig()).start();
}
public static List<DouyuLive> getDouyuHotLive(int moduleId){
return new DouyuHotLiveLoadTask(new DouyuHotLiveConfig(moduleId)).start();
}
public static List<BiliBiliLive> getBiliBiliHotLive(String parentId,String areaId,int page){
return new BiliBiliHotLiveLoadTask(new BilibiliHotLiveConfig(parentId,areaId,page)).start();
}
}

View File

@@ -30,4 +30,6 @@ public class FocusLiver implements Serializable {
private String platform;
private String tag;
private Boolean isAuto;
}

View File

@@ -10,6 +10,7 @@ import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.constpool.PluginName;
import org.example.core.creeper.loadtask.HotModuleLoadTask;
import org.example.core.loadtask.LoadTask;
import org.example.core.manager.CreeperGroupCenter;
import org.example.core.manager.CreeperManager;
@@ -25,7 +26,9 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -159,14 +162,15 @@ public class HotModuleDataCenter{
}
private List<? extends Live> checkPlatformAndGetList(String platform,HotModule hotModule){
List<? extends Live> lives = new ArrayList<>();
if(ConstPool.PLATFORM.DOUYU.getName().equals(platform)){
lives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId()));
}else if (ConstPool.PLATFORM.BILIBILI.getName().equals(platform)){
BilibiliHotModule temp = (BilibiliHotModule) hotModule;
lives = HotModuleApi.getBiliBiliHotLive(temp.getParent_id(),temp.getTagId(),1);
}
return lives;
SafeBag<List<? extends Live>> bag = new SafeBag<>();
Optional.ofNullable(InitPluginRegister.getPlugin(PluginName.CREEPER_MANAGER_PLUGIN, CreeperManager.class))
.ifPresent(
(plugin)->{
HotModuleLoadTask<List<? extends Live>> loadTask = plugin.getLoadTask(CreeperGroupCenter.getGroupName(platform, ConstGroup.HOT_LIVE), hotModule);
bag.setData((loadTask.start()));
}
);
return bag.getData();
}
//根据平台的模块名获取某模块

View File

@@ -0,0 +1,24 @@
package org.example.core.creeper.builder;
import org.example.api.HotModuleApi;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.BilibiliHotModule;
import org.example.core.creeper.loadconfig.BilibiliHotLiveConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/17 01:17
**/
@Component
public class BilibiliHotLiveBuilder extends CommonLoadConfigBuilder<BilibiliHotLiveConfig> {
@Override
public BilibiliHotLiveConfig build(Object obj) {
if(obj instanceof BilibiliHotModule){
return new BilibiliHotLiveConfig(((BilibiliHotModule) obj).getParent_id(),((BilibiliHotModule) obj).getAct_id(),1);
}
return null;
}
}

View File

@@ -0,0 +1,23 @@
package org.example.core.creeper.builder;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.DouyuHotModule;
import org.example.core.creeper.loadconfig.DouyuHotLiveConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/17 01:18
**/
@Component
public class DouyuHotLiveBuilder extends CommonLoadConfigBuilder<DouyuHotLiveConfig> {
@Override
public DouyuHotLiveConfig build(Object obj) {
if(obj instanceof DouyuHotModule){
return new DouyuHotLiveConfig(Integer.parseInt(((DouyuHotModule) obj).getTagId()));
}
return null;
}
}

View File

@@ -0,0 +1,23 @@
package org.example.core.creeper.builder;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HuyaHotModule;
import org.example.core.creeper.loadconfig.HuyaHotLiveConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/17 01:17
**/
@Component
public class HuyaHotLiveBuilder extends CommonLoadConfigBuilder<HuyaHotLiveConfig> {
@Override
public HuyaHotLiveConfig build(Object obj) {
if(obj instanceof HuyaHotModule){
return new HuyaHotLiveConfig(((HuyaHotModule) obj).getTagId());
}
return null;
}
}

View File

@@ -0,0 +1,21 @@
package org.example.core.creeper.builder;
import org.example.bean.live.HuyaLive;
import org.example.core.creeper.loadconfig.HuyaLiverCheckerConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/16 23:06
**/
@Component
public class HuyaLiverCheckerBuilder extends CommonLoadConfigBuilder<HuyaLiverCheckerConfig> {
@Override
public HuyaLiverCheckerConfig build(Object obj) {
if(obj instanceof HuyaLive){
return new HuyaLiverCheckerConfig(((HuyaLive) obj).getLiveId());
}
return null;
}
}

View File

@@ -1,25 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:11
**/
@Component
public class BilibiliHotLiveGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.PLATFORM.BILIBILI.getName();
}
@Override
public String getFunctionName() {
return ConstGroup.HOT_LIVE;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 16:52
**/
@Component
public class BilibiliHotModuleGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.PLATFORM.BILIBILI.getName();
}
@Override
public String getFunctionName() {
return ConstGroup.HOT_MODULE;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/10 01:02
**/
@Component
public class BilibiliLiverCheckerGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.BILIBILI;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVER_CHECKER;
}
}

View File

@@ -1,25 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:10
**/
@Component
public class DouyuHotLiveGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.PLATFORM.DOUYU.getName();
}
@Override
public String getFunctionName() {
return ConstGroup.HOT_LIVE;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 16:54
**/
@Component
public class DouyuHotModuleGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.PLATFORM.DOUYU.getName();
}
@Override
public String getFunctionName() {
return ConstGroup.HOT_MODULE;
}
}

View File

@@ -1,23 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/10 01:54
**/
@Component
public class DouyuLiverCheckerGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.DOUYU;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVER_CHECKER;
}
}

View File

@@ -2,10 +2,8 @@ package org.example.core.creeper.loadconfig;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.creeper.group.BilibiliHotModuleGroup;
import org.example.core.creeper.loadtask.BiliBiliHotLiveLoadTask;
import org.example.core.manager.Creeper;
import org.example.core.manager.CreeperGroupCenter;
/**
* @author Genius

View File

@@ -0,0 +1,33 @@
package org.example.core.creeper.loadconfig;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.creeper.loadtask.HuyaHotLiveLoadTask;
import org.example.core.creeper.loadtask.HuyaHotModuleLoadTask;
import org.example.core.manager.Creeper;
/**
* @author Genius
* @date 2023/10/17 00:47
**/
@Creeper(creeperName = "虎牙热门直播模块",
loadTask = HuyaHotLiveLoadTask.class,
creeperDescription = "获取虎牙的热门直播,按照人气排行,也可以获取热门模块下的直播",
priority = 10,
group = ConstGroup.HOT_LIVE,
platform = ConstPool.HUYA
)
public class HuyaHotLiveConfig extends LoadHotModuleConfig{
private String moduleId;
public HuyaHotLiveConfig(String moduleId) {
this.url = String.format("https://live.huya.com/liveHttpUI/getLiveList?iGid=%s&iPageNo=1&iPageSize=120", moduleId);
}
public HuyaHotLiveConfig() {
moduleId = "0";
this.url = String.format("https://live.huya.com/liveHttpUI/getLiveList?iGid=%s&iPageNo=1&iPageSize=120", moduleId);
}
}

View File

@@ -0,0 +1,26 @@
package org.example.core.creeper.loadconfig;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.creeper.loadtask.BilibiliHotModuleLoadTask;
import org.example.core.creeper.loadtask.HuyaHotModuleLoadTask;
import org.example.core.manager.Creeper;
/**
* @author Genius
* @date 2023/10/17 00:39
**/
@Creeper(creeperName = "虎牙热门模块",
loadTask = HuyaHotModuleLoadTask.class,
creeperDescription = "获取虎牙的热门模块,按照人气排行",
priority = 10,
group = ConstGroup.HOT_MODULE,
platform = ConstPool.HUYA
)
public class HuyaHotModuleConfig extends LoadHotModuleConfig{
public HuyaHotModuleConfig() {
this.url = "https://www.huya.com/g";
}
}

View File

@@ -0,0 +1,36 @@
package org.example.core.creeper.loadconfig;
import lombok.Data;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.creeper.loadtask.HuyaLiverCheckerLoadTask;
import org.example.core.loadconfig.LoadConfig;
import org.example.core.manager.Creeper;
/**
* @author Genius
* @date 2023/10/16 23:01
**/
@Data
@Creeper(creeperName = "虎牙直播检测爬虫",
loadTask = HuyaLiverCheckerLoadTask.class,
creeperDescription = "用于检测虎牙主播是否开播,并且获取直播详细信息",
priority = 10,
group = ConstGroup.LIVER_CHECKER,
platform = ConstPool.HUYA
)
public class HuyaLiverCheckerConfig extends LoadConfig {
private String roomId;
public HuyaLiverCheckerConfig(String roomId){
this.roomId = roomId;
this.url = String.format(String.format("https://search.cdn.huya.com/?m=Search&do=getSearchContent&q=%s&typ=-5&rows=16", roomId));
}
@Override
public String getTaskId() {
return super.getTaskId()+"_"+roomId;
}
}

View File

@@ -4,6 +4,7 @@ import org.example.core.loadtask.WebMagicLoadTask;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.core.loadconfig.LoadConfig;
import org.example.util.ExceptionUtil;
/**
* @author Genius
@@ -27,7 +28,7 @@ public abstract class HotModuleLoadTask<T> extends WebMagicLoadTask<T> {
protected void fail(Exception e){
finishFlag = FinishFlag.FAIL;
ChopperLogFactory.getLogger(LoggerType.Hot).error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage());
ChopperLogFactory.getLogger(LoggerType.Hot).error("loadTask{} finish fail Error:{}",this.getClass().getName(), ExceptionUtil.getCause(e));
}
protected void success(){

View File

@@ -0,0 +1,48 @@
package org.example.core.creeper.loadtask;
import org.example.bean.live.BiliBiliLive;
import org.example.bean.live.HuyaLive;
import org.example.constpool.ConstPool;
import org.example.core.creeper.loadconfig.HuyaHotLiveConfig;
import org.example.core.creeper.processor.BiliBiliHotLiveProcessor;
import org.example.core.creeper.processor.HuyaHotLiveProcessor;
import org.example.core.factory.SpiderFactory;
import org.example.core.loadconfig.LoadConfig;
import org.example.core.loadtask.WebMagicLoadTask;
import us.codecraft.webmagic.Spider;
import java.util.List;
/**
* @author Genius
* @date 2023/10/17 00:50
**/
public class HuyaHotLiveLoadTask extends HotModuleLoadTask<List<HuyaLive>> {
public HuyaHotLiveLoadTask(HuyaHotLiveConfig loadConfig) {
super(loadConfig);
}
@Override
public List<HuyaLive> start() {
clearFinishFlag();
List<HuyaLive> lives;
HuyaHotLiveProcessor huyaHotLiveProcessor = new HuyaHotLiveProcessor();
Spider spider = SpiderFactory.buildSpider(ConstPool.PLATFORM.HUYA.getName(),
huyaHotLiveProcessor,
loadConfig.getUrl());
try {
lives = getData(spider,loadConfig.getUrl());
}catch (Exception e){
fail(e);
return null;
}
success();
return lives;
}
@Override
public void end() {
}
}

View File

@@ -0,0 +1,47 @@
package org.example.core.creeper.loadtask;
import org.example.bean.hotmodule.HotModuleList;
import org.example.bean.hotmodule.HuyaHotModule;
import org.example.core.creeper.loadconfig.HuyaHotModuleConfig;
import org.example.core.loadconfig.LoadConfig;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/10/17 00:38
**/
public class HuyaHotModuleLoadTask extends HotModuleLoadTask<HotModuleList>{
public HuyaHotModuleLoadTask(HuyaHotModuleConfig loadConfig) {
super(loadConfig);
}
@Override
public HotModuleList start() {
HotModuleList data = null;
clearFinishFlag();
try {
Document doc = Jsoup.connect(loadConfig.getUrl()).get();
Element ul = doc.selectFirst("ul#js-game-list");
if (ul != null) {
Elements liElements = ul.select("li"); // 获取ul下的所有li元素
data = new HotModuleList();
for (Element li : liElements) {
String dataGid = li.attr("data-gid");
String gName = li.text();
data.getHotModuleList().add(new HuyaHotModule(dataGid, gName));
}
}
}catch (Exception e){
fail(e);
}
success();
return data;
}
}

View File

@@ -0,0 +1,45 @@
package org.example.core.creeper.loadtask;
import org.example.bean.live.DouyuLive;
import org.example.bean.live.HuyaLive;
import org.example.constpool.ConstPool;
import org.example.core.creeper.loadconfig.HuyaLiverCheckerConfig;
import org.example.core.creeper.processor.DouyuLiverCheckerProcessor;
import org.example.core.creeper.processor.HuyaLiverCheckerProcessor;
import org.example.core.factory.SpiderFactory;
import org.example.core.loadconfig.LoadConfig;
import org.example.core.loadtask.WebMagicLoadTask;
import us.codecraft.webmagic.Spider;
/**
* @author Genius
* @date 2023/10/16 23:01
**/
public class HuyaLiverCheckerLoadTask extends WebMagicLoadTask<HuyaLive> {
public HuyaLiverCheckerLoadTask(HuyaLiverCheckerConfig loadConfig) {
super(loadConfig);
}
@Override
public HuyaLive start() {
HuyaLive live = null;
Spider spider = SpiderFactory.buildSpider(
ConstPool.PLATFORM.HUYA.getName(),
new HuyaLiverCheckerProcessor(),
loadConfig.getUrl()
);
try {
live = getData(spider,loadConfig.getUrl());
}catch (Exception e){
return null;
}
return live;
}
@Override
public void end() {
}
}

View File

@@ -0,0 +1,54 @@
package org.example.core.creeper.processor;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.live.BiliBiliLive;
import org.example.bean.live.HuyaLive;
import org.example.constpool.ConstPool;
import org.example.core.processor.AbstractProcessor;
import us.codecraft.webmagic.Page;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/10/17 00:52
**/
public class HuyaHotLiveProcessor extends AbstractProcessor {
@Override
public void process(Page page) {
List<HuyaLive> liveList = new ArrayList<>();
try {
JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("vList");
for (Object live : Lives) {
if(live instanceof JSONObject){
JSONObject jsonLive = (JSONObject) live;
String lUid = jsonLive.getString("lUid");
String nickName = jsonLive.getString("sNick");
Integer watchNum = jsonLive.getInteger("lActivityCount");
String moduleId = jsonLive.getString("iGid");
String moduleName = jsonLive.getString("sGameFullName");
String introduction = jsonLive.getString("sIntroduction");
String roomId = jsonLive.getString("lProfileRoom");
String pic = jsonLive.getString("sScreenshot");
HuyaLive huyaLive = new HuyaLive(lUid);
huyaLive.setLiveName(introduction);
huyaLive.setPlatform(ConstPool.HUYA);
huyaLive.setDescription(introduction);
huyaLive.setLiver(nickName);
huyaLive.setWatcherNum(watchNum);
huyaLive.setModuleId(moduleId);
huyaLive.setModuleName(moduleName);
huyaLive.setLiveId(roomId);
huyaLive.setRoomPic(pic);
liveList.add(huyaLive);
}
}
}catch (Exception e){
throw e;
}
page.putField("data",liveList);
}
}

View File

@@ -0,0 +1,55 @@
package org.example.core.creeper.processor;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.example.bean.live.HuyaLive;
import org.example.constpool.ConstPool;
import org.example.core.processor.AbstractProcessor;
import org.example.util.TimeUtil;
import us.codecraft.webmagic.Page;
import java.util.Optional;
/**
* @author Genius
* @date 2023/10/16 23:37
**/
public class HuyaLiverCheckerProcessor extends AbstractProcessor {
@Override
public void process(Page page) {
JSONObject live = JSON.parseObject(page.getRawText()).getJSONObject("response").getJSONObject("1");
if(live!=null){
JSONArray array = Optional.ofNullable(live.getJSONArray("docs")).orElse(new JSONArray());
for (Object doc : array) {
if(doc instanceof JSONObject){
if (Boolean.TRUE.equals(((JSONObject) doc).getBoolean("gameLiveOn"))) {
String uid = ((JSONObject) doc).getString("uid");
Integer activityCount = ((JSONObject) doc).getInteger("game_activityCount");
String avatar = ((JSONObject) doc).getString("game_avatarUrl180");
String moduleName = ((JSONObject) doc).getString("game_name");
Integer moduleId = ((JSONObject) doc).getInteger("game_id");
String roomId = ((JSONObject) doc).getString("room_id");
String liver = ((JSONObject) doc).getString("game_nick");
String liveName = ((JSONObject) doc).getString("live_intro");
Long showTime = ((JSONObject) doc).getLong("rec_live_time") * 1000;
HuyaLive huyaLive = new HuyaLive(uid);
huyaLive.setWatcherNum(activityCount);
huyaLive.setRoomPic(avatar);
huyaLive.setModuleName(moduleName);
huyaLive.setModuleId(moduleId.toString());
huyaLive.setLiveId(roomId);
huyaLive.setLiver(liver);
huyaLive.setLiveName(liveName);
huyaLive.setDescription(liveName);
huyaLive.setShowTime(TimeUtil.getFormatDate(showTime));
huyaLive.setPlatform(ConstPool.HUYA);
page.putField("data",huyaLive);
return;
}
}
}
}
}
}

View File

@@ -1,6 +1,8 @@
package org.example.core.component;
import lombok.Data;
import org.example.constpool.PluginName;
import org.example.log.ChopperLogFactory;
import java.io.InputStream;
import java.io.OutputStream;
@@ -33,6 +35,7 @@ public class LiveStreamTask {
f.parseStream(in, taskId, fileIO);
}
} catch (Exception e) {
ChopperLogFactory.getLogger(PluginName.LIVE_MANAGER_PLUGIN).error("LiveStreamTask start error:", e);
return;
}
}

View File

@@ -3,6 +3,7 @@ package org.example.core.component;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.pool.LiveModuleConstPool;
import org.example.util.ExceptionUtil;
import org.slf4j.Logger;
import java.io.BufferedReader;
@@ -50,7 +51,7 @@ public class M3U8Handle {
int exitCode = process.waitFor();
logger.error("Exit code: {}",exitCode);
} catch (IOException | InterruptedException e) {
logger.error("Error during video processing: {}",e.getMessage());
logger.error("Error during video processing: {}", ExceptionUtil.getCause(e));
e.printStackTrace();
}
}

View File

@@ -0,0 +1,36 @@
package org.example.core.creeper.builder;
import org.example.bean.live.DouyuLive;
import org.example.bean.live.HuyaLive;
import org.example.constpool.ConstPool;
import org.example.constpool.FileNameBuilder;
import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
import org.example.core.manager.CommonLoadConfigBuilder;
import org.example.pool.LiveModuleConstPool;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/10/16 22:10
**/
@Component
public class HuyaLiveLoadConfigBuilder extends CommonLoadConfigBuilder<HuyaLiveOnlineConfig> {
@Override
public HuyaLiveOnlineConfig build(Object obj) {
if(obj instanceof HuyaLive){
String liveId = ((HuyaLive) obj).getLiveId();
String liver = ((HuyaLive) obj).getLiver();
String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.HUYA);
String showTime = ((HuyaLive) obj).getShowTime();
HuyaLiveOnlineConfig huyaLiveOnlineConfig = new HuyaLiveOnlineConfig(liveId, path, null, false);
huyaLiveOnlineConfig.setShowTime(showTime);
huyaLiveOnlineConfig.setVideoName(FileNameBuilder.buildVideoFileNameNoSuffix(liver,huyaLiveOnlineConfig.getStartTime()));
huyaLiveOnlineConfig.setShowDownloadTable(true);
huyaLiveOnlineConfig.setLiverName(liver);
huyaLiveOnlineConfig.setRoomName(((HuyaLive) obj).getLiveName());
return huyaLiveOnlineConfig;
}
return null;
}
}

View File

@@ -1,24 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:31
**/
@Component
public class BiliBiliLiveGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.BILIBILI;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVE_ONLINE;
}
}

View File

@@ -1,23 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:32
**/
@Component
public class DouyuLiveGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.DOUYU;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVE_ONLINE;
}
}

View File

@@ -1,23 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
import org.springframework.stereotype.Component;
/**
* @author Genius
* @date 2023/09/07 17:33
**/
@Component
public class DouyuRecordGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.DOUYU;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVE_RECORD;
}
}

View File

@@ -1,21 +0,0 @@
package org.example.core.creeper.group;
import org.example.constpool.ConstGroup;
import org.example.constpool.ConstPool;
import org.example.core.manager.AbstractCreeperGroup;
/**
* @author dhx
* @date 2023/10/10 10:53
*/
public class HuyaLiveGroup extends AbstractCreeperGroup {
@Override
public String getPlatform() {
return ConstPool.HUYA;
}
@Override
public String getFunctionName() {
return ConstGroup.LIVE_ONLINE;
}
}

View File

@@ -8,6 +8,7 @@ import org.example.core.parser.impl.DouyuFlvUrlParser;
import org.example.core.parser.impl.DouyuM3u8UrlParser;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.util.ExceptionUtil;
import org.slf4j.Logger;
/**
@@ -37,7 +38,7 @@ public class DouyuRecordLoadTask extends CommonLoadTask<String> {
douyuRecordConfig.getVideoPath(),
douyuRecordConfig.getVideoName());
}catch (Exception e){
logger.info("{} 录播爬取失败,error:{}",douyuRecordConfig.getVideoName(),e.getMessage());
logger.info("{} 录播爬取失败,error:{}",douyuRecordConfig.getVideoName(), ExceptionUtil.getCause(e));
}
return url;
}

View File

@@ -2,11 +2,13 @@ package org.example.core.factory;
import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig;
import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
import org.example.core.creeper.loadconfig.LoadLiveConfig;
import org.example.core.parser.PlatformVideoUrlParser;
import org.example.core.parser.impl.BilibiliFlvUrlParser;
import org.example.core.component.LiveStreamTask;
import org.example.core.parser.impl.DouyuFlvUrlParser;
import org.example.core.parser.impl.HuyaFlvUrlParser;
import java.util.HashMap;
import java.util.Map;
@@ -20,7 +22,8 @@ public class LiveTaskFactory {
private final Map<Class<? extends LoadLiveConfig>, PlatformVideoUrlParser> parserMap = Map.of(
BilibiliLiveOnlineConfig.class,new BilibiliFlvUrlParser(),
DouyuLiveOnlineConfig.class, new DouyuFlvUrlParser()
DouyuLiveOnlineConfig.class, new DouyuFlvUrlParser(),
HuyaLiveOnlineConfig.class, new HuyaFlvUrlParser()
);
public LiveTaskFactory() {

View File

@@ -7,6 +7,7 @@ import org.example.constpool.PluginName;
import org.example.plugin.CommonPlugin;
import org.example.plugin.annotation.Plugin;
import org.example.pool.LiveModuleConstPool;
import org.example.util.ExceptionUtil;
import org.example.util.FileUtil;
import java.nio.file.Files;
@@ -56,7 +57,7 @@ public class LiveConfigInitMachine extends ConfigInitMachine{
}
}
}catch (Exception e){
return fail(e.getMessage());
return fail( ExceptionUtil.getCause(e));
}
return super.init();
}

View File

@@ -299,7 +299,7 @@ public class VideoUtil {
}
logger.info("转码完成,视频帧:{},音频帧:{},数据帧:{}", videoFrameNum, audioFrameNum, dataFrameNum);
} catch (Exception e) {
logger.error(e.getMessage());
logger.error( ExceptionUtil.getCause(e));
return "error";
} finally {
if (recorder != null) {
@@ -394,7 +394,7 @@ public class VideoUtil {
i++;
}
} catch (Exception e) {
logger.error(e.getMessage());
logger.error( ExceptionUtil.getCause(e));
return "error";
}finally {
if (recorder != null) {

View File

@@ -21,7 +21,7 @@ import java.util.List;
**/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ConsoleApplication.class)
@SpringBootTest(classes = ConsoleApplication.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
public class BarrageCreeperTest {
@Test
@@ -33,7 +33,7 @@ public class BarrageCreeperTest {
@Test
public void testHuyaBarrageCreeper(){
HuyaLiveBarrageLoadConfig config = new HuyaLiveBarrageLoadConfig("123","lwz6038");
HuyaLiveBarrageLoadConfig config = new HuyaLiveBarrageLoadConfig("123", "243547");
try {
List<HuyaBarrage> start = new HuyaLiveBarrageLoadTask(config).start();
} catch (FileCacheException e) {

View File

@@ -53,29 +53,29 @@ public class BarrageCurveTest {
@Test
public void testCurve(){
long l = System.currentTimeMillis();
List<BarragePoint> list = plugin.generateCurve(new BarrageEvent("bilibili", "online", "Asaki大人", "2023-09-19 22_00_06"));
System.out.println(System.currentTimeMillis()-l);
LineChartDemo demo = new LineChartDemo("Asaki大人",list);
demo.pack();
RefineryUtilities.centerFrameOnScreen(demo);
demo.setVisible(true);
List<PopularRange> range = popularRangePlugin.findRange(list);
System.out.println(range);
int i = 0;
for (PopularRange popularRange : range) {
long timeNaos = TimeUtil.getTimeNaos("2023-09-19 22:00:06");
long videoStartTime = popularRange.getStartTime()-timeNaos;
videoStartTime = videoStartTime<0?0:videoStartTime;
long videoEndTime = popularRange.getEndTime()-timeNaos;
System.out.println(videoStartTime+" "+videoEndTime);
VideoUtil.cutVideoByFFMpeg("E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06.flv",
"E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06"+i+".flv",
videoStartTime/1000,videoEndTime/1000);
i++;
}
while (true){
}
// List<BarragePoint> list = plugin.generateCurve(new BarrageEvent("bilibili", "online", "Asaki大人", "2023-09-19 22_00_06"));
// System.out.println(System.currentTimeMillis()-l);
// LineChartDemo demo = new LineChartDemo("Asaki大人",list);
// demo.pack();
// RefineryUtilities.centerFrameOnScreen(demo);
// demo.setVisible(true);
// List<PopularRange> range = popularRangePlugin.findRange(list);
// System.out.println(range);
// int i = 0;
// for (PopularRange popularRange : range) {
// long timeNaos = TimeUtil.getTimeNaos("2023-09-19 22:00:06");
// long videoStartTime = popularRange.getStartTime()-timeNaos;
// videoStartTime = videoStartTime<0?0:videoStartTime;
// long videoEndTime = popularRange.getEndTime()-timeNaos;
// System.out.println(videoStartTime+" "+videoEndTime);
// VideoUtil.cutVideoByFFMpeg("E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06.flv",
// "E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06"+i+".flv",
// videoStartTime/1000,videoEndTime/1000);
// i++;
// }
// while (true){
//
// }
}
class LineChartDemo extends ApplicationFrame {

View File

@@ -26,10 +26,10 @@ public class BarragePoupularRangeTest {
@Test
public void Test(){
BarrageEvent event = new BarrageEvent("douyu","online",null,null,"即将拥有人鱼线的PDD_2023-09-14 14_37_54.json");
List<BarragePoint> points = InitPluginRegister.getPlugin(PluginName.BARRAGE_SCORE_CURVE_PLUGIN, BarrageScoreCurvePlugin.class).generateCurve(event);
List<PopularRange> ranges = InitPluginRegister.getPlugin(PluginName.BARRAGE_POPULAR_RANGE_PLUGIN, BarragePopularRangePlugin.class).findRange(points);
System.out.println(ranges);
// BarrageEvent event = new BarrageEvent("douyu","online",null,null,"即将拥有人鱼线的PDD_2023-09-14 14_37_54.json");
// List<BarragePoint> points = InitPluginRegister.getPlugin(PluginName.BARRAGE_SCORE_CURVE_PLUGIN, BarrageScoreCurvePlugin.class).generateCurve(event);
// List<PopularRange> ranges = InitPluginRegister.getPlugin(PluginName.BARRAGE_POPULAR_RANGE_PLUGIN, BarragePopularRangePlugin.class).findRange(points);
// System.out.println(ranges);
}
}

View File

@@ -32,5 +32,5 @@
"CreeperConfig":true
}
},
"updateTime":"2023-10-16 17:13:50"
"updateTime":"2023-10-17 02:15:29"
}

Binary file not shown.