mirror of
https://github.com/Geniusay/ChopperBot.git
synced 2026-06-05 22:29:28 +08:00
虎牙平台爬虫 融入ChopperBot #33
This commit is contained in:
2
.idea/compiler.xml
generated
2
.idea/compiler.xml
generated
@@ -38,7 +38,7 @@
|
||||
</component>
|
||||
<component name="JavacSettings">
|
||||
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
|
||||
<module name="ChopperBot" options="-parameters" />
|
||||
<module name="ChopperBot" options="" />
|
||||
<module name="chopperbot-account" options="-parameters" />
|
||||
<module name="chopperbot-barrage" options="-parameters" />
|
||||
<module name="chopperbot-common" options="-parameters" />
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.example.api;
|
||||
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.constpool.GlobalFileCache;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
@@ -45,7 +46,7 @@ public class VideoApi {
|
||||
return false;
|
||||
}
|
||||
}catch (SecurityException e) {
|
||||
System.out.println("删除文件时发生安全异常: " + e.getMessage());
|
||||
System.out.println("删除文件时发生安全异常: " + ExceptionUtil.getCause(e));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import org.example.core.bgevnet.BarrageEventCenter;
|
||||
import org.example.core.bgevnet.instantslicing.InstantSlicingPlugin;
|
||||
import org.example.core.section.SectionRequest;
|
||||
import org.example.core.section.VideoSectionWorkShop;
|
||||
import org.example.core.taskcenter.TaskCenter;
|
||||
import org.example.core.taskcenter.observer.AbstractTaskCenterObserver;
|
||||
import org.example.core.taskcenter.task.ReptileTask;
|
||||
import org.example.init.InitPluginRegister;
|
||||
@@ -36,6 +37,7 @@ public class LiveAndBarrageHandlerObserver extends AbstractTaskCenterObserver {
|
||||
|
||||
@Override
|
||||
public void onFinish(ReptileTask task) {
|
||||
taskCenter.stopTask(task.getTaskId().replace("live","barrage"));
|
||||
if(!InitPluginRegister.isRegister(PluginName.INSTANT_SLICING_PLUGIN)){
|
||||
PluginCheckAndDo.CheckAndDo((plugin)->{
|
||||
Object param = task.getRequest().getParam();
|
||||
|
||||
@@ -15,9 +15,7 @@ import org.example.init.InitPluginRegister;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.thread.NamedThreadFactory;
|
||||
import org.example.util.BarrageUtil;
|
||||
import org.example.util.TimeUtil;
|
||||
import org.example.util.VideoUtil;
|
||||
import org.example.util.*;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
@@ -103,18 +101,24 @@ public class ScheduleTimeHandler implements InstantSlicingHandler {
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
ChopperLogFactory.getLogger(LoggerType.Barrage).error("Error:{}",e.toString());
|
||||
ChopperLogFactory.getLogger(LoggerType.Barrage).error("Error:{}", ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean splitBarrageFile(String oldPath,String newPath,long startTime,long endTime){
|
||||
return BarrageUtil.cutBarrageFile(oldPath,newPath,startTime,endTime);
|
||||
if(FileUtil.isFileExist(oldPath)){
|
||||
return BarrageUtil.cutBarrageFile(oldPath,newPath,startTime,endTime);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean spiltVideoFile(String oldPath,String newPath,long startSecond,long endSecond){
|
||||
return VideoUtil.cutVideoByFFMpeg(oldPath,newPath,startSecond,endSecond);
|
||||
if(FileUtil.isFileExist(oldPath)){
|
||||
return VideoUtil.cutVideoByFFMpeg(oldPath,newPath,startSecond,endSecond);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.bean.live.DouyuLive;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.core.creeper.loadconfig.DouyuLiveBarrageLoadConfig;
|
||||
import org.example.core.creeper.loadconfig.HuyaLiveBarrageLoadConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 22:03
|
||||
**/
|
||||
|
||||
@Component
|
||||
public class HuyaLiveBarrageLoadConfigBuilder extends CommonLoadConfigBuilder<HuyaLiveBarrageLoadConfig> {
|
||||
@Override
|
||||
public HuyaLiveBarrageLoadConfig build(Object obj) {
|
||||
if(obj instanceof HuyaLive){
|
||||
return new HuyaLiveBarrageLoadConfig(((HuyaLive) obj).getLiver(),((HuyaLive) obj).getLiveId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:22
|
||||
**/
|
||||
@Component
|
||||
public class BilibiliLiveBarrageGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.BILIBILI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.BARRAGE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/12 23:02
|
||||
**/
|
||||
@Component
|
||||
public class DouyuLiveBarrageGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.DOUYU;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.BARRAGE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.example.ws.handler.AbstractMessageHandler;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:28
|
||||
**/
|
||||
@Component
|
||||
public class DouyuRecordBarrageGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.DOUYU;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.BARRAGE_RECORD;
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
|
||||
/**
|
||||
* @author dhx
|
||||
* @date 2023/10/16 15:59
|
||||
*/
|
||||
public class HuyaLiveBarrageGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.HUYA;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.BARRAGE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ public class HuyaLiveBarrageLoadConfig extends LoadBarrageConfig{
|
||||
private String roomId;
|
||||
|
||||
public HuyaLiveBarrageLoadConfig(String anchorName,String roomId) {
|
||||
super(ConstPool.DOUYU, CreeperModuleConstPool.ACTION_LIVE, anchorName);
|
||||
super(ConstPool.HUYA, CreeperModuleConstPool.ACTION_LIVE, anchorName);
|
||||
this.roomId = roomId;
|
||||
this.url = "wss://cdnws.api.huya.com/";
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import org.example.core.taskmonitor.Monitor;
|
||||
import org.example.core.taskmonitor.MonitorCenter;
|
||||
import org.example.exception.FileCacheException;
|
||||
import org.example.plugin.PluginCheckAndDo;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.java_websocket.handshake.ServerHandshake;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
@@ -113,7 +114,7 @@ public class DouyuLiveBarrageLoadTask extends WebSocketLoadTask<List<DouyuBarrag
|
||||
@Override
|
||||
public void onError(Exception e) {
|
||||
client.close();
|
||||
this.logger.error("Douyu Live Barrage Error:{}",e.getMessage());
|
||||
this.logger.error("Douyu Live Barrage Error:{}", ExceptionUtil.getCause(e));
|
||||
}
|
||||
|
||||
public DouyuBarrage wrapperBarrage(ByteBuffer buffer, long startTime){
|
||||
|
||||
@@ -17,6 +17,7 @@ import org.example.core.taskmonitor.Monitor;
|
||||
import org.example.core.taskmonitor.MonitorCenter;
|
||||
import org.example.exception.FileCacheException;
|
||||
import org.example.plugin.PluginCheckAndDo;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.java_websocket.handshake.ServerHandshake;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
@@ -46,14 +47,6 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
|
||||
fileCache = new BarrageFileCache(new BarrageSaveFile<>(loadConfig,new ConcurrentLinkedQueue<>()));
|
||||
}
|
||||
|
||||
public static String stringToHex(String input) {
|
||||
StringBuilder hexString = new StringBuilder();
|
||||
for (char c : input.toCharArray()) {
|
||||
hexString.append(String.format("%02X ", (int) c));
|
||||
}
|
||||
return hexString.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HuyaBarrage> handler() {
|
||||
return list;
|
||||
@@ -100,7 +93,15 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
|
||||
|
||||
@Override
|
||||
public void onMessage(ByteBuffer buffer) {
|
||||
HuyaBarrage huyaBarrage = wrapperBarrage(buffer,0);
|
||||
HuyaBarrage huyaBarrage = wrapperBarrage(buffer);
|
||||
if(huyaBarrage!=null){
|
||||
list.add(huyaBarrage);
|
||||
try {
|
||||
fileCache.append(huyaBarrage,"-1");
|
||||
}catch (FileCacheException | InterruptedException e){
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -111,13 +112,13 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
|
||||
@Override
|
||||
public void onError(Exception e) {
|
||||
client.close();
|
||||
this.logger.error("Huya Live Barrage Error:{}",e.getMessage());
|
||||
this.logger.error("Huya Live Barrage Error:{}", ExceptionUtil.getCause(e));
|
||||
}
|
||||
|
||||
public HuyaBarrage wrapperBarrage(ByteBuffer buffer, long startTime){
|
||||
String name = "";
|
||||
public HuyaBarrage wrapperBarrage(ByteBuffer buffer){
|
||||
String content = "";
|
||||
TarsInputStream tarsInputStream = new TarsInputStream(buffer);
|
||||
long nowTime = System.currentTimeMillis();
|
||||
int int32 = 0;
|
||||
long int64 = 0L;
|
||||
byte[] bytes = null;
|
||||
@@ -125,18 +126,13 @@ public class HuyaLiveBarrageLoadTask extends WebSocketLoadTask<List<HuyaBarrage>
|
||||
tarsInputStream = new TarsInputStream(tarsInputStream.read(bytes,1,false));
|
||||
if(tarsInputStream.read(int64,1,false)==1400){
|
||||
tarsInputStream = new TarsInputStream(tarsInputStream.read(bytes,2,false));
|
||||
// name = tarsInputStream.read("",0,false);
|
||||
tarsInputStream.setServerEncoding("utf-8");
|
||||
content = tarsInputStream.read("", 3, true);
|
||||
try {
|
||||
System.out.println(new String(content.getBytes("GBK")));
|
||||
System.out.println(stringToHex(content));
|
||||
System.out.println(new String(content.replaceAll("\uFFFD", "").getBytes("GBK")));
|
||||
System.out.println(stringToHex(new String(content.getBytes("GBK"))));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return new HuyaBarrage("",nowTime,nowTime-startTime,content);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
System.out.println(name);
|
||||
|
||||
}
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -75,16 +75,11 @@ public class HuyaProtocol {
|
||||
regDatas = wscmd.toByteArray();
|
||||
return regDatas;
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("解析失败");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public static byte[] getHeartbeat(){
|
||||
return heartbeat;
|
||||
}
|
||||
public static void main(String[] args) throws IOException {
|
||||
String roomId = "dank1ng";
|
||||
System.out.println(HexUtil.encodeHex(getWsInfo(roomId)));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import java.time.LocalDateTime;
|
||||
*/
|
||||
@Data
|
||||
public abstract class Live implements Serializable {
|
||||
|
||||
private int watcherNum; //直播间观众数目
|
||||
private String liveId; //直播间ID
|
||||
private String liveName; //直播间名字
|
||||
@@ -33,6 +34,9 @@ public abstract class Live implements Serializable {
|
||||
|
||||
private String roomPic;
|
||||
|
||||
public Live() {
|
||||
}
|
||||
|
||||
public Live(int watcherNum, String liveId, String liveName, String description) {
|
||||
this.watcherNum = watcherNum;
|
||||
this.liveId = liveId;
|
||||
@@ -68,4 +72,5 @@ public abstract class Live implements Serializable {
|
||||
this.moduleName = moduleName;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package org.example.bean.hotmodule;
|
||||
|
||||
import org.example.bean.HotModule;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:42
|
||||
**/
|
||||
public class HuyaHotModule extends HotModule<HuyaLive> {
|
||||
|
||||
public HuyaHotModule(String tagId, String tagName) {
|
||||
super(tagId, tagName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package org.example.bean.live;
|
||||
|
||||
import lombok.Data;
|
||||
import org.example.bean.Live;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 22:04
|
||||
**/
|
||||
@Data
|
||||
public class HuyaLive extends Live {
|
||||
|
||||
private String uid;
|
||||
|
||||
public HuyaLive(String uid) {
|
||||
this.uid = uid;
|
||||
}
|
||||
|
||||
public HuyaLive(int watcherNum, String liveId, String liveName, String description) {
|
||||
super(watcherNum, liveId, liveName, description);
|
||||
}
|
||||
|
||||
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description) {
|
||||
super(watcherNum, liveId, liveName, liver, description);
|
||||
}
|
||||
|
||||
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description, String platform) {
|
||||
super(watcherNum, liveId, liveName, liver, description, platform);
|
||||
}
|
||||
|
||||
public HuyaLive(int watcherNum, String liveId, String liveName, String liver, String description, String platform, String moduleId, String moduleName) {
|
||||
super(watcherNum, liveId, liveName, liver, description, platform, moduleId, moduleName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.example.core;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 01:30
|
||||
**/
|
||||
public class SafeBag<T> {
|
||||
T data;
|
||||
|
||||
public SafeBag(T data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public SafeBag() {
|
||||
}
|
||||
|
||||
public T getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(T data){
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package org.example.init;
|
||||
import org.example.log.ResultLogger;
|
||||
import org.example.plugin.CommonPlugin;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -89,7 +90,7 @@ public abstract class CommonInitMachine implements ComponentInitMachine, ResultL
|
||||
return fail();
|
||||
}
|
||||
}catch (Exception e){
|
||||
return fail(e.getMessage());
|
||||
return fail(ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import org.example.plugin.CommonPlugin;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.thread.ChopperBotGuardPool;
|
||||
import org.example.util.ClassUtil;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -161,7 +162,7 @@ public class InitPluginRegister {
|
||||
}
|
||||
}catch (Exception e){
|
||||
ChopperLogFactory.getLogger(LoggerType.System)
|
||||
.error(String.format("❌ %s plugin start failed ! error:%s", pluginName,e.getMessage()));
|
||||
.error(String.format("❌ %s plugin start failed ! error:%s", pluginName, ExceptionUtil.getCause(e)));
|
||||
}
|
||||
}else{
|
||||
return false;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package org.example.init;
|
||||
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.util.PluginUtil;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
@@ -50,7 +51,7 @@ public abstract class ModuleInitMachine extends CommonInitMachine{
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return fail(e.getMessage());
|
||||
return fail( ExceptionUtil.getCause(e));
|
||||
}
|
||||
|
||||
return initLogger(()->{
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package org.example.util;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 01:51
|
||||
**/
|
||||
public class ExceptionUtil {
|
||||
|
||||
public static <T extends Exception> String getCause(T e){
|
||||
return e.getMessage()!=null?e.getMessage():e.getCause()!=null?e.getCause().toString():e.getLocalizedMessage();
|
||||
}
|
||||
}
|
||||
@@ -32,12 +32,7 @@ public class HotController {
|
||||
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
|
||||
@GetMapping("/hotLive/live")
|
||||
public Result getAllHotLive(@RequestParam(defaultValue = "0") int latest,@RequestParam String platform){
|
||||
List<? extends Live> lives;
|
||||
if(latest==1){
|
||||
lives = HotModuleApi.getDouyuHotLive();
|
||||
}else{
|
||||
lives = HotModuleDataCenter.DataCenter().getLiveList(platform);
|
||||
}
|
||||
List<? extends Live> lives = hotModuleService.hotModuleApi().getHotLiveList(platform);
|
||||
if(lives==null){
|
||||
return Result.error("403","暂无该数据");
|
||||
}
|
||||
@@ -47,12 +42,7 @@ public class HotController {
|
||||
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
|
||||
@GetMapping("/hotLive/module")
|
||||
public Result getAllHotModule(@RequestParam(defaultValue = "0") int latest,@RequestParam String platform){
|
||||
HotModuleList hotModuleList;
|
||||
if(latest==1){
|
||||
hotModuleList = HotModuleApi.getDouyuAllHotModule();
|
||||
}else{
|
||||
hotModuleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
|
||||
}
|
||||
HotModuleList hotModuleList = hotModuleService.hotModuleApi().getAllHotModule(platform);
|
||||
if(hotModuleList==null||hotModuleList.getHotModuleList()==null){
|
||||
return Result.error("403","暂无该数据");
|
||||
}
|
||||
@@ -62,7 +52,7 @@ public class HotController {
|
||||
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
|
||||
@GetMapping("/hotLive/modelLive")
|
||||
public Result getHotModuleLives(@RequestParam String moduleId,@RequestParam String platform){
|
||||
HotModule moduleHotLives = hotModuleService.getModuleHotLives(platform, moduleId);
|
||||
HotModule moduleHotLives = hotModuleService.hotModuleApi().getModuleList(platform, moduleId);
|
||||
if(moduleHotLives==null||moduleHotLives.getHotLives()==null){
|
||||
return Result.error("403","暂无该数据");
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.util.ClassUtil;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
@@ -73,7 +74,7 @@ public class InitWorld implements CommandLineRunner {
|
||||
close();
|
||||
}
|
||||
}catch (Exception e){
|
||||
ChopperLogFactory.getLogger(LoggerType.System).error("Init Plugins Error:{}",e.getMessage());
|
||||
ChopperLogFactory.getLogger(LoggerType.System).error("Init Plugins Error:{}",ExceptionUtil.getCause(e));
|
||||
close();
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.thread.ChopperBotGuardPool;
|
||||
import org.example.thread.oddjob.OddJobBoy;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.util.PluginUtil;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -50,7 +51,7 @@ public class WorldInitMachine extends ModuleInitMachine{
|
||||
|
||||
});
|
||||
} catch (Exception e) {
|
||||
return fail(e.getMessage());
|
||||
return fail( ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.example.service;
|
||||
|
||||
import org.example.api.HeatRecommendApi;
|
||||
import org.example.api.HotModuleApi;
|
||||
import org.example.api.HotModuleGuardApi;
|
||||
import org.example.api.LiverFollowApi;
|
||||
import org.example.bean.HotModule;
|
||||
@@ -13,8 +14,8 @@ import java.util.List;
|
||||
public interface HotModuleService {
|
||||
|
||||
|
||||
HotModule getModuleHotLives(String platform, String moduleId);
|
||||
|
||||
HotModuleApi hotModuleApi();
|
||||
LiverFollowApi liverFollowApi();
|
||||
|
||||
HotModuleGuardApi hotModuleGuardApi();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.example.service.impl;
|
||||
|
||||
import org.example.api.HeatRecommendApi;
|
||||
import org.example.api.HotModuleApi;
|
||||
import org.example.api.HotModuleGuardApi;
|
||||
import org.example.api.LiverFollowApi;
|
||||
import org.example.bean.Live;
|
||||
@@ -21,6 +22,8 @@ import java.util.List;
|
||||
@Service
|
||||
public class HotModuleServiceImpl implements HotModuleService {
|
||||
|
||||
@Resource
|
||||
HotModuleApi hotModuleApi;
|
||||
@Resource
|
||||
LiverFollowApi liverFollowApi;
|
||||
|
||||
@@ -34,24 +37,10 @@ public class HotModuleServiceImpl implements HotModuleService {
|
||||
* @param moduleId
|
||||
* @return
|
||||
*/
|
||||
|
||||
@Override
|
||||
public HotModule getModuleHotLives(String platform,String moduleId) {
|
||||
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
|
||||
if(moduleList==null){
|
||||
return null;
|
||||
}
|
||||
HotModule hotModule = moduleList.findHotModuleById(moduleId);
|
||||
if(hotModule!=null){
|
||||
try {
|
||||
List<? extends Live> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
|
||||
hotModule.setHotLives(moduleLiveList);
|
||||
return hotModule;
|
||||
}catch (Exception e){
|
||||
//TODO 交给Spring全局异常处理器
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
public HotModuleApi hotModuleApi() {
|
||||
return hotModuleApi;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -19,6 +19,12 @@
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jsoup</groupId>
|
||||
<artifactId>jsoup</artifactId>
|
||||
<version>1.14.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.example.core.loadtask;
|
||||
|
||||
import org.example.bean.barrage.DouyuBarrage;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.ws.Draft_6455;
|
||||
import org.java_websocket.client.WebSocketClient;
|
||||
import org.java_websocket.handshake.ServerHandshake;
|
||||
@@ -58,7 +59,7 @@ public abstract class WebSocketLoadTask<T> extends CommonLoadTask<T>{
|
||||
client.setSocketFactory(sslSocketFactory);
|
||||
client.connectBlocking(60, TimeUnit.SECONDS);
|
||||
}catch (URISyntaxException | InterruptedException e){
|
||||
this.logger.error("Websocket creeper error:{}",e.getMessage());
|
||||
this.logger.error("Websocket creeper error:{}", ExceptionUtil.getCause(e));
|
||||
return null;
|
||||
}
|
||||
if(client==null){
|
||||
|
||||
@@ -17,7 +17,7 @@ import java.util.TreeSet;
|
||||
* @date 2023/09/07 14:54
|
||||
**/
|
||||
@Component
|
||||
public abstract class AbstractCreeperGroup implements Serializable {
|
||||
public class AbstractCreeperGroup implements Serializable {
|
||||
|
||||
private final Set<CreeperMember> creeperMember;
|
||||
|
||||
@@ -51,9 +51,13 @@ public abstract class AbstractCreeperGroup implements Serializable {
|
||||
return null;
|
||||
}
|
||||
|
||||
public abstract String getPlatform();
|
||||
public String getPlatform(){
|
||||
return null;
|
||||
}
|
||||
|
||||
public abstract String getFunctionName();
|
||||
public String getFunctionName(){
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getGroupName(){
|
||||
return getPlatform()+"_"+getFunctionName();
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package org.example.core.manager;
|
||||
|
||||
import lombok.Data;
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.util.ClassUtil;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
@@ -25,10 +27,25 @@ public class CreeperGroupCenter implements Serializable {
|
||||
|
||||
@Autowired
|
||||
public CreeperGroupCenter(ApplicationContext context){
|
||||
Map<String, AbstractCreeperGroup> beans = context.getBeansOfType(AbstractCreeperGroup.class);
|
||||
beans.forEach((k,v)->{
|
||||
groupMap.put(v.getGroupName(),v);
|
||||
});
|
||||
Set<Class<?>> creepers = ClassUtil.getAnnotationClass(PROJECT_PATH + ".core.creeper.loadconfig", Creeper.class);
|
||||
for (Class<?> creeper : creepers) {
|
||||
Creeper annotation = creeper.getAnnotation(Creeper.class);
|
||||
String name = annotation.creeperName();
|
||||
boolean discard = annotation.discard();
|
||||
String platform = annotation.platform();
|
||||
String groupName = StringUtils.hasText(platform)?CreeperGroupCenter.getGroupName(platform,annotation.group()):annotation.group();
|
||||
int priority = annotation.priority();
|
||||
|
||||
AbstractCreeperGroup.CreeperMember member = new AbstractCreeperGroup.CreeperMember(priority, (Class<? extends LoadConfig>) creeper, discard, name);
|
||||
if (CreeperGroupCenter.GroupMap().containsKey(groupName)) {
|
||||
AbstractCreeperGroup group = CreeperGroupCenter.GroupMap().get(groupName);
|
||||
group.addMember(member);
|
||||
}else{
|
||||
AbstractCreeperGroup group = new AbstractCreeperGroup();
|
||||
groupMap.put(groupName,group);
|
||||
group.addMember(member);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String,AbstractCreeperGroup> GroupMap(){
|
||||
|
||||
@@ -39,11 +39,6 @@ public class CreeperManager extends CommonPlugin {
|
||||
String groupName = CreeperGroupCenter.getGroupName(annotation.platform(),annotation.group());
|
||||
int priority = annotation.priority();
|
||||
|
||||
if (CreeperGroupCenter.GroupMap().containsKey(groupName)) {
|
||||
AbstractCreeperGroup group = CreeperGroupCenter.GroupMap().get(groupName);
|
||||
group.addMember(new AbstractCreeperGroup.CreeperMember(priority, (Class<? extends LoadConfig>) creeper,discard,name));
|
||||
}
|
||||
|
||||
creeperBeans.add(new CreeperBean(name,description,author,groupName,discard,priority));
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -12,6 +12,7 @@ import org.example.init.InitPluginRegister;
|
||||
import org.example.core.taskcenter.request.ReptileRequest;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.util.TimeUtil;
|
||||
|
||||
import java.io.Serializable;
|
||||
@@ -50,8 +51,8 @@ public class ReptileTask implements Serializable {
|
||||
}
|
||||
|
||||
public void reptile(){
|
||||
TaskCenter plugin = InitPluginRegister.getPlugin(PluginName.TASK_CENTER_PLUGIN,TaskCenter.class);
|
||||
try {
|
||||
TaskCenter plugin = InitPluginRegister.getPlugin(PluginName.TASK_CENTER_PLUGIN,TaskCenter.class);
|
||||
assert plugin != null;
|
||||
//开始任务
|
||||
this.startTime = TimeUtil.getNowTime_YMDHMS();
|
||||
@@ -63,15 +64,17 @@ public class ReptileTask implements Serializable {
|
||||
Object res = loadTask.start();
|
||||
request.response(res); //让请求响应结果
|
||||
|
||||
}catch (Exception e){
|
||||
ChopperLogFactory.getLogger(LoggerType.Creeper).info("[{}] {} stop, Error:{}",
|
||||
PluginName.TASK_CENTER_PLUGIN,taskId, ExceptionUtil.getCause(e));
|
||||
}finally {
|
||||
assert plugin != null;
|
||||
//完成任务
|
||||
plugin.finishTask(taskId);
|
||||
plugin.info(String.format("%s end reptile!", taskId));
|
||||
this.type = TaskStatus.Finish;
|
||||
this.endTime = TimeUtil.getNowTime_YMDHMS();
|
||||
plugin.getTaskCenterLogger().setEndTime(taskId,endTime);
|
||||
}catch (Exception e){
|
||||
ChopperLogFactory.getLogger(LoggerType.Creeper).info("[{}] {} stop, Error:{}",
|
||||
PluginName.TASK_CENTER_PLUGIN,taskId,e.getCause());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import lombok.Data;
|
||||
import org.example.core.taskcenter.task.serializer.TaskStatusEnumDeserializer;
|
||||
import org.example.core.taskcenter.task.serializer.TaskStatusEnumSerializer;
|
||||
import org.example.exception.taskcenter.TaskSerializationException;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -51,7 +52,7 @@ public class TaskRecord {
|
||||
oos.writeObject(reptileTask);
|
||||
return baos.toByteArray();
|
||||
}catch (Exception e){
|
||||
throw new TaskSerializationException(reptileTask.getTaskId(),e.getMessage());
|
||||
throw new TaskSerializationException(reptileTask.getTaskId(), ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.example.constpool.PluginName;
|
||||
import org.example.core.loadtask.LoadTask;
|
||||
import org.example.plugin.CommonPlugin;
|
||||
import org.example.thread.NamedThreadFactory;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -52,7 +53,7 @@ public class MonitorCenter extends CommonPlugin {
|
||||
this.info(String.format("%s no setting monitor", taskId));
|
||||
}
|
||||
}catch (Exception e){
|
||||
this.error(String.format("Error:%s",e.getMessage()));
|
||||
this.error(String.format("Error:%s", ExceptionUtil.getCause(e)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSONObject;
|
||||
import org.example.bean.ConfigFile;
|
||||
import org.example.bean.FileType;
|
||||
import org.example.exception.FileCacheException;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.util.JsonFileUtil;
|
||||
import org.example.util.TimeUtil;
|
||||
import org.slf4j.Logger;
|
||||
@@ -259,7 +260,7 @@ public class FileCache <T extends ConfigFile>{
|
||||
try {
|
||||
syncChannel.put(temp);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("自动刷入失败,Error:{}",e.getMessage());
|
||||
logger.error("自动刷入失败,Error:{}", ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import org.example.plugin.CommonPlugin;
|
||||
import org.example.plugin.PluginCheckAndDo;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.util.ConfigFileUtil;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
@@ -40,7 +41,7 @@ public abstract class ConfigInitMachine extends CommonInitMachine{
|
||||
return false;
|
||||
}
|
||||
}catch (Exception e){
|
||||
return fail(e.getMessage());
|
||||
return fail( ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import org.example.constpool.ModuleName;
|
||||
import org.example.constpool.PluginName;
|
||||
import org.example.plugin.CommonPlugin;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -43,7 +44,7 @@ public class FileCacheManagerInitMachine extends CommonInitMachine{
|
||||
return fail();
|
||||
}
|
||||
}catch (Exception e){
|
||||
return fail(e.getMessage());
|
||||
return fail(ExceptionUtil.getCause(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,12 @@ package org.example;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
@@ -11,7 +16,7 @@ import java.util.TreeSet;
|
||||
*/
|
||||
public class App
|
||||
{
|
||||
public static void main(String[] args) {
|
||||
public static void main(String[] args) throws IOException {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.example.api;
|
||||
|
||||
|
||||
import org.example.bean.HotModule;
|
||||
import org.example.bean.Live;
|
||||
import org.example.bean.live.BiliBiliLive;
|
||||
import org.example.bean.live.DouyuLive;
|
||||
@@ -35,19 +36,23 @@ public class HotModuleApi {
|
||||
return HotModuleDataCenter.DataCenter().getLiveList(platform);
|
||||
}
|
||||
|
||||
public static HotModuleList getDouyuAllHotModule(){
|
||||
return new DouyuHotModuleLoadTask(new DouyuHotModuleConfig()).start();
|
||||
public HotModule getModuleList(String platform,String moduleId){
|
||||
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
|
||||
if(moduleList==null){
|
||||
return null;
|
||||
}
|
||||
HotModule hotModule = moduleList.findHotModuleById(moduleId);
|
||||
if(hotModule!=null){
|
||||
try {
|
||||
List<? extends Live> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
|
||||
hotModule.setHotLives(moduleLiveList);
|
||||
return hotModule;
|
||||
}catch (Exception e){
|
||||
//TODO 交给Spring全局异常处理器
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static List<DouyuLive> getDouyuHotLive(){
|
||||
return new DouyuHotLiveLoadTask(new DouyuHotLiveConfig()).start();
|
||||
}
|
||||
|
||||
public static List<DouyuLive> getDouyuHotLive(int moduleId){
|
||||
return new DouyuHotLiveLoadTask(new DouyuHotLiveConfig(moduleId)).start();
|
||||
}
|
||||
|
||||
public static List<BiliBiliLive> getBiliBiliHotLive(String parentId,String areaId,int page){
|
||||
return new BiliBiliHotLiveLoadTask(new BilibiliHotLiveConfig(parentId,areaId,page)).start();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,4 +30,6 @@ public class FocusLiver implements Serializable {
|
||||
private String platform;
|
||||
|
||||
private String tag;
|
||||
|
||||
private Boolean isAuto;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.example.bean.hotmodule.HotModuleList;
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.constpool.PluginName;
|
||||
import org.example.core.creeper.loadtask.HotModuleLoadTask;
|
||||
import org.example.core.loadtask.LoadTask;
|
||||
import org.example.core.manager.CreeperGroupCenter;
|
||||
import org.example.core.manager.CreeperManager;
|
||||
@@ -25,7 +26,9 @@ import org.springframework.stereotype.Component;
|
||||
import javax.annotation.Resource;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
@@ -159,14 +162,15 @@ public class HotModuleDataCenter{
|
||||
}
|
||||
|
||||
private List<? extends Live> checkPlatformAndGetList(String platform,HotModule hotModule){
|
||||
List<? extends Live> lives = new ArrayList<>();
|
||||
if(ConstPool.PLATFORM.DOUYU.getName().equals(platform)){
|
||||
lives = HotModuleApi.getDouyuHotLive(Integer.parseInt(hotModule.getTagId()));
|
||||
}else if (ConstPool.PLATFORM.BILIBILI.getName().equals(platform)){
|
||||
BilibiliHotModule temp = (BilibiliHotModule) hotModule;
|
||||
lives = HotModuleApi.getBiliBiliHotLive(temp.getParent_id(),temp.getTagId(),1);
|
||||
}
|
||||
return lives;
|
||||
SafeBag<List<? extends Live>> bag = new SafeBag<>();
|
||||
Optional.ofNullable(InitPluginRegister.getPlugin(PluginName.CREEPER_MANAGER_PLUGIN, CreeperManager.class))
|
||||
.ifPresent(
|
||||
(plugin)->{
|
||||
HotModuleLoadTask<List<? extends Live>> loadTask = plugin.getLoadTask(CreeperGroupCenter.getGroupName(platform, ConstGroup.HOT_LIVE), hotModule);
|
||||
bag.setData((loadTask.start()));
|
||||
}
|
||||
);
|
||||
return bag.getData();
|
||||
}
|
||||
|
||||
//根据平台的模块名获取某模块
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.api.HotModuleApi;
|
||||
import org.example.bean.HotModule;
|
||||
import org.example.bean.hotmodule.BilibiliHotModule;
|
||||
import org.example.core.creeper.loadconfig.BilibiliHotLiveConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 01:17
|
||||
**/
|
||||
@Component
|
||||
public class BilibiliHotLiveBuilder extends CommonLoadConfigBuilder<BilibiliHotLiveConfig> {
|
||||
|
||||
@Override
|
||||
public BilibiliHotLiveConfig build(Object obj) {
|
||||
if(obj instanceof BilibiliHotModule){
|
||||
return new BilibiliHotLiveConfig(((BilibiliHotModule) obj).getParent_id(),((BilibiliHotModule) obj).getAct_id(),1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.bean.HotModule;
|
||||
import org.example.bean.hotmodule.DouyuHotModule;
|
||||
import org.example.core.creeper.loadconfig.DouyuHotLiveConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 01:18
|
||||
**/
|
||||
|
||||
@Component
|
||||
public class DouyuHotLiveBuilder extends CommonLoadConfigBuilder<DouyuHotLiveConfig> {
|
||||
@Override
|
||||
public DouyuHotLiveConfig build(Object obj) {
|
||||
if(obj instanceof DouyuHotModule){
|
||||
return new DouyuHotLiveConfig(Integer.parseInt(((DouyuHotModule) obj).getTagId()));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.bean.HotModule;
|
||||
import org.example.bean.hotmodule.HuyaHotModule;
|
||||
import org.example.core.creeper.loadconfig.HuyaHotLiveConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 01:17
|
||||
**/
|
||||
@Component
|
||||
public class HuyaHotLiveBuilder extends CommonLoadConfigBuilder<HuyaHotLiveConfig> {
|
||||
|
||||
@Override
|
||||
public HuyaHotLiveConfig build(Object obj) {
|
||||
if(obj instanceof HuyaHotModule){
|
||||
return new HuyaHotLiveConfig(((HuyaHotModule) obj).getTagId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.core.creeper.loadconfig.HuyaLiverCheckerConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 23:06
|
||||
**/
|
||||
@Component
|
||||
public class HuyaLiverCheckerBuilder extends CommonLoadConfigBuilder<HuyaLiverCheckerConfig> {
|
||||
@Override
|
||||
public HuyaLiverCheckerConfig build(Object obj) {
|
||||
if(obj instanceof HuyaLive){
|
||||
return new HuyaLiverCheckerConfig(((HuyaLive) obj).getLiveId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:11
|
||||
**/
|
||||
@Component
|
||||
public class BilibiliHotLiveGroup extends AbstractCreeperGroup {
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.PLATFORM.BILIBILI.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.HOT_LIVE;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 16:52
|
||||
**/
|
||||
@Component
|
||||
public class BilibiliHotModuleGroup extends AbstractCreeperGroup {
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.PLATFORM.BILIBILI.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.HOT_MODULE;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/10 01:02
|
||||
**/
|
||||
@Component
|
||||
public class BilibiliLiverCheckerGroup extends AbstractCreeperGroup {
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.BILIBILI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVER_CHECKER;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:10
|
||||
**/
|
||||
@Component
|
||||
public class DouyuHotLiveGroup extends AbstractCreeperGroup {
|
||||
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.PLATFORM.DOUYU.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.HOT_LIVE;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 16:54
|
||||
**/
|
||||
@Component
|
||||
public class DouyuHotModuleGroup extends AbstractCreeperGroup {
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.PLATFORM.DOUYU.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.HOT_MODULE;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/10 01:54
|
||||
**/
|
||||
@Component
|
||||
public class DouyuLiverCheckerGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.DOUYU;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVER_CHECKER;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,8 @@ package org.example.core.creeper.loadconfig;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.group.BilibiliHotModuleGroup;
|
||||
import org.example.core.creeper.loadtask.BiliBiliHotLiveLoadTask;
|
||||
import org.example.core.manager.Creeper;
|
||||
import org.example.core.manager.CreeperGroupCenter;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package org.example.core.creeper.loadconfig;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.loadtask.HuyaHotLiveLoadTask;
|
||||
import org.example.core.creeper.loadtask.HuyaHotModuleLoadTask;
|
||||
import org.example.core.manager.Creeper;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:47
|
||||
**/
|
||||
@Creeper(creeperName = "虎牙热门直播模块",
|
||||
loadTask = HuyaHotLiveLoadTask.class,
|
||||
creeperDescription = "获取虎牙的热门直播,按照人气排行,也可以获取热门模块下的直播",
|
||||
priority = 10,
|
||||
group = ConstGroup.HOT_LIVE,
|
||||
platform = ConstPool.HUYA
|
||||
)
|
||||
public class HuyaHotLiveConfig extends LoadHotModuleConfig{
|
||||
|
||||
private String moduleId;
|
||||
|
||||
public HuyaHotLiveConfig(String moduleId) {
|
||||
this.url = String.format("https://live.huya.com/liveHttpUI/getLiveList?iGid=%s&iPageNo=1&iPageSize=120", moduleId);
|
||||
}
|
||||
|
||||
public HuyaHotLiveConfig() {
|
||||
moduleId = "0";
|
||||
this.url = String.format("https://live.huya.com/liveHttpUI/getLiveList?iGid=%s&iPageNo=1&iPageSize=120", moduleId);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package org.example.core.creeper.loadconfig;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.loadtask.BilibiliHotModuleLoadTask;
|
||||
import org.example.core.creeper.loadtask.HuyaHotModuleLoadTask;
|
||||
import org.example.core.manager.Creeper;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:39
|
||||
**/
|
||||
|
||||
@Creeper(creeperName = "虎牙热门模块",
|
||||
loadTask = HuyaHotModuleLoadTask.class,
|
||||
creeperDescription = "获取虎牙的热门模块,按照人气排行",
|
||||
priority = 10,
|
||||
group = ConstGroup.HOT_MODULE,
|
||||
platform = ConstPool.HUYA
|
||||
)
|
||||
public class HuyaHotModuleConfig extends LoadHotModuleConfig{
|
||||
|
||||
public HuyaHotModuleConfig() {
|
||||
this.url = "https://www.huya.com/g";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.example.core.creeper.loadconfig;
|
||||
|
||||
import lombok.Data;
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.loadtask.HuyaLiverCheckerLoadTask;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.core.manager.Creeper;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 23:01
|
||||
**/
|
||||
|
||||
@Data
|
||||
@Creeper(creeperName = "虎牙直播检测爬虫",
|
||||
loadTask = HuyaLiverCheckerLoadTask.class,
|
||||
creeperDescription = "用于检测虎牙主播是否开播,并且获取直播详细信息",
|
||||
priority = 10,
|
||||
group = ConstGroup.LIVER_CHECKER,
|
||||
platform = ConstPool.HUYA
|
||||
)
|
||||
public class HuyaLiverCheckerConfig extends LoadConfig {
|
||||
|
||||
private String roomId;
|
||||
|
||||
public HuyaLiverCheckerConfig(String roomId){
|
||||
this.roomId = roomId;
|
||||
this.url = String.format(String.format("https://search.cdn.huya.com/?m=Search&do=getSearchContent&q=%s&typ=-5&rows=16", roomId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTaskId() {
|
||||
return super.getTaskId()+"_"+roomId;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import org.example.core.loadtask.WebMagicLoadTask;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.util.ExceptionUtil;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
@@ -27,7 +28,7 @@ public abstract class HotModuleLoadTask<T> extends WebMagicLoadTask<T> {
|
||||
|
||||
protected void fail(Exception e){
|
||||
finishFlag = FinishFlag.FAIL;
|
||||
ChopperLogFactory.getLogger(LoggerType.Hot).error("loadTask{} finish fail Error:{}",this.getClass().getName(),e.getMessage());
|
||||
ChopperLogFactory.getLogger(LoggerType.Hot).error("loadTask{} finish fail Error:{}",this.getClass().getName(), ExceptionUtil.getCause(e));
|
||||
}
|
||||
|
||||
protected void success(){
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package org.example.core.creeper.loadtask;
|
||||
|
||||
import org.example.bean.live.BiliBiliLive;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.loadconfig.HuyaHotLiveConfig;
|
||||
import org.example.core.creeper.processor.BiliBiliHotLiveProcessor;
|
||||
import org.example.core.creeper.processor.HuyaHotLiveProcessor;
|
||||
import org.example.core.factory.SpiderFactory;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.core.loadtask.WebMagicLoadTask;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:50
|
||||
**/
|
||||
public class HuyaHotLiveLoadTask extends HotModuleLoadTask<List<HuyaLive>> {
|
||||
|
||||
public HuyaHotLiveLoadTask(HuyaHotLiveConfig loadConfig) {
|
||||
super(loadConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HuyaLive> start() {
|
||||
clearFinishFlag();
|
||||
List<HuyaLive> lives;
|
||||
HuyaHotLiveProcessor huyaHotLiveProcessor = new HuyaHotLiveProcessor();
|
||||
Spider spider = SpiderFactory.buildSpider(ConstPool.PLATFORM.HUYA.getName(),
|
||||
huyaHotLiveProcessor,
|
||||
loadConfig.getUrl());
|
||||
try {
|
||||
lives = getData(spider,loadConfig.getUrl());
|
||||
}catch (Exception e){
|
||||
fail(e);
|
||||
return null;
|
||||
}
|
||||
success();
|
||||
return lives;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end() {
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package org.example.core.creeper.loadtask;
|
||||
|
||||
import org.example.bean.hotmodule.HotModuleList;
|
||||
import org.example.bean.hotmodule.HuyaHotModule;
|
||||
import org.example.core.creeper.loadconfig.HuyaHotModuleConfig;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:38
|
||||
**/
|
||||
public class HuyaHotModuleLoadTask extends HotModuleLoadTask<HotModuleList>{
|
||||
|
||||
public HuyaHotModuleLoadTask(HuyaHotModuleConfig loadConfig) {
|
||||
super(loadConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HotModuleList start() {
|
||||
HotModuleList data = null;
|
||||
clearFinishFlag();
|
||||
try {
|
||||
Document doc = Jsoup.connect(loadConfig.getUrl()).get();
|
||||
Element ul = doc.selectFirst("ul#js-game-list");
|
||||
if (ul != null) {
|
||||
Elements liElements = ul.select("li"); // 获取ul下的所有li元素
|
||||
data = new HotModuleList();
|
||||
for (Element li : liElements) {
|
||||
String dataGid = li.attr("data-gid");
|
||||
String gName = li.text();
|
||||
data.getHotModuleList().add(new HuyaHotModule(dataGid, gName));
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
fail(e);
|
||||
}
|
||||
success();
|
||||
return data;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.example.core.creeper.loadtask;
|
||||
|
||||
import org.example.bean.live.DouyuLive;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.creeper.loadconfig.HuyaLiverCheckerConfig;
|
||||
import org.example.core.creeper.processor.DouyuLiverCheckerProcessor;
|
||||
import org.example.core.creeper.processor.HuyaLiverCheckerProcessor;
|
||||
import org.example.core.factory.SpiderFactory;
|
||||
import org.example.core.loadconfig.LoadConfig;
|
||||
import org.example.core.loadtask.WebMagicLoadTask;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 23:01
|
||||
**/
|
||||
public class HuyaLiverCheckerLoadTask extends WebMagicLoadTask<HuyaLive> {
|
||||
|
||||
public HuyaLiverCheckerLoadTask(HuyaLiverCheckerConfig loadConfig) {
|
||||
super(loadConfig);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HuyaLive start() {
|
||||
HuyaLive live = null;
|
||||
Spider spider = SpiderFactory.buildSpider(
|
||||
ConstPool.PLATFORM.HUYA.getName(),
|
||||
new HuyaLiverCheckerProcessor(),
|
||||
loadConfig.getUrl()
|
||||
);
|
||||
try {
|
||||
live = getData(spider,loadConfig.getUrl());
|
||||
}catch (Exception e){
|
||||
return null;
|
||||
}
|
||||
return live;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end() {
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package org.example.core.creeper.processor;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import org.example.bean.live.BiliBiliLive;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.processor.AbstractProcessor;
|
||||
import us.codecraft.webmagic.Page;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/17 00:52
|
||||
**/
|
||||
public class HuyaHotLiveProcessor extends AbstractProcessor {
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
List<HuyaLive> liveList = new ArrayList<>();
|
||||
try {
|
||||
JSONArray Lives = JSON.parseObject(page.getRawText()).getJSONArray("vList");
|
||||
for (Object live : Lives) {
|
||||
if(live instanceof JSONObject){
|
||||
JSONObject jsonLive = (JSONObject) live;
|
||||
String lUid = jsonLive.getString("lUid");
|
||||
String nickName = jsonLive.getString("sNick");
|
||||
Integer watchNum = jsonLive.getInteger("lActivityCount");
|
||||
String moduleId = jsonLive.getString("iGid");
|
||||
String moduleName = jsonLive.getString("sGameFullName");
|
||||
String introduction = jsonLive.getString("sIntroduction");
|
||||
String roomId = jsonLive.getString("lProfileRoom");
|
||||
String pic = jsonLive.getString("sScreenshot");
|
||||
HuyaLive huyaLive = new HuyaLive(lUid);
|
||||
huyaLive.setLiveName(introduction);
|
||||
huyaLive.setPlatform(ConstPool.HUYA);
|
||||
huyaLive.setDescription(introduction);
|
||||
huyaLive.setLiver(nickName);
|
||||
huyaLive.setWatcherNum(watchNum);
|
||||
huyaLive.setModuleId(moduleId);
|
||||
huyaLive.setModuleName(moduleName);
|
||||
huyaLive.setLiveId(roomId);
|
||||
huyaLive.setRoomPic(pic);
|
||||
liveList.add(huyaLive);
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
throw e;
|
||||
}
|
||||
page.putField("data",liveList);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package org.example.core.creeper.processor;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.processor.AbstractProcessor;
|
||||
import org.example.util.TimeUtil;
|
||||
import us.codecraft.webmagic.Page;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 23:37
|
||||
**/
|
||||
public class HuyaLiverCheckerProcessor extends AbstractProcessor {
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
JSONObject live = JSON.parseObject(page.getRawText()).getJSONObject("response").getJSONObject("1");
|
||||
if(live!=null){
|
||||
JSONArray array = Optional.ofNullable(live.getJSONArray("docs")).orElse(new JSONArray());
|
||||
for (Object doc : array) {
|
||||
if(doc instanceof JSONObject){
|
||||
if (Boolean.TRUE.equals(((JSONObject) doc).getBoolean("gameLiveOn"))) {
|
||||
String uid = ((JSONObject) doc).getString("uid");
|
||||
Integer activityCount = ((JSONObject) doc).getInteger("game_activityCount");
|
||||
String avatar = ((JSONObject) doc).getString("game_avatarUrl180");
|
||||
String moduleName = ((JSONObject) doc).getString("game_name");
|
||||
Integer moduleId = ((JSONObject) doc).getInteger("game_id");
|
||||
String roomId = ((JSONObject) doc).getString("room_id");
|
||||
String liver = ((JSONObject) doc).getString("game_nick");
|
||||
String liveName = ((JSONObject) doc).getString("live_intro");
|
||||
Long showTime = ((JSONObject) doc).getLong("rec_live_time") * 1000;
|
||||
HuyaLive huyaLive = new HuyaLive(uid);
|
||||
huyaLive.setWatcherNum(activityCount);
|
||||
huyaLive.setRoomPic(avatar);
|
||||
huyaLive.setModuleName(moduleName);
|
||||
huyaLive.setModuleId(moduleId.toString());
|
||||
huyaLive.setLiveId(roomId);
|
||||
huyaLive.setLiver(liver);
|
||||
huyaLive.setLiveName(liveName);
|
||||
huyaLive.setDescription(liveName);
|
||||
huyaLive.setShowTime(TimeUtil.getFormatDate(showTime));
|
||||
huyaLive.setPlatform(ConstPool.HUYA);
|
||||
page.putField("data",huyaLive);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package org.example.core.component;
|
||||
|
||||
import lombok.Data;
|
||||
import org.example.constpool.PluginName;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
@@ -33,6 +35,7 @@ public class LiveStreamTask {
|
||||
f.parseStream(in, taskId, fileIO);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
ChopperLogFactory.getLogger(PluginName.LIVE_MANAGER_PLUGIN).error("LiveStreamTask start error:", e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package org.example.core.component;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.pool.LiveModuleConstPool;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
@@ -50,7 +51,7 @@ public class M3U8Handle {
|
||||
int exitCode = process.waitFor();
|
||||
logger.error("Exit code: {}",exitCode);
|
||||
} catch (IOException | InterruptedException e) {
|
||||
logger.error("Error during video processing: {}",e.getMessage());
|
||||
logger.error("Error during video processing: {}", ExceptionUtil.getCause(e));
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.example.core.creeper.builder;
|
||||
|
||||
import org.example.bean.live.DouyuLive;
|
||||
import org.example.bean.live.HuyaLive;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.constpool.FileNameBuilder;
|
||||
import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
|
||||
import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
|
||||
import org.example.core.manager.CommonLoadConfigBuilder;
|
||||
import org.example.pool.LiveModuleConstPool;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/10/16 22:10
|
||||
**/
|
||||
@Component
|
||||
public class HuyaLiveLoadConfigBuilder extends CommonLoadConfigBuilder<HuyaLiveOnlineConfig> {
|
||||
@Override
|
||||
public HuyaLiveOnlineConfig build(Object obj) {
|
||||
if(obj instanceof HuyaLive){
|
||||
String liveId = ((HuyaLive) obj).getLiveId();
|
||||
String liver = ((HuyaLive) obj).getLiver();
|
||||
String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.HUYA);
|
||||
String showTime = ((HuyaLive) obj).getShowTime();
|
||||
HuyaLiveOnlineConfig huyaLiveOnlineConfig = new HuyaLiveOnlineConfig(liveId, path, null, false);
|
||||
huyaLiveOnlineConfig.setShowTime(showTime);
|
||||
huyaLiveOnlineConfig.setVideoName(FileNameBuilder.buildVideoFileNameNoSuffix(liver,huyaLiveOnlineConfig.getStartTime()));
|
||||
huyaLiveOnlineConfig.setShowDownloadTable(true);
|
||||
huyaLiveOnlineConfig.setLiverName(liver);
|
||||
huyaLiveOnlineConfig.setRoomName(((HuyaLive) obj).getLiveName());
|
||||
return huyaLiveOnlineConfig;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:31
|
||||
**/
|
||||
@Component
|
||||
public class BiliBiliLiveGroup extends AbstractCreeperGroup {
|
||||
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.BILIBILI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:32
|
||||
**/
|
||||
@Component
|
||||
public class DouyuLiveGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.DOUYU;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author Genius
|
||||
* @date 2023/09/07 17:33
|
||||
**/
|
||||
@Component
|
||||
public class DouyuRecordGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.DOUYU;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVE_RECORD;
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package org.example.core.creeper.group;
|
||||
|
||||
import org.example.constpool.ConstGroup;
|
||||
import org.example.constpool.ConstPool;
|
||||
import org.example.core.manager.AbstractCreeperGroup;
|
||||
|
||||
/**
|
||||
* @author dhx
|
||||
* @date 2023/10/10 10:53
|
||||
*/
|
||||
public class HuyaLiveGroup extends AbstractCreeperGroup {
|
||||
@Override
|
||||
public String getPlatform() {
|
||||
return ConstPool.HUYA;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFunctionName() {
|
||||
return ConstGroup.LIVE_ONLINE;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import org.example.core.parser.impl.DouyuFlvUrlParser;
|
||||
import org.example.core.parser.impl.DouyuM3u8UrlParser;
|
||||
import org.example.log.ChopperLogFactory;
|
||||
import org.example.log.LoggerType;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
/**
|
||||
@@ -37,7 +38,7 @@ public class DouyuRecordLoadTask extends CommonLoadTask<String> {
|
||||
douyuRecordConfig.getVideoPath(),
|
||||
douyuRecordConfig.getVideoName());
|
||||
}catch (Exception e){
|
||||
logger.info("{} 录播爬取失败,error:{}",douyuRecordConfig.getVideoName(),e.getMessage());
|
||||
logger.info("{} 录播爬取失败,error:{}",douyuRecordConfig.getVideoName(), ExceptionUtil.getCause(e));
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
@@ -2,11 +2,13 @@ package org.example.core.factory;
|
||||
|
||||
import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig;
|
||||
import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
|
||||
import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
|
||||
import org.example.core.creeper.loadconfig.LoadLiveConfig;
|
||||
import org.example.core.parser.PlatformVideoUrlParser;
|
||||
import org.example.core.parser.impl.BilibiliFlvUrlParser;
|
||||
import org.example.core.component.LiveStreamTask;
|
||||
import org.example.core.parser.impl.DouyuFlvUrlParser;
|
||||
import org.example.core.parser.impl.HuyaFlvUrlParser;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
@@ -20,7 +22,8 @@ public class LiveTaskFactory {
|
||||
|
||||
private final Map<Class<? extends LoadLiveConfig>, PlatformVideoUrlParser> parserMap = Map.of(
|
||||
BilibiliLiveOnlineConfig.class,new BilibiliFlvUrlParser(),
|
||||
DouyuLiveOnlineConfig.class, new DouyuFlvUrlParser()
|
||||
DouyuLiveOnlineConfig.class, new DouyuFlvUrlParser(),
|
||||
HuyaLiveOnlineConfig.class, new HuyaFlvUrlParser()
|
||||
);
|
||||
|
||||
public LiveTaskFactory() {
|
||||
|
||||
@@ -7,6 +7,7 @@ import org.example.constpool.PluginName;
|
||||
import org.example.plugin.CommonPlugin;
|
||||
import org.example.plugin.annotation.Plugin;
|
||||
import org.example.pool.LiveModuleConstPool;
|
||||
import org.example.util.ExceptionUtil;
|
||||
import org.example.util.FileUtil;
|
||||
|
||||
import java.nio.file.Files;
|
||||
@@ -56,7 +57,7 @@ public class LiveConfigInitMachine extends ConfigInitMachine{
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
return fail(e.getMessage());
|
||||
return fail( ExceptionUtil.getCause(e));
|
||||
}
|
||||
return super.init();
|
||||
}
|
||||
|
||||
@@ -299,7 +299,7 @@ public class VideoUtil {
|
||||
}
|
||||
logger.info("转码完成,视频帧:{},音频帧:{},数据帧:{}", videoFrameNum, audioFrameNum, dataFrameNum);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
logger.error( ExceptionUtil.getCause(e));
|
||||
return "error";
|
||||
} finally {
|
||||
if (recorder != null) {
|
||||
@@ -394,7 +394,7 @@ public class VideoUtil {
|
||||
i++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
logger.error( ExceptionUtil.getCause(e));
|
||||
return "error";
|
||||
}finally {
|
||||
if (recorder != null) {
|
||||
|
||||
@@ -21,7 +21,7 @@ import java.util.List;
|
||||
**/
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = ConsoleApplication.class)
|
||||
@SpringBootTest(classes = ConsoleApplication.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
|
||||
public class BarrageCreeperTest {
|
||||
|
||||
@Test
|
||||
@@ -33,7 +33,7 @@ public class BarrageCreeperTest {
|
||||
|
||||
@Test
|
||||
public void testHuyaBarrageCreeper(){
|
||||
HuyaLiveBarrageLoadConfig config = new HuyaLiveBarrageLoadConfig("123","lwz6038");
|
||||
HuyaLiveBarrageLoadConfig config = new HuyaLiveBarrageLoadConfig("123", "243547");
|
||||
try {
|
||||
List<HuyaBarrage> start = new HuyaLiveBarrageLoadTask(config).start();
|
||||
} catch (FileCacheException e) {
|
||||
|
||||
@@ -53,29 +53,29 @@ public class BarrageCurveTest {
|
||||
@Test
|
||||
public void testCurve(){
|
||||
long l = System.currentTimeMillis();
|
||||
List<BarragePoint> list = plugin.generateCurve(new BarrageEvent("bilibili", "online", "Asaki大人", "2023-09-19 22_00_06"));
|
||||
System.out.println(System.currentTimeMillis()-l);
|
||||
LineChartDemo demo = new LineChartDemo("Asaki大人",list);
|
||||
demo.pack();
|
||||
RefineryUtilities.centerFrameOnScreen(demo);
|
||||
demo.setVisible(true);
|
||||
List<PopularRange> range = popularRangePlugin.findRange(list);
|
||||
System.out.println(range);
|
||||
int i = 0;
|
||||
for (PopularRange popularRange : range) {
|
||||
long timeNaos = TimeUtil.getTimeNaos("2023-09-19 22:00:06");
|
||||
long videoStartTime = popularRange.getStartTime()-timeNaos;
|
||||
videoStartTime = videoStartTime<0?0:videoStartTime;
|
||||
long videoEndTime = popularRange.getEndTime()-timeNaos;
|
||||
System.out.println(videoStartTime+" "+videoEndTime);
|
||||
VideoUtil.cutVideoByFFMpeg("E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06.flv",
|
||||
"E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06"+i+".flv",
|
||||
videoStartTime/1000,videoEndTime/1000);
|
||||
i++;
|
||||
}
|
||||
while (true){
|
||||
|
||||
}
|
||||
// List<BarragePoint> list = plugin.generateCurve(new BarrageEvent("bilibili", "online", "Asaki大人", "2023-09-19 22_00_06"));
|
||||
// System.out.println(System.currentTimeMillis()-l);
|
||||
// LineChartDemo demo = new LineChartDemo("Asaki大人",list);
|
||||
// demo.pack();
|
||||
// RefineryUtilities.centerFrameOnScreen(demo);
|
||||
// demo.setVisible(true);
|
||||
// List<PopularRange> range = popularRangePlugin.findRange(list);
|
||||
// System.out.println(range);
|
||||
// int i = 0;
|
||||
// for (PopularRange popularRange : range) {
|
||||
// long timeNaos = TimeUtil.getTimeNaos("2023-09-19 22:00:06");
|
||||
// long videoStartTime = popularRange.getStartTime()-timeNaos;
|
||||
// videoStartTime = videoStartTime<0?0:videoStartTime;
|
||||
// long videoEndTime = popularRange.getEndTime()-timeNaos;
|
||||
// System.out.println(videoStartTime+" "+videoEndTime);
|
||||
// VideoUtil.cutVideoByFFMpeg("E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06.flv",
|
||||
// "E:\\Project\\ChopperBot\\chopperbot-test\\config\\LiveRecord\\online\\bilibili\\Asaki大人_2023-09-19 22_00_06"+i+".flv",
|
||||
// videoStartTime/1000,videoEndTime/1000);
|
||||
// i++;
|
||||
// }
|
||||
// while (true){
|
||||
//
|
||||
// }
|
||||
}
|
||||
|
||||
class LineChartDemo extends ApplicationFrame {
|
||||
|
||||
@@ -26,10 +26,10 @@ public class BarragePoupularRangeTest {
|
||||
|
||||
@Test
|
||||
public void Test(){
|
||||
BarrageEvent event = new BarrageEvent("douyu","online",null,null,"即将拥有人鱼线的PDD_2023-09-14 14_37_54.json");
|
||||
List<BarragePoint> points = InitPluginRegister.getPlugin(PluginName.BARRAGE_SCORE_CURVE_PLUGIN, BarrageScoreCurvePlugin.class).generateCurve(event);
|
||||
List<PopularRange> ranges = InitPluginRegister.getPlugin(PluginName.BARRAGE_POPULAR_RANGE_PLUGIN, BarragePopularRangePlugin.class).findRange(points);
|
||||
System.out.println(ranges);
|
||||
// BarrageEvent event = new BarrageEvent("douyu","online",null,null,"即将拥有人鱼线的PDD_2023-09-14 14_37_54.json");
|
||||
// List<BarragePoint> points = InitPluginRegister.getPlugin(PluginName.BARRAGE_SCORE_CURVE_PLUGIN, BarrageScoreCurvePlugin.class).generateCurve(event);
|
||||
// List<PopularRange> ranges = InitPluginRegister.getPlugin(PluginName.BARRAGE_POPULAR_RANGE_PLUGIN, BarragePopularRangePlugin.class).findRange(points);
|
||||
// System.out.println(ranges);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,5 +32,5 @@
|
||||
"CreeperConfig":true
|
||||
}
|
||||
},
|
||||
"updateTime":"2023-10-16 17:13:50"
|
||||
"updateTime":"2023-10-17 02:15:29"
|
||||
}
|
||||
BIN
database.db
BIN
database.db
Binary file not shown.
Reference in New Issue
Block a user