diff --git a/chopperbot-barrage/src/main/java/org/example/config/BarrageSaveFile.java b/chopperbot-barrage/src/main/java/org/example/config/BarrageSaveFile.java deleted file mode 100644 index 0de0e4d..0000000 --- a/chopperbot-barrage/src/main/java/org/example/config/BarrageSaveFile.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.example.config; - -import com.alibaba.fastjson.JSONArray; -import org.example.bean.Barrage; -import org.example.bean.ConfigFile; -import org.example.core.creeper.loadconfig.LoadBarrageConfig; -import org.example.exception.FileCacheException; -import org.example.util.FileUtil; -import org.example.util.JsonFileUtil; -import org.example.util.TimeUtil; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * @author Genius - * @date 2023/04/25 22:08 - **/ - -/** - * 弹幕保存文件配置类，通过LoadConfig获取主播信息，平台信息，弹幕爬取时间，生成弹幕文件 - */ -public class BarrageSaveFile extends ConfigFile> { - - private LoadBarrageConfig loadBarrageConfig; - - - private int alreadyRead = 0; - - public BarrageSaveFile(LoadBarrageConfig loadBarrageConfig, ConcurrentLinkedQueue data) throws FileCacheException { - super(); - this.loadBarrageConfig = loadBarrageConfig; - if (!init(data)) { - throw new FileCacheException("File init Error"); - } - } - - /** - * 自动生成主播弹幕文件夹以及当天直播弹幕数据文本 - * - * @param data - * @return - */ - private boolean init(ConcurrentLinkedQueue data) { - String fileName = this.filaName(); - setFileName(fileName); - //String rootPath = Paths.get(BARRAGE_ROOT, "anchor",loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 - String rootPath = Paths.get("./config/Barrage", "anchor",loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 - setFilePath(rootPath); - Path path = Path.of(rootPath); - //TODO 待移除 建立主播文件夹 - try { - if (!Files.exists(path)) { - Files.createDirectory(path); - } - } catch (IOException e) { - return false; - } - - setData(data); - if (!FileUtil.isFileExist(Paths.get(rootPath, fileName).toString())) { - - return !(JsonFileUtil.writeJsonFile(rootPath, fileName, this.packageConfig()) == null); - }else{ - String filePath = Path.of(rootPath.toString(),fileName).toString(); - Map stringObjectMap = JsonFileUtil.readJsonFile(filePath); - JSONArray array = (JSONArray) stringObjectMap.get("data"); - for (Object barrage : array) { - if(barrage instanceof Barrage){ - data.add((T) barrage); - alreadyRead++; - } - } - } - - return true; - } - - private String filaName() { - String format = "%s_%s_%s_%s.json"; - return String.format(format, - loadBarrageConfig.getPlatform(), - loadBarrageConfig.getAnchorName(), - loadBarrageConfig.getAction(), - TimeUtil.getToday_YMD()); - } - - public int getAlreadyRead() { - return alreadyRead; - } -} diff --git a/chopperbot-barrage/src/main/java/org/example/constpool/BarrageModuleConstPool.java b/chopperbot-barrage/src/main/java/org/example/constpool/BarrageModuleConstPool.java index 6efe209..d13f526 100644 --- a/chopperbot-barrage/src/main/java/org/example/constpool/BarrageModuleConstPool.java +++ b/chopperbot-barrage/src/main/java/org/example/constpool/BarrageModuleConstPool.java @@ -6,7 +6,7 @@ package org.example.constpool; * @Date 2023/8/2 23:38 */ public class BarrageModuleConstPool { - public static final String BARRAGE_FILE_PATH = "config/barrage/"; + public static final String BARRAGE_ROOT_PATH = (String) GlobalFileCache.ModuleSrcConfigFile.get("src", ConstPool.BARRAGE); public static final String BARRAGE_SCORE_CONFIG = "config/barrage/barrageScoreConfig/"; public static final String FILE_TYPE = ".json"; diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/builder/BilibiliLiveBarrageLoadConfigBuilder.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/builder/BilibiliLiveBarrageLoadConfigBuilder.java new file mode 100644 index 0000000..417d9a0 --- /dev/null +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/builder/BilibiliLiveBarrageLoadConfigBuilder.java @@ -0,0 +1,20 @@ +package org.example.core.creeper.builder; + +import org.example.bean.live.BiliBiliLive; +import org.example.core.creeper.loadconfig.BilibiliLiveLoadBarrageConfig; +import org.example.core.manager.CommonLoadConfigBuilder; + +/** + * @author Genius + * @date 2023/09/05 18:32 + **/ +public class BilibiliLiveBarrageLoadConfigBuilder extends CommonLoadConfigBuilder { + + @Override + public BilibiliLiveLoadBarrageConfig build(Object obj) { + if(obj instanceof BiliBiliLive){ + return new BilibiliLiveLoadBarrageConfig(((BiliBiliLive) obj).getLiveName(),((BiliBiliLive) obj).getLiveId()); + } + return null; + } +} diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/file/BarrageSaveFile.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/file/BarrageSaveFile.java index 9d950cd..544b500 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/creeper/file/BarrageSaveFile.java +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/file/BarrageSaveFile.java @@ -3,6 +3,7 @@ package org.example.core.creeper.file; import com.alibaba.fastjson.JSONArray; import org.example.bean.Barrage; import org.example.bean.ConfigFile; +import org.example.constpool.ConstPool; import org.example.core.creeper.loadconfig.LoadBarrageConfig; import org.example.exception.FileCacheException; import org.example.util.FileUtil; @@ -16,6 +17,8 @@ import java.nio.file.Paths; import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; +import static org.example.constpool.BarrageModuleConstPool.BARRAGE_ROOT_PATH; + /** * @author Genius * @date 2023/04/25 22:08 @@ -48,14 +51,16 @@ public class BarrageSaveFile extends ConfigFile data) { String fileName = this.filaName(); setFileName(fileName); - //String rootPath = Paths.get(BARRAGE_ROOT, "anchor",loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 - String rootPath = Paths.get("./config/Barrage", "anchor",loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 + + String rootPath = Paths.get(BARRAGE_ROOT_PATH, + loadBarrageConfig.getPlatform(), + loadBarrageConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径 + setFilePath(rootPath); Path path = Path.of(rootPath); - //TODO 待移除 建立主播文件夹 try { if (!Files.exists(path)) { - Files.createDirectory(path); + Files.createDirectories(path); } } catch (IOException e) { @@ -82,9 +87,10 @@ public class BarrageSaveFile extends ConfigFile> { - - private final BilibiliBarrageLiveProcessor bilibiliBarrageLiveProcessor; - - private final BarragePipelineWriteJson barragePipelineWriteJson; - - public BilibiliBarrageLiveLoadTask(BilibiliLiveLoadBarrageConfig LoadBarrageConfig) { - super(LoadBarrageConfig); - bilibiliBarrageLiveProcessor = new BilibiliBarrageLiveProcessor(loadConfig.getUrl()); - barragePipelineWriteJson = new BarragePipelineWriteJson(LoadBarrageConfig); - } - - @Override - public List start() { - Spider spider = Spider.create(bilibiliBarrageLiveProcessor) - // 设置起始Request - .addRequest(new Request(loadConfig.getUrl())) - // 设置结果处理类 - .addPipeline(barragePipelineWriteJson) - // 设置抓取线程数（可根据需要调整） - .thread(5) - .setEmptySleepTime(100); - spider.start(); - while (isRunning()){ - flushCacheAndSave("-1"); - } - spider.stop(); - return barragePipelineWriteJson.getResult(); - } - - @Override - public void end() { - bilibiliBarrageLiveProcessor.end(); - } - - @Override - public boolean isRunning() { - return bilibiliBarrageLiveProcessor.isRunning(); - } - - @Override - public int getCacheSize() { - return barragePipelineWriteJson.getCacheSize(); - } - - @Override - public int flushCacheAndSave(String key) { - return barragePipelineWriteJson.writeDataToFileAndFlushCache(key); - } -} diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/BilibiliLiveBarrageLoadTask.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/BilibiliLiveBarrageLoadTask.java new file mode 100644 index 0000000..95b925b --- /dev/null +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/BilibiliLiveBarrageLoadTask.java @@ -0,0 +1,62 @@ +package org.example.core.creeper.loadtask; + +import org.example.bean.barrage.BilibiliBarrage; +import org.example.constpool.ConstPool; +import org.example.core.creeper.loadconfig.BilibiliLiveLoadBarrageConfig; +import org.example.core.creeper.pipline.BarragePipelineWriteJson; +import org.example.core.creeper.processor.BilibiliBarrageLiveProcessor; +import org.example.core.factory.SpiderFactory; +import org.example.core.loadtask.ASyncLoadTask; + +import us.codecraft.webmagic.Spider; + +import java.util.List; + +/** + * (B站直播)一个任务 + * @author 燧枫 + * @date 2023/4/23 18:19 +*/ +public class BilibiliLiveBarrageLoadTask extends ASyncLoadTask> { + + + public BilibiliLiveBarrageLoadTask(BilibiliLiveLoadBarrageConfig LoadBarrageConfig) { + super(LoadBarrageConfig); + } + + @Override + public List start() { + BilibiliBarrageLiveProcessor bilibiliBarrageLiveProcessor = new BilibiliBarrageLiveProcessor(loadConfig.getUrl()); + BarragePipelineWriteJson barragePipelineWriteJson = + new BarragePipelineWriteJson((BilibiliLiveLoadBarrageConfig)loadConfig); + Spider spider = SpiderFactory.buildSpider( + ConstPool.BILIBILI,bilibiliBarrageLiveProcessor,loadConfig.getUrl() + ); + spider.addPipeline(barragePipelineWriteJson).start(); + while (bilibiliBarrageLiveProcessor.isRunning()){ + barragePipelineWriteJson.writeDataToFileAndFlushCache("-1"); + } + spider.stop(); + return barragePipelineWriteJson.getResult(); + } + + @Override + public void end() { + + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public int getCacheSize() { + return 0; + } + + @Override + public int flushCacheAndSave(String key) { + return 0; + } +} diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuBarrageLoadTask.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuRecordBarrageLoadTask.java similarity index 83% rename from chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuBarrageLoadTask.java rename to chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuRecordBarrageLoadTask.java index 136f522..909e7ae 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuBarrageLoadTask.java +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/loadtask/DouyuRecordBarrageLoadTask.java @@ -2,7 +2,7 @@ package org.example.core.creeper.loadtask; import org.example.bean.barrage.DouyuBarrage; import org.example.constpool.ConstPool; -import org.example.core.creeper.loadconfig.DouyuLiveLoadBarrageConfig; +import org.example.core.creeper.loadconfig.DouyuRecordLoadBarrageConfig; import org.example.core.creeper.loadconfig.LoadBarrageConfig; import org.example.core.creeper.pipline.BarragePipelineWriteJson; import org.example.core.creeper.processor.DouyuBarrageRecordProcessor; @@ -17,10 +17,10 @@ import java.util.List; * @author 燧枫 * @date 2023/4/23 18:12 */ -public class DouyuBarrageLoadTask extends ASyncLoadTask> { +public class DouyuRecordBarrageLoadTask extends ASyncLoadTask> { - public DouyuBarrageLoadTask(DouyuLiveLoadBarrageConfig loadBarrageConfig) { + public DouyuRecordBarrageLoadTask(DouyuRecordLoadBarrageConfig loadBarrageConfig) { super(loadBarrageConfig); } @@ -69,7 +69,7 @@ public class DouyuBarrageLoadTask extends ASyncLoadTask> { } public static void main(String[] args) { - new DouyuBarrageLoadTask(new DouyuLiveLoadBarrageConfig("yjj","0Q8mMYYE18mM49Ad")).start(); + new DouyuRecordBarrageLoadTask(new DouyuRecordLoadBarrageConfig("yjj","0Q8mMYYE18mM49Ad")).start(); } } diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/pipline/BarragePipelineWriteJson.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/pipline/BarragePipelineWriteJson.java index 1647116..fe65137 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/creeper/pipline/BarragePipelineWriteJson.java +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/pipline/BarragePipelineWriteJson.java @@ -2,7 +2,7 @@ package org.example.core.creeper.pipline; import org.example.bean.Barrage; import org.example.cache.FileCache; -import org.example.config.BarrageSaveFile; +import org.example.core.creeper.file.BarrageSaveFile; import org.example.core.creeper.loadconfig.LoadBarrageConfig; import org.example.exception.FileCacheException; import org.example.log.LoggerType; diff --git a/chopperbot-barrage/src/main/java/org/example/core/creeper/processor/BilibiliBarrageLiveProcessor.java b/chopperbot-barrage/src/main/java/org/example/core/creeper/processor/BilibiliBarrageLiveProcessor.java index 4b03855..4c825fb 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/creeper/processor/BilibiliBarrageLiveProcessor.java +++ b/chopperbot-barrage/src/main/java/org/example/core/creeper/processor/BilibiliBarrageLiveProcessor.java @@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.example.bean.Barrage; import org.example.core.processor.AbstractProcessor; +import org.example.util.TimeUtil; import us.codecraft.webmagic.Page; import java.util.ArrayList; @@ -41,7 +42,6 @@ public class BilibiliBarrageLiveProcessor extends AbstractProcessor { if (isFirst) { isFirst = false; init(page); - return; } processOnePage(page); } @@ -49,7 +49,7 @@ public class BilibiliBarrageLiveProcessor extends AbstractProcessor { // 初始化 public void init(Page page) { // 记录开始爬取时间 - startTime = System.currentTimeMillis() / 1000; + startTime = System.currentTimeMillis(); page.addTargetRequest(url + "&_=" + System.currentTimeMillis()); } @@ -71,11 +71,12 @@ public class BilibiliBarrageLiveProcessor extends AbstractProcessor { JSONObject temp = (JSONObject) o; JSONObject check_info = temp.getJSONObject("check_info"); // 唯一id + String timeLine = temp.getString("timeline"); String mid = check_info.getString("ct"); if (!processedMids.contains(mid)) { // 检查mid是否已经存在 // 真实时间戳 - Long timeReal = check_info.getLong("ts"); + Long timeReal = TimeUtil.getTimeNaos(timeLine); // 相对时间戳 Long timeIndex = timeReal - startTime; if (timeIndex < 0) timeIndex = 0L; diff --git a/chopperbot-barrage/src/main/java/org/example/core/factory/BarrageLoadConfigFactory.java b/chopperbot-barrage/src/main/java/org/example/core/factory/BarrageLoadConfigFactory.java deleted file mode 100644 index 5d56bbb..0000000 --- a/chopperbot-barrage/src/main/java/org/example/core/factory/BarrageLoadConfigFactory.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.example.core.factory; - -import org.example.constpool.ConstPool; -import org.example.core.creeper.loadconfig.BilibiliLiveLoadBarrageConfig; -import org.example.core.creeper.loadconfig.DouyuLiveLoadBarrageConfig; -import org.example.core.creeper.loadconfig.LoadBarrageConfig; - -/** - * @author Genius - * @date 2023/09/01 00:19 - **/ -public class BarrageLoadConfigFactory { - - public static T buildBarrageConfig(String platform,String liver,String roomId){ - platform = platform.toLowerCase(); - if(ConstPool.PLATFORM.DOUYU.getName().equals(platform)){ - return (T) new DouyuLiveLoadBarrageConfig(liver,roomId); - }else if(ConstPool.PLATFORM.BILIBILI.getName().equals(platform)){ - return (T) new BilibiliLiveLoadBarrageConfig(liver,roomId); - } - return null; - } -} diff --git a/chopperbot-barrage/src/main/java/org/example/core/listen/BarrageFileMonitor.java b/chopperbot-barrage/src/main/java/org/example/core/listen/BarrageFileMonitor.java index d77446c..98657ed 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/listen/BarrageFileMonitor.java +++ b/chopperbot-barrage/src/main/java/org/example/core/listen/BarrageFileMonitor.java @@ -82,7 +82,7 @@ public class BarrageFileMonitor extends GuardPlugin { CreepBarrage creepBarrage = (CreepBarrage) res; String fileName = creepBarrage.getRoomId()+creepBarrage.getAnchorName()+creepBarrage.getCreeperTime(); try { - BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(BarrageModuleConstPool.BARRAGE_FILE_PATH+fileName+BarrageModuleConstPool.FILE_TYPE)); + BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(BarrageModuleConstPool.BARRAGE_ROOT_PATH+fileName+BarrageModuleConstPool.FILE_TYPE)); barrageMap.put(fileName,creepBarrage.getBarrageList()); bufferedWriter.write(creepBarrage.getBarrageList().toString()); logger.info("[BarrageModule]: file write suc!"); diff --git a/chopperbot-barrage/src/main/java/org/example/core/process/FileConfigProcessor.java b/chopperbot-barrage/src/main/java/org/example/core/process/FileConfigProcessor.java index 4f5009b..0bca725 100644 --- a/chopperbot-barrage/src/main/java/org/example/core/process/FileConfigProcessor.java +++ b/chopperbot-barrage/src/main/java/org/example/core/process/FileConfigProcessor.java @@ -39,7 +39,7 @@ public class FileConfigProcessor { } public static void barrageFileLoadingProcessor(Map> barrageMap){ - File file = new File(BarrageModuleConstPool.BARRAGE_FILE_PATH); + File file = new File(BarrageModuleConstPool.BARRAGE_ROOT_PATH); for (File listFile : Objects.requireNonNull(file.listFiles())) { if(!"barrageScoreConfig".equals(listFile.getName())){ for (File barrageFile : Objects.requireNonNull(listFile.listFiles())) { diff --git a/chopperbot-common/src/main/java/org/example/util/TimeUtil.java b/chopperbot-common/src/main/java/org/example/util/TimeUtil.java index e0177a8..f582da5 100644 --- a/chopperbot-common/src/main/java/org/example/util/TimeUtil.java +++ b/chopperbot-common/src/main/java/org/example/util/TimeUtil.java @@ -1,5 +1,7 @@ package org.example.util; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -53,6 +55,16 @@ public class TimeUtil { return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); } + public static long getTimeNaos(String date){ + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + try { + return sdf.parse(date).getTime(); + } catch (ParseException e) { + return 0; + } + } + public static String getHMS(Long seconds){ long hours = seconds / 3600; long minutes = (seconds % 3600) / 60; diff --git a/chopperbot-creeper/src/main/java/org/example/core/manager/CommonLoadConfigBuilder.java b/chopperbot-creeper/src/main/java/org/example/core/manager/CommonLoadConfigBuilder.java new file mode 100644 index 0000000..db21ed3 --- /dev/null +++ b/chopperbot-creeper/src/main/java/org/example/core/manager/CommonLoadConfigBuilder.java @@ -0,0 +1,12 @@ +package org.example.core.manager; + +import org.example.core.loadconfig.LoadConfig; +import org.example.core.loadtask.LoadTask; + +/** + * @author Genius + * @date 2023/09/05 18:33 + **/ +public abstract class CommonLoadConfigBuilder implements LoadConfigBuilder { + +} diff --git a/chopperbot-creeper/src/main/java/org/example/core/manager/annotation/Creeper.java b/chopperbot-creeper/src/main/java/org/example/core/manager/Creeper.java similarity index 64% rename from chopperbot-creeper/src/main/java/org/example/core/manager/annotation/Creeper.java rename to chopperbot-creeper/src/main/java/org/example/core/manager/Creeper.java index f65ab2b..97f429a 100644 --- a/chopperbot-creeper/src/main/java/org/example/core/manager/annotation/Creeper.java +++ b/chopperbot-creeper/src/main/java/org/example/core/manager/Creeper.java @@ -1,6 +1,9 @@ -package org.example.core.manager.annotation; +package org.example.core.manager; import org.example.core.loadtask.LoadTask; +import org.example.core.manager.CommonLoadConfigBuilder; +import org.example.core.manager.CreeperManager; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -17,4 +20,6 @@ public @interface Creeper { String creeperDescription() default "该爬虫暂无介绍"; //插件介绍 + Class builder() default CreeperManager.ValidLoadConfigBuilder.class; + } diff --git a/chopperbot-creeper/src/main/java/org/example/core/manager/CreeperManager.java b/chopperbot-creeper/src/main/java/org/example/core/manager/CreeperManager.java index 156b544..5eec8c1 100644 --- a/chopperbot-creeper/src/main/java/org/example/core/manager/CreeperManager.java +++ b/chopperbot-creeper/src/main/java/org/example/core/manager/CreeperManager.java @@ -1,16 +1,13 @@ package org.example.core.manager; import org.example.core.loadtask.LoadTask; -import org.example.core.manager.annotation.Creeper; import org.example.core.taskcenter.request.ReptileRequest; import org.example.core.taskcenter.task.ReptileTask; import org.example.plugin.CommonPlugin; import org.example.core.loadconfig.LoadConfig; import org.example.util.ClassUtil; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static org.example.constpool.ConstPool.PROJECT_PATH; @@ -23,11 +20,14 @@ public class CreeperManager extends CommonPlugin { private ConcurrentHashMap> nameToLoadTaskMapping; + private Map loadConfigBuilder; + private ArrayList creeperBeans; public CreeperManager(String module, String pluginName, List needPlugins, boolean isAutoStart) { super(module, pluginName, needPlugins, isAutoStart); nameToLoadTaskMapping = new ConcurrentHashMap<>(); + loadConfigBuilder = new HashMap<>(); creeperBeans = new ArrayList<>(); } @@ -38,14 +38,26 @@ public class CreeperManager extends CommonPlugin { Creeper annotation = creeper.getAnnotation(Creeper.class); String name = annotation.creeperName(); String description = annotation.creeperDescription(); - Class loadTask = annotation.loadTask(); + Class builder = annotation.builder(); + try { + if(!builder.getName().equals(ValidLoadConfigBuilder.class.getName())){ + CommonLoadConfigBuilder commonLoadConfigBuilder = builder.getDeclaredConstructor().newInstance(); + loadConfigBuilder.put(name,commonLoadConfigBuilder); + } + } catch (Exception e) { + throw new RuntimeException(e); + } nameToLoadTaskMapping.put(name, (Class) creeper); + creeperBeans.add(new CreeperBean(name,description)); } return true; } public T getLoadTask(V loadConfig){ + if(loadConfig==null){ + return null; + } Class task = loadConfig.getClass().getAnnotation(Creeper.class).loadTask(); try { T loadTask = (T) task.getDeclaredConstructor(loadConfig.getClass()).newInstance(loadConfig); @@ -65,6 +77,18 @@ public class CreeperManager extends CommonPlugin { } } + public T buildLoadConfig(String name,Object param){ + if(loadConfigBuilder.containsKey(name)) { + CommonLoadConfigBuilder commonLoadConfigBuilder = loadConfigBuilder.get(name); + return (T) commonLoadConfigBuilder.build(param); + } + return null; + } + + public T getLoadTask(String name,Object param){ + return getLoadTask(buildLoadConfig(name,param)); + } + public ReptileTask getReptileTask(ReptileRequest request){ LoadTask loadTask = getLoadTask(request.getLoadConfig()); if(loadTask==null){ @@ -81,4 +105,13 @@ public class CreeperManager extends CommonPlugin { return creeperBeans; } + protected class ValidLoadConfigBuilder extends CommonLoadConfigBuilder{ + + @Override + public LoadConfig build(Object obj) { + return null; + } + + } + } diff --git a/chopperbot-creeper/src/main/java/org/example/core/manager/LoadConfigBuilder.java b/chopperbot-creeper/src/main/java/org/example/core/manager/LoadConfigBuilder.java new file mode 100644 index 0000000..7c02d78 --- /dev/null +++ b/chopperbot-creeper/src/main/java/org/example/core/manager/LoadConfigBuilder.java @@ -0,0 +1,13 @@ +package org.example.core.manager; + +import org.example.core.loadconfig.LoadConfig; +import org.example.core.loadtask.LoadTask; + +/** + * @author Genius + * @date 2023/09/05 18:23 + **/ +public interface LoadConfigBuilder { + + T build(Object obj); +} diff --git a/chopperbot-creeper/src/main/java/org/example/core/taskcenter/TaskCenter.java b/chopperbot-creeper/src/main/java/org/example/core/taskcenter/TaskCenter.java index c01e664..a524b35 100644 --- a/chopperbot-creeper/src/main/java/org/example/core/taskcenter/TaskCenter.java +++ b/chopperbot-creeper/src/main/java/org/example/core/taskcenter/TaskCenter.java @@ -10,11 +10,10 @@ import org.example.config.TaskCenterConfig; import org.example.constpool.PluginName; import org.example.core.loadtask.LoadTask; import org.example.core.manager.CreeperManager; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; import org.example.core.taskcenter.task.TaskRecord; import org.example.core.taskcenter.task.TaskStatus; import org.example.exception.FileCacheException; -import org.example.init.InitPluginRegister; import org.example.log.ChopperLogFactory; import org.example.log.LoggerType; import org.example.plugin.GuardPlugin; @@ -223,7 +222,7 @@ public class TaskCenter extends GuardPlugin { creeperLogFileCache.writeKeys(taskRecord,"task",String.valueOf(i)); waitingTask.put(reptileTask); } catch (InterruptedException |FileCacheException e) { - return false; + throw new RuntimeException(e); } } } diff --git a/chopperbot-creeper/src/main/java/org/example/exception/TaskCenterException.java b/chopperbot-creeper/src/main/java/org/example/exception/TaskCenterException.java index 40ceb36..257105e 100644 --- a/chopperbot-creeper/src/main/java/org/example/exception/TaskCenterException.java +++ b/chopperbot-creeper/src/main/java/org/example/exception/TaskCenterException.java @@ -11,7 +11,8 @@ public class TaskCenterException extends RuntimeException{ this.msg = msg; } - public String getMsg() { + @Override + public String getMessage() { return msg; } } diff --git a/chopperbot-hot/src/main/java/org/example/config/HotModuleConfig.java b/chopperbot-hot/src/main/java/org/example/config/HotModuleConfig.java index d81010a..7157f65 100644 --- a/chopperbot-hot/src/main/java/org/example/config/HotModuleConfig.java +++ b/chopperbot-hot/src/main/java/org/example/config/HotModuleConfig.java @@ -35,7 +35,15 @@ public class HotModuleConfig extends ConfigFile> { new HotModuleSetting(CreeperModuleConstPool.DOUYING, 2,true,true,true, new ArrayList<>(), false, List.of(allLiveDog()), OneDay, FiveMinute) ), - "GuardNum",10 + "GuardNum",10, + "HeatRecommendation",Map.of( + "recommendation_creeper",Map.of( + "douyu_live","douyu_live", + "douyu_live_barrage","douyu_live_barrage", + "bilibili_live","bilibili_live", + "bilibili_live_barrage","bilibili_live_barrage" + ) + ) )); } diff --git a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotLiveConfig.java b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotLiveConfig.java index 8a6926b..b052fab 100644 --- a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotLiveConfig.java +++ b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotLiveConfig.java @@ -1,8 +1,7 @@ package org.example.core.creeper.loadconfig; import org.example.core.creeper.loadtask.BiliBiliHotLiveLoadTask; -import org.example.core.creeper.loadtask.BilibiliHotModuleLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** * @author Genius diff --git a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotModuleConfig.java b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotModuleConfig.java index c1ab87c..52a171f 100644 --- a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotModuleConfig.java +++ b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/BilibiliHotModuleConfig.java @@ -1,8 +1,7 @@ package org.example.core.creeper.loadconfig; import org.example.core.creeper.loadtask.BilibiliHotModuleLoadTask; -import org.example.core.creeper.loadtask.DouyuHotLiveLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** * @author Genius diff --git a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotLiveConfig.java b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotLiveConfig.java index f7144fc..3ce697e 100644 --- a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotLiveConfig.java +++ b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotLiveConfig.java @@ -1,7 +1,7 @@ package org.example.core.creeper.loadconfig; import org.example.core.creeper.loadtask.DouyuHotLiveLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** diff --git a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotModuleConfig.java b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotModuleConfig.java index 86f5a42..903dc5b 100644 --- a/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotModuleConfig.java +++ b/chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyuHotModuleConfig.java @@ -1,7 +1,7 @@ package org.example.core.creeper.loadconfig; import org.example.core.creeper.loadtask.DouyuHotModuleLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** diff --git a/chopperbot-hot/src/main/java/org/example/core/recommend/HeatRecommendation.java b/chopperbot-hot/src/main/java/org/example/core/recommend/HeatRecommendation.java index 21daf32..ea476aa 100644 --- a/chopperbot-hot/src/main/java/org/example/core/recommend/HeatRecommendation.java +++ b/chopperbot-hot/src/main/java/org/example/core/recommend/HeatRecommendation.java @@ -5,9 +5,11 @@ package org.example.core.recommend; * @date 2023/07/22 23:20 **/ +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson2.util.TypeUtils; +import lombok.Data; import org.example.bean.HotModule; import org.example.bean.Live; import org.example.cache.FileCacheManagerInstance; @@ -17,14 +19,10 @@ import org.example.config.HotModuleSetting; import org.example.constpool.PluginName; import org.example.core.HotModuleDataCenter; import org.example.core.creeper.loadconfig.*; -import org.example.core.factory.BarrageLoadConfigFactory; -import org.example.core.factory.LiveLoadConfigFactory; +import org.example.core.manager.CreeperManager; import org.example.core.taskcenter.request.ReptileRequest; import org.example.core.taskcenter.task.TaskRecord; import org.example.init.InitPluginRegister; -import org.example.log.ChopperLogFactory; -import org.example.log.LoggerType; -import org.example.plugin.CommonPlugin; import org.example.plugin.GuardPlugin; import org.example.core.taskcenter.TaskCenter; import org.example.plugin.PluginCheckAndDo; @@ -39,14 +37,19 @@ import java.util.regex.Pattern; /** * 根据热门的直播以及看门狗设置来推荐热门直播间到系统 */ +@Data public class HeatRecommendation extends GuardPlugin { + private static final long serialVersionUID = 4749216808636623354L; private Map> platformFollowDogMap ; //每个平台的跟风列表 private BlockingQueue hotEventList; //用于接收每一次的热榜更新信息，并进行跟风狗推送 + private Map recommendationCreeper; private static final String SHUTDOWN_SIGN = "shutdown"; + + public HeatRecommendation(String module, String pluginName, List needPlugins, boolean isAutoStart) { super(module, pluginName, needPlugins, isAutoStart); } @@ -55,7 +58,9 @@ public class HeatRecommendation extends GuardPlugin { public boolean init() { try { platformFollowDogMap = new ConcurrentHashMap<>(); + recommendationCreeper = new HashMap<>(); hotEventList = new ArrayBlockingQueue<>(1024); + List modules = new ArrayList<>(); JSONArray jsonModules = (JSONArray) FileCacheManagerInstance @@ -63,6 +68,15 @@ public class HeatRecommendation extends GuardPlugin { .getFileCache(HotModuleConfig.getFullFilePath()) .get("Module"); + JSONObject jsonCreepers = (JSONObject) FileCacheManagerInstance + .getInstance() + .getFileCache(HotModuleConfig.getFullFilePath()) + .get("HeatRecommendation","recommendation_creeper"); + + for (String s : jsonCreepers.keySet()) { + recommendationCreeper.put(s,jsonCreepers.getString(s)); + } + jsonModules.forEach(jsonModule->{ modules.add(JSONObject.parseObject(jsonModule.toString(),HotModuleSetting.class)); }); @@ -107,27 +121,47 @@ public class HeatRecommendation extends GuardPlugin { logger.error("[{}] cannot found {} {} hot lives",PluginName.HOT_RECOMMENDATION_PLUGIN,platform,moduleName); break; } - //TODO 待完善 1，需要发送弹幕爬虫请求 2，callback更改 for (Live live : needRecommend(tempLives, followDog.getBanLiver(), followDog.getTop())) { String tempPlatform = live.getPlatform(); this.info(String.format("推荐请求:平台 %s,分区 %s,直播间 %s,主播 %s",tempPlatform,live.getModuleName(),live.getLiveId(),live.getLiver())); -// LoadLiveConfig loadLiveConfig = LiveLoadConfigFactory.buildLiveConfig( -// tempPlatform, live.getLiveId(), live.getLiver(), -// true, true); -// -//// LoadBarrageConfig loadBarrageConfig = BarrageLoadConfigFactory.buildBarrageConfig(tempPlatform, -//// live.getLiver(), live.getLiveId()); -// PluginCheckAndDo.CheckAndDo( -// plugin -> { -// ((TaskCenter)plugin).request(new ReptileRequest(loadLiveConfig,(t)->{ -// System.out.println(String.format("%s 文件已保存", t)); -// })); -//// ((TaskCenter)plugin).request(new ReptileRequest(loadBarrageConfig,(t)->{ -//// System.out.println(String.format("%s 爬虫任务已结束", live.getLiver())); -//// })); -// }, -// PluginName.TASK_CENTER_PLUGIN -// ); + String liveCreeper = tempPlatform+"_live"; + String barrageCreeper = tempPlatform+"_live_barrage"; + if (recommendationCreeper.containsKey(liveCreeper)) { + PluginCheckAndDo.CheckAndDo( + taskCenter -> { + CreeperManager plugin = InitPluginRegister.getPlugin(PluginName.CREEPER_MANAGER_PLUGIN, + CreeperManager.class); + assert plugin != null; + LoadLiveConfig loadLiveConfig = plugin.buildLoadConfig(recommendationCreeper.get(liveCreeper),live); + if(loadLiveConfig!=null){ + ((TaskCenter)taskCenter).request(new ReptileRequest(loadLiveConfig,(t)->{ + System.out.printf("%s 文件已保存%n", t); + })); + }else{ + this.logger.error("[{}] {} fail found creeper shell，cannot recommend!",this.getPluginName(),liveCreeper); + } + }, + PluginName.TASK_CENTER_PLUGIN + ); + } + if (recommendationCreeper.containsKey(barrageCreeper)) { + PluginCheckAndDo.CheckAndDo( + taskCenter -> { + CreeperManager plugin = InitPluginRegister.getPlugin(PluginName.CREEPER_MANAGER_PLUGIN, + CreeperManager.class); + assert plugin != null; + LoadBarrageConfig loadBarrageConfig = plugin.buildLoadConfig(recommendationCreeper.get(barrageCreeper),live); + if(loadBarrageConfig!=null){ + ((TaskCenter)taskCenter).request(new ReptileRequest(loadBarrageConfig,(t)->{ + System.out.printf("%s 文件已保存%n", t); + })); + }else{ + this.logger.error("[{}] {} fail found creeper shell，cannot recommend!",this.getPluginName(),barrageCreeper); + } + }, + PluginName.TASK_CENTER_PLUGIN + ); + } } } } @@ -182,7 +216,4 @@ public class HeatRecommendation extends GuardPlugin { hotEventList.offer(platform); } - public static void main(String[] args) { - System.out.println(TypeUtils.isProxy(TaskRecord.class)); - } } diff --git a/chopperbot-hot/src/main/java/org/example/init/HeatRecommendationInitMachine.java b/chopperbot-hot/src/main/java/org/example/init/HeatRecommendationInitMachine.java index d6bf7b1..4034ee6 100644 --- a/chopperbot-hot/src/main/java/org/example/init/HeatRecommendationInitMachine.java +++ b/chopperbot-hot/src/main/java/org/example/init/HeatRecommendationInitMachine.java @@ -18,7 +18,7 @@ import java.util.List; pluginName = PluginName.HOT_RECOMMENDATION_PLUGIN, pluginName_CN = "热门推荐插件", pluginDescription = "推荐各个平台的热门直播，根据跟风狗设置，自动爬取直播", - needPlugin = {PluginName.HOT_CONFIG_PLUGIN,PluginName.HOT_GUARD_PLUGIN}, + needPlugin = {PluginName.HOT_CONFIG_PLUGIN,PluginName.HOT_GUARD_PLUGIN,PluginName.CREEPER_MANAGER_PLUGIN}, pluginClass= HeatRecommendation.class ) public class HeatRecommendationInitMachine extends CommonInitMachine{ diff --git a/chopperbot-live/src/main/java/org/example/core/creeper/builder/BiliBiliLiveLoadConfigBuilder.java b/chopperbot-live/src/main/java/org/example/core/creeper/builder/BiliBiliLiveLoadConfigBuilder.java new file mode 100644 index 0000000..a83b377 --- /dev/null +++ b/chopperbot-live/src/main/java/org/example/core/creeper/builder/BiliBiliLiveLoadConfigBuilder.java @@ -0,0 +1,29 @@ +package org.example.core.creeper.builder; + +import org.example.bean.Live; +import org.example.bean.live.BiliBiliLive; +import org.example.constpool.ConstPool; +import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig; +import org.example.core.creeper.loadconfig.LoadLiveConfig; +import org.example.core.manager.CommonLoadConfigBuilder; +import org.example.pool.LiveModuleConstPool; + +/** + * @author Genius + * @date 2023/09/05 19:18 + **/ +public class BiliBiliLiveLoadConfigBuilder extends CommonLoadConfigBuilder { + @Override + public BilibiliLiveOnlineConfig build(Object obj) { + if(obj instanceof BiliBiliLive){ + String liveId = ((BiliBiliLive) obj).getLiveId(); + String liver = ((BiliBiliLive) obj).getLiver(); + String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.BILIBILI); + BilibiliLiveOnlineConfig bilibiliLiveOnlineConfig = new BilibiliLiveOnlineConfig(liveId, path, LoadLiveConfig.fileName(liveId, liver), true); + bilibiliLiveOnlineConfig.setShowDownloadTable(true); + bilibiliLiveOnlineConfig.setLiverName(liver); + return bilibiliLiveOnlineConfig; + } + return null; + } +} diff --git a/chopperbot-live/src/main/java/org/example/core/creeper/builder/DouyuLiveLoadConfigBuilder.java b/chopperbot-live/src/main/java/org/example/core/creeper/builder/DouyuLiveLoadConfigBuilder.java new file mode 100644 index 0000000..6525140 --- /dev/null +++ b/chopperbot-live/src/main/java/org/example/core/creeper/builder/DouyuLiveLoadConfigBuilder.java @@ -0,0 +1,32 @@ +package org.example.core.creeper.builder; + +import org.example.bean.live.BiliBiliLive; +import org.example.bean.live.DouyuLive; +import org.example.constpool.ConstPool; +import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig; +import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig; +import org.example.core.creeper.loadconfig.LoadLiveConfig; +import org.example.core.manager.CommonLoadConfigBuilder; +import org.example.pool.LiveModuleConstPool; + +/** + * @author Genius + * @date 2023/09/05 19:27 + **/ +public class DouyuLiveLoadConfigBuilder extends CommonLoadConfigBuilder { + + @Override + public DouyuLiveOnlineConfig build(Object obj) { + if(obj instanceof DouyuLive){ + String liveId = ((DouyuLive) obj).getLiveId(); + String liver = ((DouyuLive) obj).getLiver(); + String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.BILIBILI); + DouyuLiveOnlineConfig douyuLiveOnlineConfig = new DouyuLiveOnlineConfig(liveId, path, LoadLiveConfig.fileName(liveId, liver), true); + + douyuLiveOnlineConfig.setShowDownloadTable(true); + douyuLiveOnlineConfig.setLiverName(liver); + return douyuLiveOnlineConfig; + } + return null; + } +} diff --git a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/BilibiliLiveOnlineConfig.java b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/BilibiliLiveOnlineConfig.java index 21a50ae..8804acb 100644 --- a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/BilibiliLiveOnlineConfig.java +++ b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/BilibiliLiveOnlineConfig.java @@ -1,17 +1,19 @@ package org.example.core.creeper.loadconfig; import lombok.Data; +import org.example.core.creeper.builder.BiliBiliLiveLoadConfigBuilder; import org.example.core.creeper.loadtask.BilibiliLiveOnlineLoadTask; -import org.example.core.creeper.loadtask.DouyuLiveOnlineLoadTask; -import org.example.core.creeper.loadtask.DouyuRecordLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** * @author Genius * @date 2023/08/30 18:05 **/ @Data -@Creeper(creeperName = "bilibili_live",loadTask = BilibiliLiveOnlineLoadTask.class,creeperDescription = "B站直播爬取") +@Creeper(creeperName = "bilibili_live", + loadTask = BilibiliLiveOnlineLoadTask.class, + builder = BiliBiliLiveLoadConfigBuilder.class, + creeperDescription = "B站直播爬取") public class BilibiliLiveOnlineConfig extends LoadLiveConfig{ diff --git a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuLiveOnlineConfig.java b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuLiveOnlineConfig.java index 89f858b..df2f0e0 100644 --- a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuLiveOnlineConfig.java +++ b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuLiveOnlineConfig.java @@ -1,17 +1,19 @@ package org.example.core.creeper.loadconfig; import lombok.Data; -import org.example.bean.live.DouyuLive; +import org.example.core.creeper.builder.DouyuLiveLoadConfigBuilder; import org.example.core.creeper.loadtask.DouyuLiveOnlineLoadTask; -import org.example.core.creeper.loadtask.DouyuRecordLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** * @author Genius * @date 2023/07/28 23:17 **/ @Data -@Creeper(creeperName = "douyu_live",loadTask = DouyuLiveOnlineLoadTask.class,creeperDescription = "斗鱼直播爬取") +@Creeper(creeperName = "douyu_live", + loadTask = DouyuLiveOnlineLoadTask.class, + builder = DouyuLiveLoadConfigBuilder.class, + creeperDescription = "斗鱼直播爬取") public class DouyuLiveOnlineConfig extends LoadLiveConfig { public DouyuLiveOnlineConfig(String roomId, String videoPath, String videoName,int clarity) { super(roomId, videoPath, videoName, false); diff --git a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuRecordConfig.java b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuRecordConfig.java index f721c02..40f77a8 100644 --- a/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuRecordConfig.java +++ b/chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyuRecordConfig.java @@ -1,9 +1,8 @@ package org.example.core.creeper.loadconfig; import lombok.Data; -import org.example.core.VideoType; import org.example.core.creeper.loadtask.DouyuRecordLoadTask; -import org.example.core.manager.annotation.Creeper; +import org.example.core.manager.Creeper; /** * @author Genius diff --git a/chopperbot-live/src/main/java/org/example/core/factory/LiveLoadConfigFactory.java b/chopperbot-live/src/main/java/org/example/core/factory/LiveLoadConfigFactory.java deleted file mode 100644 index 0883cc2..0000000 --- a/chopperbot-live/src/main/java/org/example/core/factory/LiveLoadConfigFactory.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.example.core.factory; - -import org.example.constpool.ConstPool; -import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig; -import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig; -import org.example.core.creeper.loadconfig.LoadLiveConfig; -import org.example.pool.LiveModuleConstPool; - -import javax.swing.*; -import java.awt.*; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.io.PrintWriter; - - -/** - * @author Genius - * @date 2023/08/31 23:57 - **/ -public class LiveLoadConfigFactory { - - public static T buildLiveConfig(String platform, String roomId, String liver, - boolean convertMp4, boolean showDownloadTable){ - platform = platform.toLowerCase(); - - if(platform.equals(ConstPool.PLATFORM.DOUYU.getName())){ - String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.DOUYU); - DouyuLiveOnlineConfig config = new DouyuLiveOnlineConfig(roomId, path, LoadLiveConfig.fileName(roomId, liver), convertMp4); - config.setShowDownloadTable(showDownloadTable); - config.setLiverName(liver); - return (T) config; - }else if(platform.equals(ConstPool.PLATFORM.BILIBILI.getName())){ - String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.BILIBILI); - BilibiliLiveOnlineConfig config = new BilibiliLiveOnlineConfig(roomId, path, LoadLiveConfig.fileName(roomId, liver), convertMp4); - config.setShowDownloadTable(showDownloadTable); - config.setLiverName(liver); - return (T) config; - } - return null; - } - - public static T buildLiveConfig(String platform, String roomId,String liver, - int clarity,boolean showDownloadTable){ - platform = platform.toLowerCase(); - if(platform.equals(ConstPool.PLATFORM.DOUYU.getName())){ - String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.DOUYU); - DouyuLiveOnlineConfig config = new DouyuLiveOnlineConfig(roomId, path, LoadLiveConfig.fileName(roomId, liver), clarity); - config.setShowDownloadTable(showDownloadTable); - config.setLiverName(liver); - return (T) config; - }else if(platform.equals(ConstPool.PLATFORM.BILIBILI.getName())){ - String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.BILIBILI); - BilibiliLiveOnlineConfig config = new BilibiliLiveOnlineConfig(roomId, path, LoadLiveConfig.fileName(roomId, liver), clarity); - config.setShowDownloadTable(showDownloadTable); - config.setLiverName(liver); - return (T) config; - } - return null; - } - -} diff --git a/chopperbot-test/pom.xml b/chopperbot-test/pom.xml index 1c841a4..8d4af5e 100644 --- a/chopperbot-test/pom.xml +++ b/chopperbot-test/pom.xml @@ -41,5 +41,11 @@ spring-boot-starter-test test + + + org.java-websocket + Java-WebSocket + 1.5.1 + diff --git a/chopperbot-test/src/test/java/org/example/WebSocketTest.java b/chopperbot-test/src/test/java/org/example/WebSocketTest.java new file mode 100644 index 0000000..b8d7de9 --- /dev/null +++ b/chopperbot-test/src/test/java/org/example/WebSocketTest.java @@ -0,0 +1,109 @@ +package org.example; + +import cn.hutool.http.ssl.DefaultSSLFactory; +import org.example.bean.barrage.DouyuBarrage; +import org.java_websocket.client.WebSocketClient; + +import org.java_websocket.handshake.ServerHandshake; +import org.junit.Test; +import org.example.ws.Draft_6455; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +import static org.example.ws.DouyuProtoctl.*; + + +/** + * @author Genius + * @date 2023/09/04 19:53 + **/ +public class WebSocketTest { + private static WebSocketClient client; // 类级别的WebSocketClient变量 + + private static String roomId = "25214"; + public static void main(String[] args) throws InterruptedException { + String serverUri = "wss://danmuproxy.douyu.com:8505/"; // WebSocket服务器的地址 + long startTime = System.currentTimeMillis(); + try { + client = new WebSocketClient(new URI(serverUri), new Draft_6455()) { + @Override + public void onOpen(ServerHandshake handshakedata) { + System.out.println("WebSocket connection opened"); + // 发送二进制数据 + client.send(loginReq(roomId,"visitor4090855","1111387111","20220825","218101901","0")); + client.send(joinGroup(roomId,"1")); + client.send(heartBeat()); + } + + @Override + public void onMessage(String s) { + System.out.println("Received binary data: " + s); + } + + @Override + public void onMessage(ByteBuffer buffer) { + // 处理二进制消息 + DouyuBarrage douyuBarrage = wrapperBarrage(buffer, startTime); + if(douyuBarrage!=null){ + System.out.println(douyuBarrage); + } + } + + @Override + public void onClose(int code, String reason, boolean remote) { + client.close(); + System.out.println(code); + System.out.println(reason); + System.out.println("WebSocket connection closed"); + } + + @Override + public void onError(Exception ex) { + ex.printStackTrace(); + } + }; + SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + client.setSocketFactory(sslSocketFactory); + client.connect(); // 连接到WebSocket服务器 + } catch (URISyntaxException e) { + e.printStackTrace(); + } + while (true){ + Thread.sleep(60*1000); + System.out.println("heartBeat"); + client.send(heartBeat()); + if(client.isClosed()){ + break; + } + } + } + + // 将十六进制字符串转换为字节数组 + private static byte[] hexStringToByteArray(String hex) { + int len = hex.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4) + + Character.digit(hex.charAt(i + 1), 16)); + } + return data; + } + + // 发送二进制数据 + private static void sendBinaryData(byte[] data) { + ByteBuffer buffer = ByteBuffer.wrap(data); + getClient().send(buffer); + } + + private static WebSocketClient getClient() { + return client; + } + // 发送二进制数据 +} + diff --git a/chopperbot-test/src/test/java/org/example/barrage/BarrageCreeperTest.java b/chopperbot-test/src/test/java/org/example/barrage/BarrageCreeperTest.java index 4bed41d..22107a2 100644 --- a/chopperbot-test/src/test/java/org/example/barrage/BarrageCreeperTest.java +++ b/chopperbot-test/src/test/java/org/example/barrage/BarrageCreeperTest.java @@ -1,9 +1,9 @@ package org.example.barrage; import org.example.ConsoleApplication; +import org.example.bean.barrage.BilibiliBarrage; import org.example.core.creeper.loadconfig.BilibiliLiveLoadBarrageConfig; -import org.example.core.creeper.loadtask.BilibiliBarrageLiveLoadTask; -import org.example.pojo.Barrage; +import org.example.core.creeper.loadtask.BilibiliLiveBarrageLoadTask; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; @@ -22,8 +22,8 @@ public class BarrageCreeperTest { @Test public void testBilibiliBarrageCreeper(){ - BilibiliLiveLoadBarrageConfig config = new BilibiliLiveLoadBarrageConfig("猪猪女孩", "71002"); - List start = new BilibiliBarrageLiveLoadTask(config).start(); + BilibiliLiveLoadBarrageConfig config = new BilibiliLiveLoadBarrageConfig("某幻君", "271744"); + List start = new BilibiliLiveBarrageLoadTask(config).start(); } } diff --git a/chopperbot-test/src/test/java/org/example/ws/DouyuProtoctl.java b/chopperbot-test/src/test/java/org/example/ws/DouyuProtoctl.java new file mode 100644 index 0000000..c2518e5 --- /dev/null +++ b/chopperbot-test/src/test/java/org/example/ws/DouyuProtoctl.java @@ -0,0 +1,121 @@ +package org.example.ws; + +import org.example.bean.barrage.DouyuBarrage; +import org.example.util.TimeUtil; +import org.openqa.selenium.By; + +import javax.xml.bind.DatatypeConverter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * @author Genius + * @date 2023/09/04 20:26 + **/ +public class DouyuProtoctl { + + public static byte[] loginReq(String roomId,String username,String uid,String ver,String aver,String ct){ + ByteBuffer buffer = ByteBuffer.allocate(1024); + String head = "8d0000008d000000b1020000"; + buffer.put(DatatypeConverter.parseHexBinary(head)); + String content = String.format("type@=loginreq/roomid@=%s/dfl@=sn/username@=%s@AA=105@ASss@AA=1/uid@=%s/ver@=%s/aver@=%s/ct@=%s/\0", + roomId,username,uid,ver,aver,ct); + buffer.put(content.getBytes()); + buffer.flip(); + byte[] byteArray = new byte[buffer.remaining()]; + // 将ByteBuffer中的内容复制到byte数组中 + buffer.get(byteArray); + return byteArray; + } + + public static byte[] joinGroup(String roomId,String gid){ + ByteBuffer buffer = ByteBuffer.allocate(1024); + String head = "2c0000002c000000b1020000"; + buffer.put(DatatypeConverter.parseHexBinary(head)); + String content = String.format("type@=joingroup/rid@=%s/gid@=%s/\0",roomId,gid); + buffer.put(content.getBytes()); + buffer.flip(); + byte[] byteArray = new byte[buffer.remaining()]; + // 将ByteBuffer中的内容复制到byte数组中 + buffer.get(byteArray); + return byteArray; + } + + public static byte[] heartBeat(){ + ByteBuffer buffer = ByteBuffer.allocate(1024); + String head = "1400000014000000b1020000"; + buffer.put(DatatypeConverter.parseHexBinary(head)); + String content = "type@=mrkl/\0"; + buffer.put(content.getBytes()); + buffer.flip(); + byte[] byteArray = new byte[buffer.remaining()]; + buffer.get(byteArray); + return byteArray; + } + + public static ByteBuffer heartBeat2(){ + ByteBuffer buffer = ByteBuffer.allocate(1024); + String head = "1400000014000000b1020000"; + buffer.put(DatatypeConverter.parseHexBinary(head)); + String content = "type@=mrkl/\0"; + buffer.put(content.getBytes()); + buffer.flip(); + return buffer; + } + + public static DouyuBarrage wrapperBarrage(ByteBuffer buffer,long startTime){ + buffer.position(buffer.position()+12); + byte[] remainingBytes = new byte[buffer.remaining()]; + buffer.get(remainingBytes); + String content = new String(remainingBytes,StandardCharsets.UTF_8); + String[] split = content.split("/"); + if(split.length>0){ + String type = split[0]; + if(type.contains("chatmsg")){ + String txt = ""; + String mid = ""; + long timeReal = 0; + long timeIndex = 0; + for(int i=1;i knownExtensions; + private IProtocol protocol; + private List knownProtocols; + private Framedata currentContinuousFrame; + private final List byteBufferList; + private ByteBuffer incompleteframe; + private final Random reuseableRandom; + private int maxFrameSize; + + public Draft_6455() { + this(Collections.emptyList()); + } + + public Draft_6455(IExtension inputExtension) { + this(Collections.singletonList(inputExtension)); + } + + public Draft_6455(List inputExtensions) { + this(inputExtensions, Collections.singletonList(new Protocol(""))); + } + + public Draft_6455(List inputExtensions, List inputProtocols) { + this(inputExtensions, inputProtocols, Integer.MAX_VALUE); + } + + public Draft_6455(List inputExtensions, int inputMaxFrameSize) { + this(inputExtensions, Collections.singletonList(new Protocol("")), inputMaxFrameSize); + } + + public Draft_6455(List inputExtensions, List inputProtocols, int inputMaxFrameSize) { + this.log = LoggerFactory.getLogger(Draft_6455.class); + this.extension = new DefaultExtension(); + this.reuseableRandom = new Random(); + if (inputExtensions != null && inputProtocols != null && inputMaxFrameSize >= 1) { + this.knownExtensions = new ArrayList(inputExtensions.size()); + this.knownProtocols = new ArrayList(inputProtocols.size()); + boolean hasDefault = false; + this.byteBufferList = new ArrayList(); + Iterator var5 = inputExtensions.iterator(); + + while(var5.hasNext()) { + IExtension inputExtension = (IExtension)var5.next(); + if (inputExtension.getClass().equals(DefaultExtension.class)) { + hasDefault = true; + } + } + + this.knownExtensions.addAll(inputExtensions); + if (!hasDefault) { + this.knownExtensions.add(this.knownExtensions.size(), this.extension); + } + + this.knownProtocols.addAll(inputProtocols); + this.maxFrameSize = inputMaxFrameSize; + } else { + throw new IllegalArgumentException(); + } + } + + public HandshakeState acceptHandshakeAsServer(ClientHandshake handshakedata) throws InvalidHandshakeException { + + HandshakeState extensionState = HandshakeState.NOT_MATCHED; + String requestedExtension = handshakedata.getFieldValue("Sec-WebSocket-Extensions"); + Iterator var5 = this.knownExtensions.iterator(); + + while(var5.hasNext()) { + IExtension knownExtension = (IExtension)var5.next(); + if (knownExtension.acceptProvidedExtensionAsServer(requestedExtension)) { + this.extension = knownExtension; + extensionState = HandshakeState.MATCHED; + this.log.trace("acceptHandshakeAsServer - Matching extension found: {}", this.extension); + break; + } + } + + HandshakeState protocolState = this.containsRequestedProtocol(handshakedata.getFieldValue("Sec-WebSocket-Protocol")); + if (protocolState == HandshakeState.MATCHED && extensionState == HandshakeState.MATCHED) { + return HandshakeState.MATCHED; + } else { + this.log.trace("acceptHandshakeAsServer - No matching extension or protocol found."); + return HandshakeState.NOT_MATCHED; + } + + } + + private HandshakeState containsRequestedProtocol(String requestedProtocol) { + Iterator var2 = this.knownProtocols.iterator(); + + IProtocol knownProtocol; + do { + if (!var2.hasNext()) { + return HandshakeState.NOT_MATCHED; + } + + knownProtocol = (IProtocol)var2.next(); + } while(!knownProtocol.acceptProvidedProtocol(requestedProtocol)); + + this.protocol = knownProtocol; + this.log.trace("acceptHandshake - Matching protocol found: {}", this.protocol); + return HandshakeState.MATCHED; + } + + public HandshakeState acceptHandshakeAsClient(ClientHandshake request, ServerHandshake response) throws InvalidHandshakeException { + if (!this.basicAccept(response)) { + this.log.trace("acceptHandshakeAsClient - Missing/wrong upgrade or connection in handshake."); + return HandshakeState.NOT_MATCHED; + } else if (request.hasFieldValue("Sec-WebSocket-Key") && response.hasFieldValue("Sec-WebSocket-Accept")) { + String seckeyAnswer = response.getFieldValue("Sec-WebSocket-Accept"); + String seckeyChallenge = request.getFieldValue("Sec-WebSocket-Key"); + seckeyChallenge = this.generateFinalKey(seckeyChallenge); + if (!seckeyChallenge.equals(seckeyAnswer)) { + this.log.trace("acceptHandshakeAsClient - Wrong key for Sec-WebSocket-Key."); + return HandshakeState.NOT_MATCHED; + } else { + HandshakeState extensionState = HandshakeState.NOT_MATCHED; + String requestedExtension = response.getFieldValue("Sec-WebSocket-Extensions"); + Iterator var7 = this.knownExtensions.iterator(); + + while(var7.hasNext()) { + IExtension knownExtension = (IExtension)var7.next(); + if (knownExtension.acceptProvidedExtensionAsClient(requestedExtension)) { + this.extension = knownExtension; + extensionState = HandshakeState.MATCHED; + this.log.trace("acceptHandshakeAsClient - Matching extension found: {}", this.extension); + break; + } + } + + HandshakeState protocolState = this.containsRequestedProtocol(response.getFieldValue("Sec-WebSocket-Protocol")); + if (protocolState == HandshakeState.MATCHED && extensionState == HandshakeState.MATCHED) { + return HandshakeState.MATCHED; + } else { + this.log.trace("acceptHandshakeAsClient - No matching extension or protocol found."); + return HandshakeState.NOT_MATCHED; + } + } + } else { + this.log.trace("acceptHandshakeAsClient - Missing Sec-WebSocket-Key or Sec-WebSocket-Accept"); + return HandshakeState.NOT_MATCHED; + } + } + + public IExtension getExtension() { + return this.extension; + } + + public List getKnownExtensions() { + return this.knownExtensions; + } + + public IProtocol getProtocol() { + return this.protocol; + } + + public int getMaxFrameSize() { + return this.maxFrameSize; + } + + public List getKnownProtocols() { + return this.knownProtocols; + } + + public ClientHandshakeBuilder postProcessHandshakeRequestAsClient(ClientHandshakeBuilder request) { + request.put("Upgrade", "websocket"); + request.put("Connection", "Upgrade"); + byte[] random = new byte[16]; + this.reuseableRandom.nextBytes(random); + request.put("Sec-WebSocket-Key", Base64.encodeBytes(random)); + request.put("Sec-WebSocket-Version", "13"); + StringBuilder requestedExtensions = new StringBuilder(); + Iterator var4 = this.knownExtensions.iterator(); + + while(var4.hasNext()) { + IExtension knownExtension = (IExtension)var4.next(); + if (knownExtension.getProvidedExtensionAsClient() != null && knownExtension.getProvidedExtensionAsClient().length() != 0) { + if (requestedExtensions.length() > 0) { + requestedExtensions.append(", "); + } + + requestedExtensions.append(knownExtension.getProvidedExtensionAsClient()); + } + } + + if (requestedExtensions.length() != 0) { + request.put("Sec-WebSocket-Extensions", requestedExtensions.toString()); + } + + StringBuilder requestedProtocols = new StringBuilder(); + Iterator var8 = this.knownProtocols.iterator(); + + while(var8.hasNext()) { + IProtocol knownProtocol = (IProtocol)var8.next(); + if (knownProtocol.getProvidedProtocol().length() != 0) { + if (requestedProtocols.length() > 0) { + requestedProtocols.append(", "); + } + + requestedProtocols.append(knownProtocol.getProvidedProtocol()); + } + } + + if (requestedProtocols.length() != 0) { + request.put("Sec-WebSocket-Protocol", requestedProtocols.toString()); + } + + return request; + } + + public HandshakeBuilder postProcessHandshakeResponseAsServer(ClientHandshake request, ServerHandshakeBuilder response) throws InvalidHandshakeException { + response.put("Upgrade", "websocket"); + response.put("Connection", request.getFieldValue("Connection")); + String seckey = request.getFieldValue("Sec-WebSocket-Key"); + if (seckey == null) { + throw new InvalidHandshakeException("missing Sec-WebSocket-Key"); + } else { + response.put("Sec-WebSocket-Accept", this.generateFinalKey(seckey)); + if (this.getExtension().getProvidedExtensionAsServer().length() != 0) { + response.put("Sec-WebSocket-Extensions", this.getExtension().getProvidedExtensionAsServer()); + } + + if (this.getProtocol() != null && this.getProtocol().getProvidedProtocol().length() != 0) { + response.put("Sec-WebSocket-Protocol", this.getProtocol().getProvidedProtocol()); + } + + response.setHttpStatusMessage("Web Socket Protocol Handshake"); + response.put("Server", "TooTallNate Java-WebSocket"); + response.put("Date", this.getServerTime()); + return response; + } + } + + public Draft copyInstance() { + ArrayList newExtensions = new ArrayList(); + Iterator var2 = this.getKnownExtensions().iterator(); + + while(var2.hasNext()) { + IExtension iExtension = (IExtension)var2.next(); + newExtensions.add(iExtension.copyInstance()); + } + + ArrayList newProtocols = new ArrayList(); + Iterator var6 = this.getKnownProtocols().iterator(); + + while(var6.hasNext()) { + IProtocol iProtocol = (IProtocol)var6.next(); + newProtocols.add(iProtocol.copyInstance()); + } + + return new Draft_6455(newExtensions, newProtocols, this.maxFrameSize); + } + + public ByteBuffer createBinaryFrame(Framedata framedata) { + this.getExtension().encodeFrame(framedata); + if (this.log.isTraceEnabled()) { + this.log.trace("afterEnconding({}): {}", framedata.getPayloadData().remaining(), framedata.getPayloadData().remaining() > 1000 ? "too big to display" : new String(framedata.getPayloadData().array())); + } + + return this.createByteBufferFromFramedata(framedata); + } + + private ByteBuffer createByteBufferFromFramedata(Framedata framedata) { + ByteBuffer mes = framedata.getPayloadData(); + boolean mask = this.role == Role.CLIENT; + int sizebytes = this.getSizeBytes(mes); + ByteBuffer buf = ByteBuffer.allocate(1 + (sizebytes > 1 ? sizebytes + 1 : sizebytes) + (mask ? 4 : 0) + mes.remaining()); + byte optcode = this.fromOpcode(framedata.getOpcode()); + byte one = (byte)(framedata.isFin() ? -128 : 0); + one |= optcode; + if (framedata.isRSV1()) { + one |= this.getRSVByte(1); + } + + if (framedata.isRSV2()) { + one |= this.getRSVByte(2); + } + + if (framedata.isRSV3()) { + one |= this.getRSVByte(3); + } + + buf.put(one); + byte[] payloadlengthbytes = this.toByteArray((long)mes.remaining(), sizebytes); + + assert payloadlengthbytes.length == sizebytes; + + if (sizebytes == 1) { + buf.put((byte)(payloadlengthbytes[0] | this.getMaskByte(mask))); + } else if (sizebytes == 2) { + buf.put((byte)(126 | this.getMaskByte(mask))); + buf.put(payloadlengthbytes); + } else { + if (sizebytes != 8) { + throw new IllegalStateException("Size representation not supported/specified"); + } + + buf.put((byte)(127 | this.getMaskByte(mask))); + buf.put(payloadlengthbytes); + } + + if (mask) { + ByteBuffer maskkey = ByteBuffer.allocate(4); + maskkey.putInt(this.reuseableRandom.nextInt()); + buf.put(maskkey.array()); + + for(int i = 0; mes.hasRemaining(); ++i) { + buf.put((byte)(mes.get() ^ maskkey.get(i % 4))); + } + } else { + buf.put(mes); + mes.flip(); + } + + assert buf.remaining() == 0 : buf.remaining(); + + buf.flip(); + return buf; + } + + private Framedata translateSingleFrame(ByteBuffer buffer) throws IncompleteException, InvalidDataException { + if (buffer == null) { + throw new IllegalArgumentException(); + } else { + int maxpacketsize = buffer.remaining(); + int realpacketsize = 2; + this.translateSingleFrameCheckPacketSize(maxpacketsize, realpacketsize); + byte b1 = buffer.get(); + boolean fin = b1 >> 8 != 0; + boolean rsv1 = (b1 & 64) != 0; + boolean rsv2 = (b1 & 32) != 0; + boolean rsv3 = (b1 & 16) != 0; + byte b2 = buffer.get(); + boolean mask = (b2 & -128) != 0; + int payloadlength = (byte)(b2 & 127); + Opcode optcode = this.toOpcode((byte)(b1 & 15)); + if (payloadlength < 0 || payloadlength > 125) { + TranslatedPayloadMetaData payloadData = this.translateSingleFramePayloadLength(buffer, optcode, payloadlength, maxpacketsize, realpacketsize); + payloadlength = payloadData.getPayloadLength(); + realpacketsize = payloadData.getRealPackageSize(); + } + + this.translateSingleFrameCheckLengthLimit((long)payloadlength); + realpacketsize += mask ? 4 : 0; + realpacketsize += payloadlength; + this.translateSingleFrameCheckPacketSize(maxpacketsize, realpacketsize); + ByteBuffer payload = ByteBuffer.allocate(this.checkAlloc(payloadlength)); + if (mask) { + byte[] maskskey = new byte[4]; + buffer.get(maskskey); + + for(int i = 0; i < payloadlength; ++i) { + payload.put((byte)(buffer.get() ^ maskskey[i % 4])); + } + } else { + payload.put(buffer.array(), buffer.position(), payload.limit()); + buffer.position(buffer.position() + payload.limit()); + } + + FramedataImpl1 frame = FramedataImpl1.get(optcode); + frame.setFin(fin); + frame.setRSV1(rsv1); + frame.setRSV2(rsv2); + frame.setRSV3(rsv3); + payload.flip(); + frame.setPayload(payload); + this.getExtension().isFrameValid(frame); + this.getExtension().decodeFrame(frame); + if (this.log.isTraceEnabled()) { + this.log.trace("afterDecoding({}): {}", frame.getPayloadData().remaining(), frame.getPayloadData().remaining() > 1000 ? "too big to display" : new String(frame.getPayloadData().array())); + } + +// frame.isValid(); + return frame; + } + } + + private TranslatedPayloadMetaData translateSingleFramePayloadLength(ByteBuffer buffer, Opcode optcode, int oldPayloadlength, int maxpacketsize, int oldRealpacketsize) throws InvalidFrameException, IncompleteException, LimitExceededException { + if (optcode != Opcode.PING && optcode != Opcode.PONG && optcode != Opcode.CLOSING) { + int payloadlength; + int realpacketsize; + byte[] bytes; + if (oldPayloadlength == 126) { + realpacketsize = oldRealpacketsize + 2; + this.translateSingleFrameCheckPacketSize(maxpacketsize, realpacketsize); + bytes = new byte[]{0, buffer.get(), buffer.get()}; + payloadlength = (new BigInteger(bytes)).intValue(); + } else { + realpacketsize = oldRealpacketsize + 8; + this.translateSingleFrameCheckPacketSize(maxpacketsize, realpacketsize); + bytes = new byte[8]; + + for(int i = 0; i < 8; ++i) { + bytes[i] = buffer.get(); + } + + long length = (new BigInteger(bytes)).longValue(); + this.translateSingleFrameCheckLengthLimit(length); + payloadlength = (int)length; + } + + return new TranslatedPayloadMetaData(payloadlength, realpacketsize); + } else { + this.log.trace("Invalid frame: more than 125 octets"); + throw new InvalidFrameException("more than 125 octets"); + } + } + + private void translateSingleFrameCheckLengthLimit(long length) throws LimitExceededException { + if (length > 2147483647L) { + this.log.trace("Limit exedeed: Payloadsize is to big..."); + throw new LimitExceededException("Payloadsize is to big..."); + } else if (length > (long)this.maxFrameSize) { + this.log.trace("Payload limit reached. Allowed: {} Current: {}", this.maxFrameSize, length); + throw new LimitExceededException("Payload limit reached.", this.maxFrameSize); + } else if (length < 0L) { + this.log.trace("Limit underflow: Payloadsize is to little..."); + throw new LimitExceededException("Payloadsize is to little..."); + } + } + + private void translateSingleFrameCheckPacketSize(int maxpacketsize, int realpacketsize) throws IncompleteException { + if (maxpacketsize < realpacketsize) { + this.log.trace("Incomplete frame: maxpacketsize < realpacketsize"); + throw new IncompleteException(realpacketsize); + } + } + + private byte getRSVByte(int rsv) { + if (rsv == 1) { + return 64; + } else if (rsv == 2) { + return 32; + } else { + return (byte)(rsv == 3 ? 16 : 0); + } + } + + private byte getMaskByte(boolean mask) { + return (byte)(mask ? -128 : 0); + } + + private int getSizeBytes(ByteBuffer mes) { + if (mes.remaining() <= 125) { + return 1; + } else { + return mes.remaining() <= 65535 ? 2 : 8; + } + } + + public List translateFrame(ByteBuffer buffer) throws InvalidDataException { + while(true) { + List frames = new LinkedList(); + Framedata cur; + int pref; + if (this.incompleteframe != null) { + try { + buffer.mark(); + int availableNextByteCount = buffer.remaining(); + pref = this.incompleteframe.remaining(); + if (pref > availableNextByteCount) { + this.incompleteframe.put(buffer.array(), buffer.position(), availableNextByteCount); + buffer.position(buffer.position() + availableNextByteCount); + return Collections.emptyList(); + } + + this.incompleteframe.put(buffer.array(), buffer.position(), pref); + buffer.position(buffer.position() + pref); + cur = this.translateSingleFrame((ByteBuffer)this.incompleteframe.duplicate().position(0)); + frames.add(cur); + this.incompleteframe = null; + } catch (IncompleteException var6) { + ByteBuffer extendedframe = ByteBuffer.allocate(this.checkAlloc(var6.getPreferredSize())); + + assert extendedframe.limit() > this.incompleteframe.limit(); + + this.incompleteframe.rewind(); + extendedframe.put(this.incompleteframe); + this.incompleteframe = extendedframe; + continue; + } + } + + while(buffer.hasRemaining()) { + buffer.mark(); + + try { + cur = this.translateSingleFrame(buffer); + frames.add(cur); + } catch (IncompleteException var7) { + buffer.reset(); + pref = var7.getPreferredSize(); + this.incompleteframe = ByteBuffer.allocate(this.checkAlloc(pref)); + this.incompleteframe.put(buffer); + break; + } + } + + return frames; + } + } + + public List createFrames(ByteBuffer binary, boolean mask) { + BinaryFrame curframe = new BinaryFrame(); + curframe.setPayload(binary); + curframe.setTransferemasked(mask); + + try { + curframe.isValid(); + } catch (InvalidDataException var5) { + throw new NotSendableException(var5); + } + + return Collections.singletonList(curframe); + } + + public List createFrames(String text, boolean mask) { + TextFrame curframe = new TextFrame(); + curframe.setPayload(ByteBuffer.wrap(Charsetfunctions.utf8Bytes(text))); + curframe.setTransferemasked(mask); + + try { + curframe.isValid(); + } catch (InvalidDataException var5) { + throw new NotSendableException(var5); + } + + return Collections.singletonList(curframe); + } + + public void reset() { + this.incompleteframe = null; + if (this.extension != null) { + this.extension.reset(); + } + + this.extension = new DefaultExtension(); + this.protocol = null; + } + + private String getServerTime() { + Calendar calendar = Calendar.getInstance(); + SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + return dateFormat.format(calendar.getTime()); + } + + private String generateFinalKey(String in) { + String seckey = in.trim(); + String acc = seckey + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + + MessageDigest sh1; + try { + sh1 = MessageDigest.getInstance("SHA1"); + } catch (NoSuchAlgorithmException var6) { + throw new IllegalStateException(var6); + } + + return Base64.encodeBytes(sh1.digest(acc.getBytes())); + } + + private byte[] toByteArray(long val, int bytecount) { + byte[] buffer = new byte[bytecount]; + int highest = 8 * bytecount - 8; + + for(int i = 0; i < bytecount; ++i) { + buffer[i] = (byte)((int)(val >>> highest - 8 * i)); + } + + return buffer; + } + + private byte fromOpcode(Opcode opcode) { + if (opcode == Opcode.CONTINUOUS) { + return 0; + } else if (opcode == Opcode.TEXT) { + return 1; + } else if (opcode == Opcode.BINARY) { + return 2; + } else if (opcode == Opcode.CLOSING) { + return 8; + } else if (opcode == Opcode.PING) { + return 9; + } else if (opcode == Opcode.PONG) { + return 10; + } else { + throw new IllegalArgumentException("Don't know how to handle " + opcode.toString()); + } + } + + private Opcode toOpcode(byte opcode) throws InvalidFrameException { + switch (opcode) { + case 0: + return Opcode.CONTINUOUS; + case 1: + return Opcode.TEXT; + case 2: + return Opcode.BINARY; + case 3: + case 4: + case 5: + case 6: + case 7: + default: + throw new InvalidFrameException("Unknown opcode " + (short)opcode); + case 8: + return Opcode.CLOSING; + case 9: + return Opcode.PING; + case 10: + return Opcode.PONG; + } + } + + public void processFrame(WebSocketImpl webSocketImpl, Framedata frame) throws InvalidDataException { + Opcode curop = frame.getOpcode(); + if (curop == Opcode.CLOSING) { + this.processFrameClosing(webSocketImpl, frame); + } else if (curop == Opcode.PING) { + webSocketImpl.getWebSocketListener().onWebsocketPing(webSocketImpl, frame); + } else if (curop == Opcode.PONG) { + webSocketImpl.updateLastPong(); + webSocketImpl.getWebSocketListener().onWebsocketPong(webSocketImpl, frame); + } else if (frame.isFin() && curop != Opcode.CONTINUOUS) { + if (this.currentContinuousFrame != null) { + this.log.error("Protocol error: Continuous frame sequence not completed."); + throw new InvalidDataException(1002, "Continuous frame sequence not completed."); + } + + if (curop == Opcode.TEXT) { + this.processFrameText(webSocketImpl, frame); + } else { + if (curop != Opcode.BINARY) { + this.log.error("non control or continious frame expected"); + throw new InvalidDataException(1002, "non control or continious frame expected"); + } + + this.processFrameBinary(webSocketImpl, frame); + } + } else { + this.processFrameContinuousAndNonFin(webSocketImpl, frame, curop); + } + + } + + private void processFrameContinuousAndNonFin(WebSocketImpl webSocketImpl, Framedata frame, Opcode curop) throws InvalidDataException { + if (curop != Opcode.CONTINUOUS) { + this.processFrameIsNotFin(frame); + } else if (frame.isFin()) { + this.processFrameIsFin(webSocketImpl, frame); + } else if (this.currentContinuousFrame == null) { + this.log.error("Protocol error: Continuous frame sequence was not started."); + throw new InvalidDataException(1002, "Continuous frame sequence was not started."); + } + + if (curop == Opcode.TEXT && !Charsetfunctions.isValidUTF8(frame.getPayloadData())) { + this.log.error("Protocol error: Payload is not UTF8"); + throw new InvalidDataException(1007); + } else { + if (curop == Opcode.CONTINUOUS && this.currentContinuousFrame != null) { + this.addToBufferList(frame.getPayloadData()); + } + + } + } + + private void processFrameBinary(WebSocketImpl webSocketImpl, Framedata frame) { + try { + webSocketImpl.getWebSocketListener().onWebsocketMessage(webSocketImpl, frame.getPayloadData()); + } catch (RuntimeException var4) { + this.logRuntimeException(webSocketImpl, var4); + } + + } + + private void logRuntimeException(WebSocketImpl webSocketImpl, RuntimeException e) { + this.log.error("Runtime exception during onWebsocketMessage", e); + webSocketImpl.getWebSocketListener().onWebsocketError(webSocketImpl, e); + } + + private void processFrameText(WebSocketImpl webSocketImpl, Framedata frame) throws InvalidDataException { + try { + webSocketImpl.getWebSocketListener().onWebsocketMessage(webSocketImpl, Charsetfunctions.stringUtf8(frame.getPayloadData())); + } catch (RuntimeException var4) { + this.logRuntimeException(webSocketImpl, var4); + } + + } + + private void processFrameIsFin(WebSocketImpl webSocketImpl, Framedata frame) throws InvalidDataException { + if (this.currentContinuousFrame == null) { + this.log.trace("Protocol error: Previous continuous frame sequence not completed."); + throw new InvalidDataException(1002, "Continuous frame sequence was not started."); + } else { + this.addToBufferList(frame.getPayloadData()); + this.checkBufferLimit(); + if (this.currentContinuousFrame.getOpcode() == Opcode.TEXT) { + ((FramedataImpl1)this.currentContinuousFrame).setPayload(this.getPayloadFromByteBufferList()); + ((FramedataImpl1)this.currentContinuousFrame).isValid(); + + try { + webSocketImpl.getWebSocketListener().onWebsocketMessage(webSocketImpl, Charsetfunctions.stringUtf8(this.currentContinuousFrame.getPayloadData())); + } catch (RuntimeException var5) { + this.logRuntimeException(webSocketImpl, var5); + } + } else if (this.currentContinuousFrame.getOpcode() == Opcode.BINARY) { + ((FramedataImpl1)this.currentContinuousFrame).setPayload(this.getPayloadFromByteBufferList()); + ((FramedataImpl1)this.currentContinuousFrame).isValid(); + + try { + webSocketImpl.getWebSocketListener().onWebsocketMessage(webSocketImpl, this.currentContinuousFrame.getPayloadData()); + } catch (RuntimeException var4) { + this.logRuntimeException(webSocketImpl, var4); + } + } + + this.currentContinuousFrame = null; + this.clearBufferList(); + } + } + + private void processFrameIsNotFin(Framedata frame) throws InvalidDataException { + if (this.currentContinuousFrame != null) { + this.log.trace("Protocol error: Previous continuous frame sequence not completed."); + throw new InvalidDataException(1002, "Previous continuous frame sequence not completed."); + } else { + this.currentContinuousFrame = frame; + this.addToBufferList(frame.getPayloadData()); + this.checkBufferLimit(); + } + } + + private void processFrameClosing(WebSocketImpl webSocketImpl, Framedata frame) { + int code = 1005; + String reason = ""; + if (frame instanceof CloseFrame) { + CloseFrame cf = (CloseFrame)frame; + code = cf.getCloseCode(); + reason = cf.getMessage(); + } + + if (webSocketImpl.getReadyState() == ReadyState.CLOSING) { + webSocketImpl.closeConnection(code, reason, true); + } else if (this.getCloseHandshakeType() == CloseHandshakeType.TWOWAY) { + webSocketImpl.close(code, reason, true); + } else { + webSocketImpl.flushAndClose(code, reason, false); + } + + } + + private void clearBufferList() { + synchronized(this.byteBufferList) { + this.byteBufferList.clear(); + } + } + + private void addToBufferList(ByteBuffer payloadData) { + synchronized(this.byteBufferList) { + this.byteBufferList.add(payloadData); + } + } + + private void checkBufferLimit() throws LimitExceededException { + long totalSize = this.getByteBufferListSize(); + if (totalSize > (long)this.maxFrameSize) { + this.clearBufferList(); + this.log.trace("Payload limit reached. Allowed: {} Current: {}", this.maxFrameSize, totalSize); + throw new LimitExceededException(this.maxFrameSize); + } + } + + public CloseHandshakeType getCloseHandshakeType() { + return CloseHandshakeType.TWOWAY; + } + + public String toString() { + String result = super.toString(); + if (this.getExtension() != null) { + result = result + " extension: " + this.getExtension().toString(); + } + + if (this.getProtocol() != null) { + result = result + " protocol: " + this.getProtocol().toString(); + } + + result = result + " max frame size: " + this.maxFrameSize; + return result; + } + + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o != null && this.getClass() == o.getClass()) { + Draft_6455 that = (Draft_6455)o; + if (this.maxFrameSize != that.getMaxFrameSize()) { + return false; + } else { + if (this.extension != null) { + if (this.extension.equals(that.getExtension())) { + return this.protocol != null ? this.protocol.equals(that.getProtocol()) : that.getProtocol() == null; + } + } else if (that.getExtension() == null) { + return this.protocol != null ? this.protocol.equals(that.getProtocol()) : that.getProtocol() == null; + } + + return false; + } + } else { + return false; + } + } + + public int hashCode() { + int result = this.extension != null ? this.extension.hashCode() : 0; + result = 31 * result + (this.protocol != null ? this.protocol.hashCode() : 0); + result = 31 * result + (this.maxFrameSize ^ this.maxFrameSize >>> 32); + return result; + } + + private ByteBuffer getPayloadFromByteBufferList() throws LimitExceededException { + long totalSize = 0L; + ByteBuffer resultingByteBuffer; + synchronized(this.byteBufferList) { + Iterator var5 = this.byteBufferList.iterator(); + + while(true) { + ByteBuffer buffer; + if (!var5.hasNext()) { + this.checkBufferLimit(); + resultingByteBuffer = ByteBuffer.allocate((int)totalSize); + var5 = this.byteBufferList.iterator(); + + while(var5.hasNext()) { + buffer = (ByteBuffer)var5.next(); + resultingByteBuffer.put(buffer); + } + break; + } + + buffer = (ByteBuffer)var5.next(); + totalSize += (long)buffer.limit(); + } + } + + resultingByteBuffer.flip(); + return resultingByteBuffer; + } + + private long getByteBufferListSize() { + long totalSize = 0L; + synchronized(this.byteBufferList) { + ByteBuffer buffer; + for(Iterator var4 = this.byteBufferList.iterator(); var4.hasNext(); totalSize += (long)buffer.limit()) { + buffer = (ByteBuffer)var4.next(); + } + + return totalSize; + } + } + + private class TranslatedPayloadMetaData { + private int payloadLength; + private int realPackageSize; + + private int getPayloadLength() { + return this.payloadLength; + } + + private int getRealPackageSize() { + return this.realPackageSize; + } + + TranslatedPayloadMetaData(int newPayloadLength, int newRealPackageSize) { + this.payloadLength = newPayloadLength; + this.realPackageSize = newRealPackageSize; + } + } +} diff --git a/config/chopperBotConfig.json b/config/chopperBotConfig.json index cea9487..5b5bc1d 100644 --- a/config/chopperBotConfig.json +++ b/config/chopperBotConfig.json @@ -23,5 +23,5 @@ "HotRecommendation":true } }, - "updateTime":"2023-09-04 00:03:07" + "updateTime":"2023-09-05 20:52:21" } \ No newline at end of file