lywsvip/ChopperBot
1
0
Fork 0
mirror of https://github.com/Geniusay/ChopperBot.git synced 2026-06-01 11:30:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

v1.0.3一些bug修改和一些模块重构优化

Browse Source
This commit is contained in:
userA
2023-04-26 00:52:14 +08:00
parent eba9c32c8f
commit 1233ca8a00
22 changed files with 389 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
CHANGELOG.md
View File

@@ -18,7 +18,14 @@
------
```
------
# 目录
* [V 1.0.3]()
* [V 1.0.2]()
* [V 1.0.1]()
* [V 1.0.0]()
------
## [V 1.0.3] - 2023.4.25
### CreeperModule
- 🎈新增: 新增 `LoadTaskManager` 任务管理器(类),对用户开放的顶层api,用户需要的所有操作都通过此管理器
@@ -31,13 +38,23 @@
- 🎈新增: 新增 `LoadTask_R_Douyu` 斗鱼录播下载任务(类)
- 🎈新增: 新增 `Process_R_Douyu` 斗鱼录播处理器(类)
- 🎈新增: 新增 `AbstractProcessor` 处理器抽象类(类)
- 🎈新增: 新增 `ConstPool` BARRAGE_ROOT常量
- 🎈新增: 新增 `BarrageSaveFile` 弹幕存储文件，负责存储当天直播弹幕
- 🧹重构: 重构 `PipelineWriteJson` 弹幕缓存 与 弹幕文件缓存建立联系
HelloWorld:
<img src=https://twj666.oss-cn-hangzhou.aliyuncs.com/img1/QQ%E6%88%AA%E5%9B%BE20230425201236.png style="zoom:40%;">
### FileModule
- ❌移除: 新增 `FileCache的oldJsonFile变量`,不在用map来进行版本更替判断，取而代之的是判断写入字节是否为0的高效率方法
- 🎈新增: 新增 `FileCache` 方法 `get,writekeys,append` 更加方便的缓存获取，更加方便的写入与内容追加
- 🧪测试: 测试 `FileCache` 方法 `get,writekeys,append`,功能正常，可以使用
HelloWorld:
![QQ截图20230425201236](https://twj666.oss-cn-hangzhou.aliyuncs.com/img1/QQ%E6%88%AA%E5%9B%BE20230425201236.png)
- ❌移除: 移除 `FileCache-oldJsonFile变量`,不在用map来进行版本更替判断，取而代之的是判断写入字节是否为0的高效率方法
- 🎈新增: 新增 `FileCache` 方法 `get,writekeys,append` 更加方便的缓存获取，更加方便的写入与内容追加
- 🎈新增: 新增 `FileCacheManagerInstance` 将整个FileCacheManager转变为全局单例，防止重复使用调用
- 🎈新增: 新增 `GlobalFileCache` 全局文件缓存，也负责为`FileCacheManagerInstance`提供初始化的文件缓存队列
- 🎈新增: 新增 `FileCacheManager` 新增方法 `addFileCache` 负责在之后动态的添加新的文件缓存
- 🐞Bug: **#00001** `FileCache append()` 在进行数组追加时产生溢出
- ⛏修复: 修复 **#00001**, 在进行数组追加时不会再溢出
------

6
CreeperModule/src/main/java/org/example/constpool/ConstPool.java
View File

@@ -7,6 +7,12 @@ package org.example.constpool;
*/
public class ConstPool {
/**
* 文件路径
*/
//弹幕保存文件的根目录
public static final String BARRAGE_ROOT = (String)GlobalFileCache.ModuleSrcConfigFile.get("barrage","src");
/**
* 下载的方式
*/

62
CreeperModule/src/main/java/org/example/core/pipeline/PipelineWriteJson.java
View File

@@ -1,6 +1,10 @@
package org.example.core.pipeline;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.exception.FileCacheException;
import org.example.pojo.Barrage;
import org.example.pojo.configfile.BarrageSaveFile;
import org.example.pojo.download.LoadConfig;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
@@ -19,45 +23,47 @@ import java.util.concurrent.ConcurrentLinkedQueue;
*/
public class PipelineWriteJson implements Pipeline {
// Barrage缓存
private final ConcurrentLinkedQueue<Barrage> cache;
private FileCache filecache;
private final ConcurrentLinkedQueue<Barrage> cache;
LoadConfig loadConfig;
LoadConfig loadConfig;
private BarrageSaveFile barrageSaveFile;
public PipelineWriteJson(LoadConfig loadConfig) {
this.loadConfig = loadConfig;
this.cache = new ConcurrentLinkedQueue<>();
}
@Override
public void process(ResultItems resultItems, Task task) {
List<Barrage> barrageList = resultItems.get("barrageList");
if (barrageList != null) {
cache.addAll(barrageList);
public PipelineWriteJson(LoadConfig loadConfig) throws FileCacheException {
this.loadConfig = loadConfig;
this.cache = new ConcurrentLinkedQueue<>();
this.barrageSaveFile = new BarrageSaveFile(loadConfig,cache);
this.filecache = new FileCache(barrageSaveFile);
FileCacheManagerInstance.getInstance().addFileCache(filecache);
}
}
public int getCacheSize() {
return cache.size();
}
@Override
public void process(ResultItems resultItems, Task task) {
List<Barrage> barrageList = resultItems.get("barrageList");
if (barrageList != null) {
cache.addAll(barrageList);
}
}
// 将缓存写入到别处并且清空缓存
public int writeDataToFileAndFlushCache(String key) {
int successCount = 0;
public int getCacheSize() {
return cache.size();
}
String filePath = "F:\\" + key + ".txt";
// 将缓存写入到别处并且清空缓存
public int writeDataToFileAndFlushCache(String key) {
int successCount = 0;
try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath, true))) {
Barrage barrage;
while ((barrage = cache.poll()) != null) {
writer.write(barrage.toString());
writer.newLine();
try {
filecache.append(barrage,"-1");
} catch (InterruptedException |FileCacheException e) {
throw new RuntimeException(e);
}
successCount++;
}
} catch (IOException e) {
e.printStackTrace();
return successCount;
}
return successCount;
}
}

69
CreeperModule/src/main/java/org/example/pojo/configfile/BarrageSaveFile.java Normal file
View File

@@ -0,0 +1,69 @@
package org.example.pojo.configfile;
import org.example.common.ConfigFile;
import org.example.constpool.GlobalFileCache;
import org.example.exception.FileCacheException;
import org.example.pojo.Barrage;
import org.example.pojo.download.LoadConfig;
import org.example.util.FileUtil;
import org.example.util.JsonFileUtil;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import static org.example.constpool.ConstPool.BARRAGE_ROOT;
/**
* @author Genius
* @date 2023/04/25 22:08
**/
/**
* 弹幕保存文件配置类，通过LoadConfig获取主播信息，平台信息，弹幕爬取时间，生成弹幕文件
*/
public class BarrageSaveFile extends ConfigFile<ConcurrentLinkedQueue<Barrage>> {
private LoadConfig loadConfig;
public BarrageSaveFile(LoadConfig loadConfig,ConcurrentLinkedQueue<Barrage> data) throws FileCacheException {
super();
this.loadConfig = loadConfig;
if (!init(data)) {
throw new FileCacheException("File init Error");
}
}
/**
* 自动生成主播弹幕文件夹以及当天直播弹幕数据文本
* @param data
* @return
*/
private boolean init(ConcurrentLinkedQueue<Barrage> data) {
String fileName = this.filaName();
setFileName(fileName);
String rootPath = Paths.get( BARRAGE_ROOT,loadConfig.getAnchorName()).toString(); //获取当前主播的文件夹路径
setFilePath(rootPath);
//TODO 待移除 建立主播文件夹
try {
Files.createFile(Path.of(rootPath));
} catch (IOException e) {
return false;
}
setData(data);
if (!FileUtil.isFileExist(Paths.get(rootPath,fileName).toString())) {
return !(JsonFileUtil.writeJsonFile(rootPath,fileName,this.packageConfig())==null);
}
return true;
}
private String filaName(){
String format = "%s_%s_%s.json";
return String.format(format,loadConfig.getPlatform(),loadConfig.getAnchorName(),loadConfig.getStartTime());
}
}

31
FileModule/src/main/java/org/example/cache/FileCache.java vendored
View File

@@ -31,7 +31,7 @@ public class FileCache <T extends ConfigFile>{
private ConcurrentHashMap<String,Object> jsonFile; //文件内容缓存
private static final int MAX_WRITE_BUFFER_LIMIT = 4096; //最大写入缓存上线
private static int MAX_WRITE_BUFFER_LIMIT = 4096; //最大写入缓存上线
private AtomicInteger writeByte; //当前写入的字节数
@@ -46,7 +46,15 @@ public class FileCache <T extends ConfigFile>{
init(configFile,10);
}
public FileCache(T configFile,long autoSyncTime)throws FileCacheException {
/**
* 构造方法
* @param configFile 指定的配置文件
* @param autoSyncTime 自动刷新时间
* @param maxWriteBufferLimit 写入上限
* @throws FileCacheException
*/
public FileCache(T configFile,long autoSyncTime,int maxWriteBufferLimit)throws FileCacheException {
MAX_WRITE_BUFFER_LIMIT = maxWriteBufferLimit;
init(configFile,autoSyncTime);
}
@@ -143,7 +151,7 @@ public class FileCache <T extends ConfigFile>{
//元素添加
int index = Integer.parseInt(keys[keys.length-1]);
if(index==-1){
((JSONArray) temp).add(index,value);
((JSONArray) temp).add(value);
}else{
String oldValue = ((JSONArray) temp).get(index).toString();
value = isAppend?oldValue+value.toString():value;
@@ -211,7 +219,6 @@ public class FileCache <T extends ConfigFile>{
return jsonFile.get(key);
}
/**
* 清除已写入的字节数记录
*/
@@ -285,4 +292,20 @@ public class FileCache <T extends ConfigFile>{
}
}
@Override
public int hashCode() {
return Objects.hash(configFile);
}
@Override
public boolean equals(Object obj) {
if(obj instanceof FileCache){
if(((FileCache) obj).getFileName().equals(this.getFileName())){
return true;
}else if(obj.hashCode() == this.hashCode()){
return true;
}
}
return false;
}
}

34
FileModule/src/main/java/org/example/cache/FileCacheManager.java vendored
View File

@@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
@@ -27,14 +28,16 @@ public class FileCacheManager {
private Logger logger = LoggerFactory.getLogger(FileCacheManager.class);
private final List<FileCache> fileCaches;
private long sleepTime; //睡眠时间
private AtomicLong sleepTime; //睡眠时间
private ExecutorService watchPool; //巡逻线程
private ExecutorService autoSyncer; //生产者线程
public FileCacheManager(List<FileCache> fileCaches){
this.fileCaches = fileCaches;
private volatile Watcher watcher;
protected FileCacheManager(List<FileCache> fileCaches){
this.fileCaches = new CopyOnWriteArrayList<>(fileCaches);
initSleepTime();
this.watchPool = Executors.newSingleThreadExecutor();
this.autoSyncer = Executors.newFixedThreadPool(fileCaches.size());
@@ -48,11 +51,28 @@ public class FileCacheManager {
fileCaches.forEach(item->{
minSleepTime.set(Long.min(minSleepTime.get(), item.getSyncTime()));
});
this.sleepTime = minSleepTime.get();
this.sleepTime = minSleepTime;
}
public void start(){
this.watchPool.submit(new Watcher());
if(!fileCaches.isEmpty()){
if(watcher==null){
synchronized (FileCacheManager.class){
if(watcher==null){
watcher = new Watcher();
this.watchPool.submit(watcher);
}
}
}
}
}
public boolean addFileCache(FileCache fileCache){
if (this.fileCaches.indexOf(fileCache)==-1) {
fileCaches.add(fileCache);
initSleepTime();
}
return false;
}
class Watcher implements Runnable{
@@ -71,9 +91,9 @@ public class FileCacheManager {
}
}
now -= TimeUtil.getCurrentSecond();
if(now<sleepTime){
if(now<sleepTime.get()){
try {
Thread.sleep((sleepTime-now)*1000);
Thread.sleep((sleepTime.get()-now)*1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}

30
FileModule/src/main/java/org/example/cache/FileCacheManagerInstance.java vendored Normal file
View File

@@ -0,0 +1,30 @@
package org.example.cache;
import org.example.constpool.GlobalFileCache;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/04/25 22:14
**/
//FileCacheManager单例实体类
public class FileCacheManagerInstance {
//获取全局的一个fileCaches
private static List<FileCache> fileCaches = GlobalFileCache.fileCaches;
private static volatile FileCacheManager Instance;
public static FileCacheManager getInstance(){
if(Instance==null){
synchronized (FileCacheManagerInstance.class){
if(Instance==null){
Instance = new FileCacheManager(fileCaches);
}
}
}
return Instance;
}
}

31
FileModule/src/main/java/org/example/constpool/GlobalFileCache.java Normal file
View File

@@ -0,0 +1,31 @@
package org.example.constpool;
import org.example.cache.FileCache;
import org.example.exception.FileCacheException;
import org.example.pojo.configfile.ModuleSrcConfigFile;
import java.util.List;
/**
* @author Genius
* @date 2023/04/25 23:03
**/
/**
* 全局文件缓存池，用于存放全局文件缓存，便于跨模块调用
*/
public class GlobalFileCache {
public static FileCache ModuleSrcConfigFile;
static {
try {
ModuleSrcConfigFile = new FileCache(new ModuleSrcConfigFile());
} catch (FileCacheException e) {
throw new RuntimeException(e);
}
}
public static List<FileCache> fileCaches
= List.of(ModuleSrcConfigFile);
}

2
FileModule/src/main/java/org/example/init/ModuleSrcConfigFileInit.java
View File

@@ -1,6 +1,6 @@
package org.example.init;
import org.example.bean.ModuleSrcConfigFile;
import org.example.pojo.configfile.ModuleSrcConfigFile;
import org.example.util.FileUtil;
import org.example.util.JsonFileUtil;
import org.slf4j.Logger;

14
FileModule/src/main/java/org/example/pojo/CommonConfigFile.java Normal file
View File

@@ -0,0 +1,14 @@
package org.example.pojo;
import org.example.common.ConfigFile;
/**
* @author Genius
* @date 2023/04/26 00:12
**/
public class CommonConfigFile extends ConfigFile {
public CommonConfigFile(String filePath, String fileName, Object data) {
super(filePath, fileName, data);
}
}

2
FileModule/src/main/java/org/example/bean/ModuleSrcConfigFile.java → FileModule/src/main/java/org/example/pojo/configfile/ModuleSrcConfigFile.java
View File

@@ -1,4 +1,4 @@
package org.example.bean;
package org.example.pojo.configfile;
import org.example.common.ConfigFile;
import org.example.constpool.ConstPool;

73
FileModule/src/main/resources/test.json
View File

@@ -1,10 +1,65 @@
{
"name": "ChopperBot",
"description": "A bot for the ChopperMC server",
"version": "1.0.0",
"module": {
"type": ["Account","Creeper","File","Hot","Publish","Section","SectionWork","VideoSection"],
"main": "console",
"ui": "console-ui"
}
}
"data":[
{
"age":0,
"major":"major0",
"name":"0",
"school":"jsu0"
},
{
"age":1,
"major":"major1",
"name":"1",
"school":"jsu1"
},
{
"age":2,
"major":"major2",
"name":"2",
"school":"jsu2"
},
{
"age":3,
"major":"major3",
"name":"3",
"school":"jsu3"
},
{
"age":4,
"major":"major4",
"name":"4",
"school":"jsu4"
},
{
"age":5,
"major":"major5",
"name":"5",
"school":"jsu5"
},
{
"age":6,
"major":"major6",
"name":"6",
"school":"jsu6"
},
{
"age":7,
"major":"major7",
"name":"7",
"school":"jsu7"
},
{
"age":8,
"major":"major8",
"name":"8",
"school":"jsu8"
},
{
"age":9,
"major":"major9",
"name":"9",
"school":"jsu9"
}
],
"updateTime":"2023-04-26 00:24:47"
}

82
FileModule/src/test/java/org/example/cache/FileCacheTest.java vendored
View File

@@ -1,19 +1,16 @@
package org.example.cache;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPObject;
import org.example.bean.ModuleSrcConfigFile;
import org.example.bean.Student;
import org.example.cache.FileCache;
import org.example.common.ConfigFile;
import org.example.constpool.GlobalFileCache;
import org.example.pojo.CommonConfigFile;
import org.example.pojo.Student;
import org.example.pojo.configfile.ModuleSrcConfigFile;
import org.example.exception.FileCacheException;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.logging.log4j.message.MapMessage.MapFormat.JSON;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
* @author Genius
@@ -21,6 +18,23 @@ import static org.apache.logging.log4j.message.MapMessage.MapFormat.JSON;
**/
public class FileCacheTest {
static FileCache cache;
static{
try {
cache = new FileCache<>(
new CommonConfigFile(
"E:\\Project\\ChopperBot\\FileModule\\src\\main\\resources\\",
"test.json",
new ConcurrentLinkedQueue<Student>()
)
);
FileCacheManagerInstance.getInstance().addFileCache(cache);
} catch (FileCacheException e) {
throw new RuntimeException(e);
}
}
@Test
public void TestFileCache() throws FileCacheException, InterruptedException {
@@ -55,55 +69,59 @@ public class FileCacheTest {
@Test
public void TestAppend() throws FileCacheException, InterruptedException {
ModuleSrcConfigFile moduleSrcConfigFile = new ModuleSrcConfigFile();
FileCache fileCache = new FileCache(moduleSrcConfigFile);
FileCacheManager manager = new FileCacheManager(List.of(fileCache));
manager.start();
FileCacheManagerInstance.getInstance().start();
//追加数组的某个元素
for(int i=0;i<10;i++){
fileCache.append(i,"sectionwork","src","0");
GlobalFileCache.ModuleSrcConfigFile.append(i,"sectionwork","src","0");
}
//追加数组
for(int i=0;i<10;i++){
fileCache.append(i,"sectionwork","src","-1");
GlobalFileCache.ModuleSrcConfigFile.append(i,"sectionwork","src","-1");
}
//追加类中的元素
for(int i=0;i<10;i++){
fileCache.append(i,"barrage","src");
GlobalFileCache.ModuleSrcConfigFile.append(i,"barrage","src");
}
Thread.sleep(500000);
}
@Test
public void TestWrite() throws FileCacheException,InterruptedException{
ModuleSrcConfigFile moduleSrcConfigFile = new ModuleSrcConfigFile();
FileCache fileCache = new FileCache(moduleSrcConfigFile);
FileCacheManager manager = new FileCacheManager(List.of(fileCache));
manager.start();
FileCacheManagerInstance.getInstance().start();
//更改数组元素
for(int i=0;i<10;i++){
fileCache.writeKeys(i,"sectionwork","src","0");
GlobalFileCache.ModuleSrcConfigFile.writeKeys(i,"sectionwork","src","0");
}
//追加数组
for(int i=0;i<10;i++){
fileCache.writeKeys(i,"sectionwork","src","-1");
GlobalFileCache.ModuleSrcConfigFile.writeKeys(i,"sectionwork","src","-1");
}
//更改类中的元素
for(int i=0;i<10;i++){
fileCache.writeKeys(i,"barrage","src");
GlobalFileCache.ModuleSrcConfigFile.writeKeys(i,"barrage","src");
}
Thread.sleep(500000);
}
@Test
public void TestJson(){
Student student = new Student("Genius",18,"qinghua","SC",List.of("1","2","3"),null);
String s = JSONObject.toJSONString("");
JSONObject jsonObject = JSONObject.parseObject(s);
jsonObject.put("hobby",List.of("1","2","3","4").toString());
System.out.println(jsonObject);
public void TestArrayAdd() throws InterruptedException, FileCacheException {
ModuleSrcConfigFile moduleSrcConfigFile = new ModuleSrcConfigFile();
FileCache fileCache = new FileCache(moduleSrcConfigFile);
FileCacheManager manager = new FileCacheManager(List.of(fileCache));
manager.start();
System.out.println(manager.addFileCache(fileCache));
}
@Test
public void TestAddCache() throws FileCacheException, InterruptedException {
FileCacheManagerInstance.getInstance().start();
for(int i=0;i<10;i++){
Student student = new Student(Integer.toString(i),i,"jsu"+i,"major"+i,null,null);
cache.writeKeys(student,"-1");
}
Thread.sleep(500000);
}
}

2
FileModule/src/test/java/org/example/bean/Student.java → FileModule/src/test/java/org/example/pojo/Student.java
View File

@@ -1,4 +1,4 @@
package org.example.bean;
package org.example.pojo;
import lombok.Data;

8
FileModule/src/test/java/org/example/util/JsonFileUtilTest.java
View File

@@ -1,15 +1,9 @@
package org.example.util;
import com.alibaba.fastjson.JSONReader;
import com.alibaba.fastjson.JSONWriter;
import org.example.bean.Student;
import org.example.pojo.Student;
import org.junit.jupiter.api.Test;
import org.junit.platform.commons.util.PackageUtils;
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;

10
FileModule/target/classes/test.json
View File

@@ -1,10 +1,4 @@
{
"name": "ChopperBot",
"description": "A bot for the ChopperMC server",
"version": "1.0.0",
"module": {
"type": ["Account","Creeper","File","Hot","Publish","Section","SectionWork","VideoSection"],
"main": "console",
"ui": "console-ui"
}
"data":[],
"updateTime":"2023-04-25 23:46:35"
}

BIN
FileModule/target/test-classes/org/example/bean/Student.class
View File

Binary file not shown.

BIN
FileModule/target/test-classes/org/example/util/JsonFileUtilTest.class
View File

Binary file not shown.

3
README.md
View File

@@ -60,7 +60,8 @@
介绍：支持文件上传，支持文件下载，支持文件管理，信息持久化
#### 待完成功能：
- [ ] 一套完整的文件操控工具
- [x] 一套完整的文件操控工具
- [x] 文件缓存池，文件缓存自动刷入器
- [ ] 远端文件上传：支持将本地文件上传至远端服务器
- [ ] 远端文件下载：支持将远端服务器的文件下载至本地
- [ ] 远端文件管理：支持对远端服务器的文件进行管理

8
common/src/main/java/org/example/common/ConfigFile.java
View File

@@ -87,4 +87,12 @@ public abstract class ConfigFile<T> {
}
public void setData(T data){this.data = data;}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public void setFilePath(String filePath){
this.filePath = filePath;
}
}

2
console/src/main/java/org/example/ConsoleApplication.java
View File

@@ -1,8 +1,6 @@
package org.example;
import org.example.bean.ModuleSrcConfigFile;
import org.example.cache.FileCache;
import org.example.exception.FileCacheException;
import org.example.init.InitWorld;
import org.example.init.ModuleSrcConfigFileInit;

1
console/src/main/java/org/example/init/InitWorld.java
View File

@@ -41,7 +41,6 @@ public class InitWorld {
return false;
}
}
return true;
}
}