更改项目模块名称,整合b站爬虫

This commit is contained in:
userA
2023-08-31 01:36:20 +08:00
parent 3334e2ec8f
commit c1010a5fb1
277 changed files with 150 additions and 98 deletions

View File

@@ -0,0 +1,19 @@
package org.example;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author Genius
* @date 2023/04/20 00:16
**/
@SpringBootApplication
public class ConsoleApplication {
public static void main(String[] args) {
SpringApplication.run(ConsoleApplication.class, args);
}
}

View File

@@ -0,0 +1,22 @@
package org.example.bean;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
/**
* @author Genius
* @date 2023/08/02 18:17
**/
@Data
@AllArgsConstructor
public class Plugin {
private String pluginName;
private String pluginModule;
private String pluginName_CN;
private String pluginDescription;
private List<String> needPlugins;
private boolean isStart;
private boolean isRegister;
}

View File

@@ -0,0 +1,36 @@
package org.example.controller;
import com.genius.assistant.common.Result;
import org.example.service.FileService;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
/**
* @author Genius
* @date 2023/04/26 01:05
**/
@RestController
@RequestMapping("/config")
public class FileController {
@Resource
FileService fileService;
/**
* 获取全部配置文件的模块
*/
@GetMapping("/allConfigModule")
public Result getAllConfigModule(){
return Result.success(fileService.getAllModule());
}
@GetMapping("/allConfigFiles")
public Result getAllConfigFiles(){
return Result.success(fileService.getAllConfigs());
}
}

View File

@@ -0,0 +1,62 @@
package org.example.controller;
import com.genius.assistant.common.Result;
import org.example.api.HotModuleApi;
import org.example.bean.Live;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.constpool.ConstPool;
import org.example.constpool.PluginName;
import org.example.core.HotModuleDataCenter;
import org.example.plugin.annotation.CheckPlugin;
import org.example.service.HotModuleService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 17:13
**/
@RestController
@RequestMapping("/hot")
public class HotController {
@Autowired
HotModuleService hotModuleService;
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/douyu/allHotLive")
public Result getDouyuAllHotLive(@RequestParam(defaultValue = "0") int latest){
List<? extends Live> lives;
if(latest==1){
lives = HotModuleApi.getDouyuHotLive();
}else{
lives = HotModuleDataCenter.DataCenter().getLiveList(ConstPool.PLATFORM.DOUYU.getName());
}
return Result.success(lives);
}
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/douyu/allHotModule")
public Result getDouyuAllHotModule(@RequestParam(defaultValue = "0") int latest){
HotModuleList hotModuleList;
if(latest==1){
hotModuleList = HotModuleApi.getDouyuAllHotModule();
}else{
hotModuleList = HotModuleDataCenter.DataCenter().getModuleList(ConstPool.PLATFORM.DOUYU.getName());
}
return Result.success(hotModuleList.getHotModuleList());
}
@CheckPlugin(needPlugin = {PluginName.HOT_GUARD_PLUGIN})
@GetMapping("/douyu/getHotModuleLives")
public Result getDouyuHotModuleLives(@RequestParam int moduleId){
HotModule moduleHotLives = hotModuleService.getModuleHotLives(ConstPool.PLATFORM.DOUYU.getName(), moduleId);
return Result.success(moduleHotLives);
}
}

View File

@@ -0,0 +1,54 @@
package org.example.controller;
import org.example.bean.Plugin;
import org.example.constpool.GlobalFileCache;
import org.example.constpool.ModuleName;
import org.example.init.InitPluginRegister;
import org.example.service.PluginService;
import org.example.util.Result;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author Genius
* @date 2023/08/01 23:39
**/
@RestController
@RequestMapping("/plugin")
public class PluginController {
@Resource
PluginService pluginService;
@GetMapping("/get")
public Result getPlugin(){
return Result.success(pluginService.getPlugins(null));
}
@GetMapping("/get/{moduleName}")
public Result getPlugin(@PathVariable(required = false) String moduleName){
return Result.success(pluginService.getPlugins(moduleName));
}
@GetMapping("/close")
public Result closePlugin(@RequestParam("plugin")String plugin){
boolean res = InitPluginRegister.closePlugin(plugin);
if(!res){
return Result.error(plugin+" not register");
}
return Result.success(Map.of("plugin",plugin));
}
@GetMapping("/start")
public Result startPlugin(@RequestParam("plugin")String plugin){
if (InitPluginRegister.startPlugin(plugin)) {
return Result.success(Map.of("plugin",plugin));
}
return Result.error("The plugin has been started or does not exist");
}
}

View File

@@ -0,0 +1,73 @@
package org.example.exception;
import org.example.exception.Impl.ResultCode;
import org.example.exception.plugin.PluginDependOnException;
import org.example.exception.plugin.PluginException;
import org.example.exception.plugin.PluginNotRegisterException;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.util.Result;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.servlet.http.HttpServletRequest;
import java.util.Map;
/**
* @author : [Welsir]
* @description : [统一接口格式输出类]
* @createTime : [2023/5/19 20:20]
*/
@ControllerAdvice
@ResponseBody
public class GlobalExceptionHandler {
public static Logger logger = ChopperLogFactory.getLogger(LoggerType.System);
@ExceptionHandler(PluginException.class)
public Result handlerPluginException(HttpServletRequest request,PluginException ex){
logger.error("Handle Exception Request Url:{},Exception:{}", request.getRequestURL(), ex);
if(ex instanceof PluginNotRegisterException){
PluginNotRegisterException exception = (PluginNotRegisterException) ex;
return Result.error(exception.getResultCode());
}else if(ex instanceof PluginDependOnException){
PluginDependOnException exception = (PluginDependOnException) ex;
return Result.error(exception.getResultCode(), Map.of(
"fatherName",exception.getFatherName(),
"sonName",exception.getSonName()
));
}
return Result.error("error");
}
@ExceptionHandler(BaseException.class)
public Result handleException(HttpServletRequest request,
Exception ex) {
logger.error("Handle Exception Request Url:{},Exception:{}", request.getRequestURL(), ex);
Result result = new Result();
//系统异常
if (ex instanceof BaseException) {
BaseException se = (BaseException) ex;
ResultCode resultCode = se.getResultCode();
if (resultCode == null) {
result = Result.error(se.getMessage());
} else {
result = new Result(resultCode.getCode(),
StringUtils.isEmpty(se.getMessage()) ? se.getMessage() : resultCode.getMsg());
}
}
//参数错误
else {
result = new Result(ResultCode.ERROR.getCode(), ex.getMessage());
}
logger.info("exception handle result:" + result);
return result;
}
}

View File

@@ -0,0 +1,129 @@
package org.example.init;
import com.alibaba.fastjson.JSONObject;
import org.example.cache.FileCache;
import org.example.constpool.ConstPool;
import org.example.constpool.GlobalFileCache;
import org.example.constpool.ModuleName;
import org.example.constpool.PluginName;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.pojo.configfile.ChopperBotConfigFile;
import org.example.util.FileUtil;
import org.example.util.JsonFileUtil;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;
/**
* @author Genius
* @date 2023/04/20 18:34
**/
@Component
public class ChopperBotConfigFileInitMachine extends CommonInitMachine {
ChopperBotConfigFile chopperBotConfigFile;
private boolean initFlag;
public ChopperBotConfigFileInitMachine() {
super(ModuleName.CHOPPER_BOT,ChopperLogFactory.getLogger(LoggerType.System));
chopperBotConfigFile = new ChopperBotConfigFile();
initFlag = true;
isAutoStart = true;
pluginName = PluginName.CHOPPER_BOT_CONFIG_PLUGIN;
pluginClass = ChopperBotConfigFile.class;
plugin = chopperBotConfigFile;
}
@PostConstruct
@Override
public boolean init() {
Path dir = Paths.get(chopperBotConfigFile.getFilePath());
if (!createConfigDirectory(dir)) {
initFlag = fail("./config directory");
return initFlag;
}
if (!createConfigFile(dir)) {
initFlag = fail(ChopperBotConfigFile.getFullPath());
return initFlag;
}
if (!createModuleDirectory()) {
initFlag = fail("module directory");
return initFlag;
}
return initFlag;
}
private boolean createConfigDirectory(Path dir) {
try {
if (!Files.exists(dir)) {
Files.createDirectory(dir);
}
}catch (Exception e) {
return false;
}
return true;
}
private boolean createConfigFile(Path dir) {
Path path = Paths.get(dir.toString(), chopperBotConfigFile.getFileName());
try {
if (!Files.exists(path)) {
JsonFileUtil.writeJsonFile(path.toString(), chopperBotConfigFile.packageConfig());
GlobalFileCache.ModuleSrcConfigFile = new FileCache(chopperBotConfigFile);
}else{
Map<String, Object> data = JsonFileUtil.readJsonFile(Paths.get(dir.toString(), chopperBotConfigFile.getFileName()).toString());
chopperBotConfigFile.setData(data);
GlobalFileCache.ModuleSrcConfigFile = new FileCache(chopperBotConfigFile);
}
//全局插件注册表插入
InitPluginRegister.pluginStartSetting = JSONObject.parseObject(GlobalFileCache.ModuleSrcConfigFile.get("pluginStart").toString(),Map.class);
InitPluginRegister.allPlugins.put(PluginName.CHOPPER_BOT_CONFIG_PLUGIN,this);
}catch (Exception e) {
return false;
}
return true;
}
private boolean createModuleDirectory() {
Map<String, Object> moduleSrcConfigFileMap = JSONObject.parseObject(GlobalFileCache.ModuleSrcConfigFile.get("src").toString());
for (Map.Entry<String, Object> stringModuleConfigSrcEntry : moduleSrcConfigFileMap.entrySet()) {
String src = stringModuleConfigSrcEntry.getValue().toString();
try {
if (!FileUtil.isFileExist(src)) {
Files.createDirectory(Path.of(src));
}
}catch (Exception e) {
return false;
}
}
return true;
}
@Override
public boolean fail(String failCause) {
failLog(failCause);
return false;
}
@Override
public void failLog(String str) {
logger.error("[Wilderness] Create {} fail!Please delete your config directory.",str);
}
public boolean isInitFlag() {
return initFlag;
}
}

View File

@@ -0,0 +1,84 @@
package org.example.init;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.config.HotModuleConfig;
import org.example.constpool.GlobalFileCache;
import org.example.core.creeper.loadconfig.DouyuHotModuleConfig;
import org.example.core.manager.CreeperManager;
import org.example.exception.FileCacheException;
import org.example.plugin.annotation.Plugin;
import org.example.util.ClassUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.example.constpool.ConstPool.PROJECT_PATH;
/**
* @author Genius
* @date 2023/04/20 19:36
**/
@Component
//初始化整个项目,各个模块配置文件
public class InitWorld {
@Autowired
ChopperBotConfigFileInitMachine moduleSrcConfigFileInitMachine;
private ConfigurableApplicationContext ctx;
private InitWorld(ConfigurableApplicationContext ctx) {
this.ctx = ctx;
}
private void initPluginStartSetting() throws FileCacheException, InterruptedException {
GlobalFileCache.ModuleSrcConfigFile.write(InitPluginRegister.pluginStartSetting,"pluginStart");
}
@PostConstruct
private void init() throws FileCacheException, InterruptedException {
if (!InitPluginRegister.initPluginRegister()) {
close();
}
if(moduleSrcConfigFileInitMachine.isInitFlag()){
WorldInitMachine world = null;
try {
initPluginStartSetting();
world = new WorldInitMachine();
} catch (Exception e) {
close();
return;
}
if (world.init()) {
world.afterInit();
return;
}else{
world.shutdown();
}
}
close();
}
private void getAllPlugin(){
Set<Class<?>> annotationClass = ClassUtil.getAnnotationClass(PROJECT_PATH + ".init", Plugin.class);
System.out.println(annotationClass);
}
private void close(){
int exit = SpringApplication.exit(ctx, () -> 0);
System.exit(exit);
}
}

View File

@@ -0,0 +1,106 @@
package org.example.init;
import org.example.log.ChopperLogFactory;
import org.example.log.LoggerType;
import org.example.thread.ChopperBotGuardPool;
import org.example.thread.oddjob.OddJobBoy;
import org.example.util.PluginUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Supplier;
/**
* @author Genius
* @date 2023/07/22 17:52
**/
public class WorldInitMachine extends ModuleInitMachine{
private static final String githubUrl = "https://github.com/969025903/ChopperBot";
public WorldInitMachine() throws Exception {
super("ChopperBot",ChopperLogFactory.getLogger(LoggerType.System));
}
@Override
public boolean init() {
ChopperBotGuardPool.init();
OddJobBoy.Boy().guardian();
try {
initMachines = PluginUtil.getAllModuleInit();
return initLogger(()->{
if(checkNeedPlugin()){
for (CommonInitMachine initMachine : initMachines) {
if ((initMachine).checkNeedPlugin()) {
if(!initMachine.init()){
return fail();
}
(initMachine).registerPlugin();
}else{
return fail();
}
}
registerPlugin();
return success();
}
return fail();
});
} catch (Exception e) {
return fail(e.getMessage());
}
}
@Override
public void afterInit() {
try {
OddJobBoy.Boy().addWork(
()->{
getInitMachines().forEach(
InitMachine::afterInit
);
}
);
}catch (Exception e){
e.printStackTrace();
}
}
@Override
protected boolean initLogger(Supplier<Boolean> init) {
logger.info("🌏 <{}> Wake up,Find {} module need to init,please wait.....","ChopperBot",initMachines);
return init.get();
}
@Override
public void shutdown() {
logger.info("🌏 <{}> is shutting down,{} modules need to be closed,please wait.....","ChopperBot",getInitMachines().size());
ChopperBotGuardPool.GuardPool().shutdown();
this.getInitMachines().forEach(
InitMachine::shutdown
);
logger.info("🌏 <{}> all modules have been closed. Good Bye~🤗","ChopperBot");
}
@Override
public void successLog() {
successLog(String.format("🤖 <%s> Already Start,Click here http://localhost:8888/ to Enjoy it .You can goto the %s to support us,THX 😘! ","ChopperBot",githubUrl));
}
@Override
public void failLog(){
failLog(String.format("👻 <%s> Fail to Start,Please ensure that your module is correct or you can go to the " +
"%s write your error info 😥 ! ","ChopperBot",githubUrl));
}
}

View File

@@ -0,0 +1,19 @@
package org.example.service;
import org.example.pojo.vo.ConfigVO;
import java.util.List;
/**
* @author Genius
* @date 2023/04/26 00:59
**/
public interface FileService {
//获取所有模块
List<String> getAllModule();
//获取所有文件
List<ConfigVO> getAllConfigs();
}

View File

@@ -0,0 +1,14 @@
package org.example.service;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.springframework.stereotype.Service;
import java.util.List;
public interface HotModuleService {
HotModule getModuleHotLives(String platform, int moduleId);
}

View File

@@ -0,0 +1,10 @@
package org.example.service;
import org.example.bean.Plugin;
import java.util.List;
public interface PluginService {
List<Plugin> getPlugins(String moduleName);
}

View File

@@ -0,0 +1,46 @@
package org.example.service.impl;
import org.example.cache.FileCache;
import org.example.cache.FileCacheManagerInstance;
import org.example.bean.FileType;
import org.example.pojo.vo.ConfigVO;
import org.example.service.FileService;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* @author Genius
* @date 2023/04/26 01:01
**/
@Service
public class FileServiceImpl implements FileService {
@Override
public List<String> getAllModule() {
List<String> moduleList = new ArrayList<>();
for (FileType value : FileType.values()) {
if(!value.equals(FileType.COMMON)){
moduleList.add(value.getName());
}
}
return moduleList;
}
@Override
public List<ConfigVO> getAllConfigs() {
List<ConfigVO> configVOList = new ArrayList<>();
List<FileCache> runnableFileCaches = FileCacheManagerInstance.getInstance().getRunnableFileCaches();
for (FileCache runnableFileCache : runnableFileCaches) {
if (runnableFileCache.getFileType().equals(FileType.COMMON)) {
continue;
}
configVOList.add(new ConfigVO(runnableFileCache.getFileName()
,runnableFileCache.getFullFilePath()
,runnableFileCache.getFileType().getName()));
}
return configVOList;
}
}

View File

@@ -0,0 +1,44 @@
package org.example.service.impl;
import org.example.bean.Live;
import org.example.bean.HotModule;
import org.example.bean.hotmodule.HotModuleList;
import org.example.core.HotModuleDataCenter;
import org.example.service.HotModuleService;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @author Genius
* @date 2023/07/21 17:23
**/
@Service
public class HotModuleServiceImpl implements HotModuleService {
/**
* 获得热门模块下的热门直播
* @param moduleId
* @return
*/
@Override
public HotModule getModuleHotLives(String platform,int moduleId) {
HotModuleList moduleList = HotModuleDataCenter.DataCenter().getModuleList(platform);
if(moduleList==null){
return null;
}
HotModule hotModule = moduleList.findHotModule(moduleId);
if(hotModule!=null){
try {
List<? extends Live> moduleLiveList = HotModuleDataCenter.DataCenter().getModuleLiveList(platform, hotModule);
hotModule.setHotLives(moduleLiveList);
return hotModule;
}catch (Exception e){
//TODO 交给Spring全局异常处理器
return null;
}
}
return null;
}
}

View File

@@ -0,0 +1,54 @@
package org.example.service.impl;
import org.example.bean.Plugin;
import org.example.init.CommonInitMachine;
import org.example.init.InitPluginRegister;
import org.example.service.PluginService;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author Genius
* @date 2023/08/02 18:28
**/
@Service
public class PluginServiceImpl implements PluginService {
@Override
public List<Plugin> getPlugins(String moduleName) {
List<Plugin> list = new ArrayList<>();
Map<String,CommonInitMachine> commonInitMachineMap;
if(moduleName==null){
commonInitMachineMap = InitPluginRegister.allPlugins;
}else{
commonInitMachineMap = new HashMap<>();
for (String s : InitPluginRegister.modulePlugin.get(moduleName)) {
commonInitMachineMap.put(s,InitPluginRegister.allPlugins.get(s));
}
}
commonInitMachineMap.forEach(
(k,v)->{
list.add(packToPlugin(v));
}
);
return list;
}
private Plugin packToPlugin(CommonInitMachine commonInitMachine){
String moduleName = commonInitMachine.getModuleName();
String pluginName = commonInitMachine.getPluginName();
boolean autoStart = commonInitMachine.isAutoStart();
boolean register = InitPluginRegister.isRegister(pluginName);
List<String> needPlugins = commonInitMachine.getNeedPlugins();
String pluginName_cn = commonInitMachine.getPluginName_CN();
String pluginDescription = commonInitMachine.getPluginDescription();
return new Plugin(pluginName,moduleName,pluginName_cn,pluginDescription,needPlugins,autoStart,register);
}
}

View File

@@ -0,0 +1,9 @@
server:
port: 8888
spring:
datasource:
url: jdbc:mysql://localhost:3306/illuminator?useSSL=false&serverTimezone=UTC
username: root
password: root
driver-class-name: com.mysql.jdbc.Driver

View File

@@ -0,0 +1 @@
import request from '@/utils/request.ts';

View File

@@ -0,0 +1 @@
import request from '@/utils/request.ts';

View File

@@ -0,0 +1 @@
import request from '@/utils/request.ts';

View File

@@ -0,0 +1,25 @@
package org.example;
import com.genius.assistant.warmup.generate.AutoApiJsGenerate;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
/**
* @author Genius
* @date 2023/04/26 01:13
**/
@SpringBootTest
public class AutoGenerateApiJsTest {
@Resource
AutoApiJsGenerate autoApiJsGenerate;
@Test
public void autoGenerateApiJs(){
autoApiJsGenerate.setIsJsMoodleGenerated(true).setFileSavePath("E:\\Project\\ChopperBot\\console\\src\\main\\resources\\js");
autoApiJsGenerate.setAxiosPath("@/utils/request.ts");
autoApiJsGenerate.generate();
}
}

View File

@@ -0,0 +1,18 @@
package org.example;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
/**
* @author Genius
* @date 2023/08/06 13:50
**/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ConsoleApplication.class)
public class CreeperBarrageTest {
}