From 28e7e3bf8c4751f53cff56ed3acaa431be1c0802 Mon Sep 17 00:00:00 2001 From: userA Date: Mon, 24 Apr 2023 20:36:32 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=BC=93=E5=86=B2=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 12 + .../org/example/bean/ModuleSrcConfigFile.java | 7 +- .../java/org/example/cache/FileCache.java | 232 ++++++++++++++++++ .../org/example/cache/FileCacheManager.java | 103 ++++++++ .../java/org/example/util/JsonFileUtil.java | 6 +- FileModule/src/main/resources/student2.json | 4 +- .../java/org/example/cache/FileCacheTest.java | 45 ++++ .../org/example/util/JsonFileUtil.class | Bin 7802 -> 7820 bytes FileModule/target/classes/student2.json | 4 +- common/pom.xml | 10 + .../java/org/example/common/ConfigFile.java | 50 +++- .../example/exception/FileCacheException.java | 19 ++ .../main/java/org/example/util/TimeUtil.java | 19 ++ console/pom.xml | 5 + .../java/org/example/ConsoleApplication.java | 6 +- 15 files changed, 506 insertions(+), 16 deletions(-) create mode 100644 FileModule/src/main/java/org/example/cache/FileCache.java create mode 100644 FileModule/src/main/java/org/example/cache/FileCacheManager.java create mode 100644 FileModule/src/test/java/org/example/cache/FileCacheTest.java create mode 100644 common/src/main/java/org/example/exception/FileCacheException.java create mode 100644 common/src/main/java/org/example/util/TimeUtil.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cf55af..4d8a23b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,18 @@ ``` ------ +## [V 1.0.2] - 2023.4.24 +### common +- 🎈新增: 新增 `ConfigFile` 方法 `onlyUpdateTime,updateConfigTime` 负责更新外部数据上传时间和配置文件本身的上传时间 +- 🎈新增: 新增 `TimeUtil` 工具类，用于获取LocalDateTime的秒数 +- 🎈新增: 新增 `FileCacheException` 异常类，用于处理文件池异常 + +### FileModule +- 🎈新增: 新增 `FileCache` 文件缓冲池类，负责缓存文件内容，文件的读取，修改，追加，能够根据刷入时间或者写入字节，来进行自动刷盘操作 +- 🎈新增: 新增 `FileCacheManager` 文件缓冲池管理类，管理所有文件缓存池，轮询查看每个文件是否需要自动刷入，目前包含巡逻线程与刷入线程 + +------ + ## [V 1.0.1] - 2023.4.21 ### common - 🎈新增: 新增 `ConstPool` 常量池，用于存放常量，目前存放了模块名称常量，便于开发统一 diff --git a/FileModule/src/main/java/org/example/bean/ModuleSrcConfigFile.java b/FileModule/src/main/java/org/example/bean/ModuleSrcConfigFile.java index 0dec81b..35cac06 100644 --- a/FileModule/src/main/java/org/example/bean/ModuleSrcConfigFile.java +++ b/FileModule/src/main/java/org/example/bean/ModuleSrcConfigFile.java @@ -13,7 +13,7 @@ import java.util.Map; public class ModuleSrcConfigFile extends ConfigFile> { - private static final Map config; + private static Map config; public static class SRC{ private String src; @@ -24,6 +24,8 @@ public class ModuleSrcConfigFile extends ConfigFile packageConfig() { return super.packageConfig(); } + } diff --git a/FileModule/src/main/java/org/example/cache/FileCache.java b/FileModule/src/main/java/org/example/cache/FileCache.java new file mode 100644 index 0000000..9ca784c --- /dev/null +++ b/FileModule/src/main/java/org/example/cache/FileCache.java @@ -0,0 +1,232 @@ +package org.example.cache; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import org.example.common.ConfigFile; +import org.example.exception.FileCacheException; +import org.example.util.JsonFileUtil; +import org.example.util.TimeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Genius + * @date 2023/04/24 00:01 + **/ + +//TODO 文件缓存写入优化 考虑是否要加一个缓存写入Buffer，将短时间内多个相同Key的内容存入Buffer中，合并存入 +public class FileCache { + private T configFile; //文件配置类，得到文件信息和文件夹结构，更新文件存入时间 + private Logger logger; + + private ConcurrentHashMap jsonFile; //文件内容缓存 + + private ConcurrentHashMap oldJsonFileTemp; //保存上一个版本的文件内容，用于优化自动刷入 + private static final int MAX_WRITE_BUFFER_LIMIT = 4096; //最大写入缓存上线 + + private AtomicInteger writeByte; //当前写入的字节数 + + private BlockingQueue> syncChannel; //磁盘刷入阻塞队列 + + //TODO 优化 考虑是否采用一个定时线程管理所有FileCache的写入 + private ExecutorService pool; //Sync线程池 + + private long autoSyncTime; //自动刷入时间 + + public FileCache(T configFile) throws FileCacheException { + init(configFile,10); + } + + public FileCache(T configFile,long autoSyncTime)throws FileCacheException { + init(configFile,autoSyncTime); + } + + /** + * 初始化方法 + * @param configFile 配置文件 + * @param autoSyncTime 自动刷入时间 + * @throws FileCacheException + */ + private void init(T configFile, long autoSyncTime) throws FileCacheException { + this.configFile = configFile; + this.configFile.updateConfigTime();//更新一下当前的时间 + + this.logger = LoggerFactory.getLogger("FileCache:"+this.configFile.getFileName()); + + this.autoSyncTime = autoSyncTime; + if(!load(getFileName())){ + throw new FileCacheException("FileCache Init Error,please Check if your path is correct"); + } + + this.writeByte = new AtomicInteger(0); + this.syncChannel = new ArrayBlockingQueue<>(20); + this.pool = Executors.newSingleThreadExecutor(); + pool.submit(new SyncMan()); + } + + /** + * 加载文件内容 + * @return boolean + */ + private boolean load(String path){ + Map stringObjectMap = JsonFileUtil.readJsonFile(path); + if(Objects.isNull(stringObjectMap)){ + logger.error("{}配置文件不存在!",path); + return false; + } + + this.jsonFile = new ConcurrentHashMap<>(stringObjectMap); + this.oldJsonFileTemp = new ConcurrentHashMap<>(stringObjectMap); + return true; + } + + /** + * 重新加载内存池Map + * @return boolean + */ + public synchronized boolean reload(){ + return load(Paths.get(this.configFile.getFilePath(),this.configFile.getFileName()).toString()); + } + + //写入 + public int write(String key,Object data) throws InterruptedException { + String jsonDataStr = JSON.toJSONString(data); + int writeBytes = key.getBytes().length + jsonDataStr.getBytes().length; + + JSONObject jsonData = writeInData(key, data); + jsonFile.put("data",jsonData); + + ConcurrentHashMap temp = new ConcurrentHashMap<>(jsonFile); + int newBytes = writeByte.updateAndGet(x -> x + writeBytes >= MAX_WRITE_BUFFER_LIMIT ? 0 : x + writeBytes); + + //TODO 此处会发生脏读问题，即put进入的Map版本不是当前版本，但是目前没有发现该问题是否会影响到文件写入 + if(newBytes==0){ + logger.debug("缓冲区已满，刷入磁盘"); + syncChannel.put(temp); + } + return writeBytes; + } + + /** + * 追加内容 + * @param key key + * @param append 追加内容 + * @return + * @throws InterruptedException + */ + public int append(String key,Object append) throws InterruptedException { + Object data = getData(key); + StringBuffer buffer = new StringBuffer(JSON.toJSONString(data)); + String jsonStr = buffer.append(JSON.toJSONString(append)).toString(); + return write(key, jsonStr); + } + + private JSONObject writeInData(String key,Object value){ + JSONObject data = JSONObject.parseObject(jsonFile.get("data").toString()); //此处必须返回一个新的JsonObject，否则会导致旧版本同步更新 + data.put(key,value); + return data; + } + + /** + * 获取文件内容 + * @param key + * @return + */ + public Object get(String key){ + return jsonFile.get(key); + } + + /** + * 获取文件数据内容 + * @param key + * @return + */ + public Object getData(String key){ + Object data = this.get("data"); + JSONObject jsonObject = JSONObject.parseObject(data.toString()); + return jsonObject.get(key); + } + + /** + * 清除已写入的字节数记录 + */ + protected void clearWriteBytes(){ + writeByte.updateAndGet(x->0); + } + + /** + * 判断当前时间是否超过更新时间 + * @return boolean + */ + public boolean needAutoSync(){ + long now = TimeUtil.getCurrentSecond(); + return now- TimeUtil.getSecond(configFile.getUpdateTime())>autoSyncTime; + } + + /** + * 强制刷入磁盘 + */ + public void forceSync(){ + clearWriteBytes(); + ConcurrentHashMap temp = new ConcurrentHashMap<>(jsonFile); + try { + syncChannel.put(temp); + } catch (InterruptedException e) { + logger.debug("自动刷入失败"); + } + } + + /** + * 超过缓冲区刷入 + * @return + */ + private boolean sync(ConcurrentHashMap take){ + configFile.updateConfigTime(); //刷新配置文件刷入时间 + String dir = getFileName(); + if(oldJsonFileTemp.get("data").toString().equals(take.get("data").toString())){ + logger.debug("未发生版本变化"); + return true; + } + + configFile.onlyUpdateTime(take); + oldJsonFileTemp = new ConcurrentHashMap<>(take); + File file = JsonFileUtil.writeJsonFile(dir, take); + logger.debug("正在写入{}新版本",dir); + return Objects.isNull(file); + } + + public BlockingQueue getFileChannel(){ + return this.syncChannel; + } + + public String getFileName(){ + return Paths.get(this.configFile.getFilePath(), this.configFile.getFileName()).toString(); + } + + public long getSyncTime(){ + return this.autoSyncTime; + } + + class SyncMan implements Runnable{ + + @Override + public void run() { + for(;;){ + try { + ConcurrentHashMap take = syncChannel.take(); + sync(take); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + +} diff --git a/FileModule/src/main/java/org/example/cache/FileCacheManager.java b/FileModule/src/main/java/org/example/cache/FileCacheManager.java new file mode 100644 index 0000000..641205a --- /dev/null +++ b/FileModule/src/main/java/org/example/cache/FileCacheManager.java @@ -0,0 +1,103 @@ +package org.example.cache; + +import org.example.util.TimeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static java.lang.Thread.sleep; + +/** + * @author Genius + * @date 2023/04/24 17:35 + **/ + +/** + * 文件自动刷入管理类，不断监听文件是否需要自动写入 + */ +public class FileCacheManager { + + private Logger logger = LoggerFactory.getLogger(FileCacheManager.class); + private final List fileCaches; + + private long sleepTime; //睡眠时间 + + private ExecutorService watchPool; //巡逻线程 + + private ExecutorService autoSyncer; //生产者线程 + + public FileCacheManager(List fileCaches){ + this.fileCaches = fileCaches; + initSleepTime(); + this.watchPool = Executors.newSingleThreadExecutor(); + this.autoSyncer = Executors.newFixedThreadPool(fileCaches.size()); + } + + /** + * 根据文件缓存的刷盘时间得到一个最小的睡眠时间，减少空转 + */ + private void initSleepTime(){ + AtomicLong minSleepTime = new AtomicLong(Long.MAX_VALUE); + fileCaches.forEach(item->{ + minSleepTime.set(Long.min(minSleepTime.get(), item.getSyncTime())); + }); + this.sleepTime = minSleepTime.get(); + } + + public void start(){ + this.watchPool.submit(new Watcher()); + } + + class Watcher implements Runnable{ + + @Override + public void run() { + for(;;){ + long now = TimeUtil.getCurrentSecond(); + for(FileCache cache:fileCaches){ + BlockingQueue fileChannel = cache.getFileChannel(); + if(fileChannel.isEmpty()){ + if(cache.needAutoSync()){ + logger.debug("检测到需要强制刷新的文件 {}",cache.getFileName()); + autoSyncer.submit(new AutoSyncer(cache)); + } + } + } + now -= TimeUtil.getCurrentSecond(); + if(now;i%v1q%}5%ka@NM4XN++^=GZjt8*RKmkl0nRt+ghjc8%avd60s3>90!#Wj}0oG^5AKnKf|Y%8j?9^Uj| zFB^G_CvWqxPe*(r-Z8KryFEC-s&`eqr{jHmpyI=_Xp!r19Tezg9&pB(s~r*M2uDiU zW@iWC`beOgnX9!j*Lan(X38F?^07eMWOJk=)-2XEQ);`MUC`Csuf;U|MB-B%lK2dt zOY}fri6I!O;tPo{@s-5a_{MbW&697yFcsfQ9L5n9Mvqr%sx9blHN#+scouEj_k*okSJRN>t;7S#C5- z5D-G5i*QKcL{}+XLQ#cV3RQTd&~Qu&T^QzBBQ{0~Ly8C<>fnqNUQt&d*({U^W{y;1 zszsy}_1Hob57FqViWqaeOm+H*vE{Ob74ql94Dxp#)ENZF?m31^*WHDnnj3vGESpO&0+C&w z3z1!(lg*GjhDi4;MU-te_LoraZ1s1xdZ!%}3Zzc3V~T?XQ;b>GodFbMHg2V}Y{Tvt z?g}rIGiXgMkz#ODcx&>?bu^3qv+1v44y+QTo9{Sz@{sOes&hLW6>yp}x?Q-iE}%X`86%x8sruByB-*pY(F1w5UX?KqXoTWVrmUN~8&tpesx10=v*M zL-8w>XvMN4!_9JQ+N!4e-Q{SL?pMpvb`!kyF7>>+Q`(@$HC} z{_*21R{Bw-x6_?6ml$wk5nbB15#1`L(?wl`I|xZ1>iau!7bS2vA}sctj4jCC!9crd z*k^HOHv-=>wvBmYgZJAl$b2@0dIo7~cNA2it%^hy^G~D4Y3LTcVRugLHl2e7b{&be zWYkTjO3le}DI?FLcEE+H?LgFCeh#Zm0<2~z>3$`^+ZD_xnOt9ehzQFFkd~&cz)HG$ zUG$Za)o^I=Ixa*#YJoh9sEz@mR)rB&8YHR|3voBqt%eP?h_Y?uTSW1$MYJo7DCQ}> zhNmQ(r_twJ&Y(jz4R;S^baC)?+k{N2|x#w;cZh#>!;GSWnWoDZ%ODZed zw9P~c8Mnx6(X>5VY+o$fwH-w}=glh0ul{)Vop@u9`|*H=2eHmY8xJ}0 zu#QLYsD{U|USfk2kMs8vZ1p57pJM50UN&kd_u(0dO>AE2#Ab;q7N2$EIVY;wZi^E& zoQz*$D{pPnkc#cR?9ed^J9RvdS{=*q0x!EHUew{mZeCvEQ!n%KiW9H0vpxK=mzURg z*{8$3AGJ=r&R=gxys6OMxVym;-4hUq^ zhQ}DrzSrs z{ES~DepNVv@y2Q`>+GZWO<@{J6n@7a3Ue_}VJ1phS1OQdq%=*nEBuMSj99&+^>6%R zHfHH=+B}Biluw&ssK*I~-Z-fQkgJ3c7AdSs*o0jP2M#GAg;OBj1U#p# zs5o*dxmXyt2poB{}mC$(6QKy6wF#_$3c`7x@w`eh?Et)71%U+xE(hRv$#2IBO z-5QTvW1Tw3k)T9#;jY@FCR?1gqLQgI1k#K=*|utYtY1{6G)uLN_EE0_0!d5=NwFkT z4QS1yuS|MV;;^T0fv6-f2A?gt?F-nqaj%gR#_ik_t4aq2w;7G;QcX2~EHiB!RvIsk3w?^36p1VIn3` zaGmTXV+#3W5CO*c)YR~)0y@QTn!;L-t`a#Cg?%LAV?sO*MhlQd2v`vTn}(~*484i1 z%H5l2%NkY42`0V(tL|`d+`-~-62-)Cgw=p%1ZkLs7;Ct2#o?IK&6p(|lXOGr&6U^F zJJ@QBeTiypgxtcY&d|fW=w?U}E+iMP7+lC;uW;QW;bu^{nV1!Z=xGSC^nXBP_Ct*m zfXGZY3URg>v-C_5r=o?m28j)mSllSoNdznsnxPgn2DQv&HUu@TFv8^W|HL@Q?6626 zH|9{F(=g5nmyt2j&@*}6o|qCK!}p5Er& z=?pt{Y?JVfSrYDyC74gQhKJG;EWkptb&-d_jp17n3;S_&qASr}=SvYADj+w^e0mnp z#eQVH2o + + com.alibaba + fastjson + 1.2.75 + junit junit 3.8.1 test + + org.springframework.boot + spring-boot-starter-test + test + diff --git a/common/src/main/java/org/example/common/ConfigFile.java b/common/src/main/java/org/example/common/ConfigFile.java index 78fa6c4..17538a0 100644 --- a/common/src/main/java/org/example/common/ConfigFile.java +++ b/common/src/main/java/org/example/common/ConfigFile.java @@ -9,26 +9,58 @@ import java.util.Map; * @date 2023/04/21 02:24 **/ -//配置文件的抽象类 +//配置文件的抽象类，只负责构建配置文件最基础的架构，一般不用来存放配置文件本身的内容 public abstract class ConfigFile { private String filePath; private String fileName; - private T data; + private T data; //json文件的结构不是文件的数据 例如 {username:"",password:""} + + //上一次更新时间 + private LocalDateTime updateTime; /** - * 用于打包配置文件 + * 用于最开始创建配置文件结构的打包 + * @return Map */ public Map packageConfig() { + return this.packageConfig(this.data); + } - + /** + * 用于给外部函数提供的内容打包 + * @return + */ + public Map packageConfig(T data){ + updateConfigTime(); return Map.of( "data",data, - "updateTime", LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + "updateTime", updateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) ); } + /** + * 只进行时间更新操作 + * @param map + * @return + */ + public Map onlyUpdateTime(Map map){ + updateConfigTime(); + if (map.containsKey("updateTime")) { + map.put("updateTime",updateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))); + } + return map; + } + + /** + * 更新配置文件类本身的时间 + */ + public void updateConfigTime(){ + updateTime = LocalDateTime.now(); + } + + public ConfigFile() { } @@ -36,6 +68,7 @@ public abstract class ConfigFile { this.filePath = filePath; this.fileName = fileName; this.data = data; + this.updateTime = LocalDateTime.now(); } public String getFilePath() { @@ -46,7 +79,12 @@ public abstract class ConfigFile { return fileName; } - protected T getData() { + public LocalDateTime getUpdateTime() {return updateTime;} + + //不推荐直接使用 + public T getData() { return data; } + + public void setData(T data){this.data = data;} } diff --git a/common/src/main/java/org/example/exception/FileCacheException.java b/common/src/main/java/org/example/exception/FileCacheException.java new file mode 100644 index 0000000..f8c4374 --- /dev/null +++ b/common/src/main/java/org/example/exception/FileCacheException.java @@ -0,0 +1,19 @@ +package org.example.exception; + +/** + * @author Genius + * @date 2023/04/24 00:57 + **/ +public class FileCacheException extends Exception{ + + String message; + + public FileCacheException(String ErrorMessage){ + this.message = ErrorMessage; + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/common/src/main/java/org/example/util/TimeUtil.java b/common/src/main/java/org/example/util/TimeUtil.java new file mode 100644 index 0000000..93ccac7 --- /dev/null +++ b/common/src/main/java/org/example/util/TimeUtil.java @@ -0,0 +1,19 @@ +package org.example.util; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +/** + * @author Genius + * @date 2023/04/24 18:47 + **/ +public class TimeUtil { + + public static long getCurrentSecond(){ + return LocalDateTime.now().toEpochSecond(ZoneOffset.of("+8")); + } + + public static long getSecond(LocalDateTime dateTime){ + return dateTime.toEpochSecond(ZoneOffset.of("+8")); + } +} diff --git a/console/pom.xml b/console/pom.xml index cb4a981..5894b60 100644 --- a/console/pom.xml +++ b/console/pom.xml @@ -33,6 +33,11 @@ FileModule 1.0-SNAPSHOT + + org.springframework.boot + spring-boot-starter-test + test + diff --git a/console/src/main/java/org/example/ConsoleApplication.java b/console/src/main/java/org/example/ConsoleApplication.java index 996e45d..b585518 100644 --- a/console/src/main/java/org/example/ConsoleApplication.java +++ b/console/src/main/java/org/example/ConsoleApplication.java @@ -1,6 +1,9 @@ package org.example; +import org.example.bean.ModuleSrcConfigFile; +import org.example.cache.FileCache; +import org.example.exception.FileCacheException; import org.example.init.InitWorld; import org.example.init.ModuleSrcConfigFileInit; import org.springframework.boot.SpringApplication; @@ -16,11 +19,12 @@ import java.util.List; @SpringBootApplication public class ConsoleApplication { - public static void main(String[] args) { + public static void main(String[] args) throws FileCacheException { if (InitWorld.getInstance() .setInitMachines(List.of(new ModuleSrcConfigFileInit())) .start()) { + SpringApplication.run(ConsoleApplication.class, args); }