diff --git a/.dockerignore b/.dockerignore index c68e752..e6bde75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,5 +7,4 @@ pnpm-lock.yaml test/* Videos/* settings-production.toml -startRecord-production.sh -src/utils/cookies.json \ No newline at end of file +startRecord-production.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7286375..14dd027 100755 --- a/.gitignore +++ b/.gitignore @@ -381,6 +381,4 @@ cookies.json src/subtitle/models/base.pt src/subtitle/models/small.pt src/burn/mergevideo.txt -src/upload/upload.yaml -src/upload/uploadVideoQueue.txt src/db/data.db \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index dd425a3..e0e6d68 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/autoslice/auto_slice_video"] path = src/autoslice/auto_slice_video url = https://github.com/timerring/auto-slice-video.git +[submodule "src/upload/bilitool"] + path = src/upload/bilitool + url = https://github.com/timerring/bilitool.git diff --git a/src/burn/render_then_merge.py b/src/burn/render_then_merge.py index cb33c8a..275a13d 100644 --- a/src/burn/render_then_merge.py +++ b/src/burn/render_then_merge.py @@ -10,8 +10,6 @@ from src.burn.render_command import render_command from src.upload.extract_video_info import get_video_info from src.log.logger import scan_log from db.conn import insert_upload_queue -from src.upload.generate_yaml import generate_yaml_template -from uuid import uuid4 def normalize_video_path(filepath): """Normalize the video path to upload @@ -92,11 +90,6 @@ def render_then_merge(video_path_list): merge_command(output_video_path, title, artist, date, merge_list) subprocess.run(['rm', '-r', tmp]) - - yaml_template = generate_yaml_template(output_video_path) - template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml') - with open(template_path, 'w', encoding='utf-8') as f: - f.write(yaml_template) - if not insert_upload_queue(output_video_path, template_path): - scan_log('插入待上传条目失败') \ No newline at end of file + if not insert_upload_queue(output_video_path): + scan_log.error('Cannot insert the video to the upload queue') \ No newline at end of file diff --git a/src/burn/render_video.py b/src/burn/render_video.py index 74f6932..5457a62 100644 --- a/src/burn/render_video.py +++ b/src/burn/render_video.py @@ -13,8 +13,6 @@ from src.autoslice.zhipu_sdk import zhipu_glm_4v_plus_generate_title from src.upload.extract_video_info import get_video_info from src.log.logger import scan_log from db.conn import insert_upload_queue -from src.upload.generate_yaml import generate_yaml_template, generate_slice_yaml_template -from uuid import uuid4 def normalize_video_path(filepath): """Normalize the video path to upload @@ -72,13 +70,8 @@ def render_video(video_path): slice_video_flv_path = slice_path[:-4] + '.flv' inject_metadata(slice_path, glm_title, slice_video_flv_path) os.remove(slice_path) - slice_yaml_template = generate_slice_yaml_template(slice_video_flv_path) - slice_template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml') - with open(slice_template_path, 'w', encoding='utf-8') as f: - f.write(slice_yaml_template) - - if not insert_upload_queue(slice_video_flv_path, slice_template_path): - scan_log('插入待上传条目失败') + if not insert_upload_queue(slice_video_flv_path): + scan_log.error('Cannot insert the video to the upload queue') except Exception as e: scan_log.error(f"Error in {slice_path}: {e}") @@ -91,10 +84,5 @@ def render_video(video_path): # test_path = original_video_path[:-4] # os.rename(original_video_path, test_path) - yaml_template = generate_yaml_template(format_video_path) - template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml') - with open(template_path, 'w', encoding='utf-8') as f: - f.write(yaml_template) - - if not insert_upload_queue(format_video_path, template_path): - scan_log('插入待上传条目失败') + if not insert_upload_queue(format_video_path): + scan_log.error('Cannot insert the video to the upload queue') diff --git a/src/config.py b/src/config.py index 74534a1..9a0f4fa 100644 --- a/src/config.py +++ b/src/config.py @@ -36,7 +36,7 @@ VIDEOS_DIR = os.path.join(BILIVE_DIR, 'Videos') if not os.path.exists(SRC_DIR + '/db/data.db'): - print("初始化数据库") + print("Initialize the database") create_table() if not os.path.exists(VIDEOS_DIR): diff --git a/src/db/conn.py b/src/db/conn.py index cf88c8e..c3deeb1 100644 --- a/src/db/conn.py +++ b/src/db/conn.py @@ -13,7 +13,7 @@ def create_table(): db = connect() cursor = db.cursor() sql = [ - "create table upload_queue (id integer primary key autoincrement, video_path text, config_path text, locked integer default 0);", + "create table upload_queue (id integer primary key autoincrement, video_path text, locked integer default 0);", "create unique index idx_video_path on upload_queue(video_path);", ] for s in sql: @@ -28,17 +28,26 @@ def create_table(): def get_single_upload_queue(): db = connect() cursor = db.cursor() - cursor.execute("select video_path, config_path from upload_queue where locked = 0 limit 1;") + cursor.execute("select video_path from upload_queue where locked = 0 limit 1;") row = cursor.fetchone() - result = {'video_path': row[0], 'config_path': row[1]} if row else None + result = {'video_path': row[0]} if row else None db.close() return result -def insert_upload_queue(video_path: str, config_path: str): +def get_all_upload_queue(): + db = connect() + cursor = db.cursor() + cursor.execute("select video_path from upload_queue;") + rows = cursor.fetchall() + result = [{'video_path': row[0]} for row in rows] + db.close() + return result + +def insert_upload_queue(video_path: str): try: db = connect() cursor = db.cursor() - cursor.execute("insert into upload_queue (video_path, config_path) values (?, ?);", (video_path, config_path)) + cursor.execute("insert into upload_queue (video_path) values (?);", (video_path,)) db.commit() db.close() return True @@ -76,12 +85,16 @@ if __name__ == "__main__": # Create Table create_table() # Insert Test Data - insert_upload_queue('test.mp4', 'config.yaml') + insert_upload_queue('') # Insert again to check the unique index - print(insert_upload_queue('test.mp4', 'config.yaml')) - # Get the single upload queue, shold be {'video_path': 'test.mp4', 'config_path': 'config.yaml'} - print(get_single_upload_queue()) + # print(insert_upload_queue('')) + # Get the single upload queue, shold be {'video_path': 'test.mp4'} + # print(get_single_upload_queue()) + # Get all upload queue + print(get_all_upload_queue()) + # unlock the upload queue + update_upload_queue_lock('test.mp4', 0) # Delete the upload queue - delete_upload_queue('test.mp4') + delete_upload_queue('') # Get the single upload queue after delete, should be None print(get_single_upload_queue()) \ No newline at end of file diff --git a/src/upload/__init__.py b/src/upload/__init__.py index 3b36319..ad2d430 100644 --- a/src/upload/__init__.py +++ b/src/upload/__init__.py @@ -2,4 +2,7 @@ import sys import os -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) \ No newline at end of file +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from .bilitool.bilitool import UploadController, FeedController + +__all__ = ['UploadController', 'FeedController'] \ No newline at end of file diff --git a/src/upload/bilitool b/src/upload/bilitool new file mode 160000 index 0000000..facd698 --- /dev/null +++ b/src/upload/bilitool @@ -0,0 +1 @@ +Subproject commit facd698944d20acbda418be1db0f8c57d32fcc8a diff --git a/src/upload/extract_video_info.py b/src/upload/extract_video_info.py index 0ae9e78..567dfd8 100644 --- a/src/upload/extract_video_info.py +++ b/src/upload/extract_video_info.py @@ -7,6 +7,7 @@ import os from datetime import datetime from src.upload.query_search_suggestion import get_bilibili_suggestions from src.config import GPU_EXIST, TITLE, DESC +from src.log.logger import upload_log def get_video_info(video_file_path): """get the title, artist and date of the video file via ffprobe @@ -15,28 +16,35 @@ def get_video_info(video_file_path): Returns: str: the title of the video file, if failed, return None """ - - command = [ - "ffprobe", - "-v", "quiet", - "-print_format", "json", - "-show_format", - video_file_path - ] - output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8') - parsed_output = json.loads(output) - title_value = parsed_output["format"]["tags"]["title"] - artist_value = parsed_output["format"]["tags"]["artist"] - date_value = parsed_output["format"]["tags"]["date"] - if len(date_value) > 8: - dt = datetime.fromisoformat(date_value) - new_date = dt.strftime('%Y%m%d') - else: - new_date = date_value - return title_value, artist_value, new_date + try: + command = [ + "ffprobe", + "-v", "quiet", + "-print_format", "json", + "-show_format", + video_file_path + ] + output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8') + parsed_output = json.loads(output) + title_value = parsed_output["format"]["tags"]["title"] + artist_value = parsed_output["format"]["tags"]["artist"] + date_value = parsed_output["format"]["tags"]["date"] + if len(date_value) > 8: + dt = datetime.fromisoformat(date_value) + new_date = dt.strftime('%Y%m%d') + else: + new_date = date_value + return title_value, artist_value, new_date + except Exception as e: + # Log the exception if needed + upload_log.error(f"Error occurred in get_video_info: {e}") + return None, None, None def generate_title(video_path): title, artist, date = get_video_info(video_path) + if title is None: + upload_log.error(f"Error occurred in generate_title: {title}") + return None source_link = generate_source(video_path) prefix = "【弹幕+字幕】" if GPU_EXIST else "【弹幕】" formatted_title = TITLE.format(artist=artist, date=date, title=title, source_link=source_link) diff --git a/src/upload/generate_upload_data.py b/src/upload/generate_upload_data.py new file mode 100644 index 0000000..4b59cec --- /dev/null +++ b/src/upload/generate_upload_data.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024 bilive. + +import os +import time +import codecs +from datetime import datetime +from src.upload.extract_video_info import generate_title, generate_desc, generate_tag, generate_source +import subprocess +import json + +def generate_video_data(video_path): + copyright = 1 + title = generate_title(video_path) + desc = generate_desc(video_path) + tid = 138 + tag = generate_tag(video_path) + source = generate_source(video_path) + cover = "" + dynamic = "" + return copyright, title, desc, tid, tag, source, cover, dynamic + +def generate_slice_data(video_path): + command = [ + "ffprobe", + "-v", "quiet", + "-print_format", "json", + "-show_format", + video_path + ] + output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8') + parsed_output = json.loads(output) + title = parsed_output["format"]["tags"]["generate"] + copyright = 1 + tid = 138 + tag = "直播切片" + return copyright, title, tid, tag + +if __name__ == "__main__": + pass diff --git a/src/upload/generate_yaml.py b/src/upload/generate_yaml.py deleted file mode 100644 index 8fcf506..0000000 --- a/src/upload/generate_yaml.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2024 bilive. - -import os -import time -import yaml -import codecs -from datetime import datetime -from src.upload.extract_video_info import generate_title, generate_desc, generate_tag, generate_source -import subprocess -import json - -def generate_yaml_template(video_path): - source = generate_source(video_path) - title = generate_title(video_path) - desc = generate_desc(video_path) - tag = generate_tag(video_path) - data = { - "line": "bda2", - "limit": 5, - "streamers": { - video_path: { - "copyright": 1, - "source": source, - "tid": 138, - "cover": "", - "title": title, - "desc_format_id": 0, - "desc": desc, - "dynamic": "", - "tag": tag - } - } - } - return yaml.dump(data, default_flow_style=False, sort_keys=False) - -def generate_slice_yaml_template(video_path): - command = [ - "ffprobe", - "-v", "quiet", - "-print_format", "json", - "-show_format", - video_path - ] - output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8') - parsed_output = json.loads(output) - title = parsed_output["format"]["tags"]["generate"] - data = { - "line": "bda2", - "limit": 5, - "streamers": { - video_path: { - "copyright": 1, - "source": "", - "tid": 138, - "cover": "", - "title": title, - "desc_format_id": 0, - "desc": "", - "dynamic": "", - "tag": "直播切片" - } - } - } - return yaml.dump(data, default_flow_style=False, sort_keys=False) - -if __name__ == "__main__": - # read the queue and upload the video - yaml_template = generate_yaml_template("") - with open('upload.yaml', 'w', encoding='utf-8') as file: - file.write(yaml_template) diff --git a/src/upload/upload.py b/src/upload/upload.py index 0d16cec..344c1ad 100644 --- a/src/upload/upload.py +++ b/src/upload/upload.py @@ -5,7 +5,7 @@ import os import sys from src.config import SRC_DIR, BILIVE_DIR from datetime import datetime -from src.upload.generate_yaml import generate_yaml_template, generate_slice_yaml_template +from src.upload.generate_upload_data import generate_video_data, generate_slice_data from src.upload.extract_video_info import generate_title from src.log.logger import upload_log import time @@ -13,31 +13,21 @@ import fcntl from concurrent.futures import ThreadPoolExecutor, as_completed from db.conn import get_single_upload_queue, delete_upload_queue, update_upload_queue_lock import threading +from .bilitool.bilitool import UploadController, FeedController, LoginController -read_lock = threading.Lock() +# read_lock = threading.Lock() -def upload_video(upload_path, yaml_file_path): +def upload_video(upload_path): try: - # Construct the command - command = [ - f"{SRC_DIR}/utils/biliup", - "-u", - f"{SRC_DIR}/utils/cookies.json", - "upload", - upload_path, - "--config", - yaml_file_path - ] - - # Execute the command - result = subprocess.run(command, check=True, capture_output=True, text=True) - upload_log.debug(f"Upload output:\nstdout: {result.stdout}\nstderr: {result.stderr}") - - # Check if the command was successful - if result.returncode == 0: + if upload_path.endswith('.flv'): + copyright, title, tid, tag = generate_slice_data(upload_path) + else: + copyright, title, desc, tid, tag, source, cover, dynamic = generate_video_data(upload_path) + yaml = "" + result = UploadController().upload_video_entry(upload_path, yaml, copyright, tid, title, desc, tag, source, cover, dynamic) + if result == True: upload_log.info("Upload successfully, then delete the video") os.remove(upload_path) - os.remove(yaml_file_path) delete_upload_queue(upload_path) else: upload_log.error("Fail to upload, the files will be reserved.") @@ -60,22 +50,9 @@ def find_bv_number(target_str, my_list): def append_upload(upload_path, bv_result): try: - # Construct the command - command = [ - f"{SRC_DIR}/utils/biliup", - "-u", - f"{SRC_DIR}/utils/cookies.json", - "append", - "--vid", - bv_result, - upload_path - ] - # Execute the command - result = subprocess.run(command, check=True, capture_output=True, text=True) - upload_log.debug(f"Append output:\nstdout: {result.stdout}\nstderr: {result.stderr}") - + result = UploadController().append_video_entry(upload_path, bv_result) # Check if the command was successful - if result.returncode == 0: + if result == True: upload_log.info("Upload successfully, then delete the video") os.remove(upload_path) delete_upload_queue(upload_path) @@ -92,51 +69,54 @@ def append_upload(upload_path, bv_result): def read_append_and_delete_lines(): while True: - upload_queue = None + # upload_queue = None # read the queue and update lock status to prevent other threads from reading the same data - with read_lock: - upload_queue = get_single_upload_queue() - # if there is a task in the queue, try to lock the task - if upload_queue: - video_path, config_path = upload_queue.values() - # lock the task by updating the locked status to 1 - update_result = update_upload_queue_lock(video_path, 1) - # if failed to lock, log the error and let the next thread to handle the task - if not update_result: - upload_log.error(f"Failed to lock task for {video_path}, possibly already locked by another thread or database error.") - upload_queue = None - continue - else: - upload_log.info("Empty queue, wait 2 minutes and check again.") - time.sleep(120) - continue - + if LoginController().check_bilibili_login(): + pass + else: + LoginController().login_bilibili(export=False) + continue + # with read_lock: + upload_queue = get_single_upload_queue() + # if there is a task in the queue, try to lock the task if upload_queue: - video_path, config_path = upload_queue.values() + video_path = upload_queue['video_path'] + time.sleep(3) + query = generate_title(video_path) + if query is None: + if not os.path.exists(video_path): + delete_upload_queue(video_path) + continue + else: + upload_log.error(f"Error occurred in ffprobe: {video_path}") + update_upload_queue_lock(video_path, 1) + continue upload_log.info(f"deal with {video_path}") # check if the live is already uploaded if video_path.endswith('.flv'): # upload slice video - upload_video(video_path, config_path) - return + upload_video(video_path) else: - query = generate_title(video_path) - result = subprocess.check_output("bilitool" + " list", shell=True) - # print(result.decode("utf-8"), flush=True) - upload_list = result.decode("utf-8").splitlines() - bv_result = find_bv_number(query, upload_list) + upload_dict = FeedController().get_video_dict_info(20, "pubed,not_pubed,is_pubing") + # result = subprocess.check_output("bilitool" + " list", shell=True) + # # print(result.decode("utf-8"), flush=True) + # upload_list = result.decode("utf-8").splitlines() + bv_result = upload_dict.get(query) if bv_result: upload_log.info(f"The series of videos has already been uploaded, the BV number is: {bv_result}") append_upload(video_path, bv_result) else: upload_log.info("First upload this live") - upload_video(video_path, config_path) - return - time.sleep(20) + upload_video(video_path) + time.sleep(20) + else: + upload_log.info("Empty queue, wait 2 minutes and check again.") + time.sleep(120) if __name__ == "__main__": - max_workers = os.getenv("MAX_WORKERS", 5) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_to_upload = {executor.submit(read_append_and_delete_lines) for _ in range(max_workers)} + # max_workers = os.getenv("MAX_WORKERS", 1) + # with ThreadPoolExecutor(max_workers=max_workers) as executor: + # future_to_upload = {executor.submit(read_append_and_delete_lines) for _ in range(max_workers)} + read_append_and_delete_lines() \ No newline at end of file diff --git a/src/upload/uploadVideoQueue.txt b/src/upload/uploadVideoQueue.txt deleted file mode 100644 index e69de29..0000000 diff --git a/src/utils/biliup b/src/utils/biliup deleted file mode 100755 index 380ba5c..0000000 Binary files a/src/utils/biliup and /dev/null differ