feat: introduce bilitool (#246)

* feat: introduce bilitool
* fix: remove config file
* fix: remove abundant dependencies

.dockerignore

@@ -7,5 +7,4 @@ pnpm-lock.yaml
 test/*
 Videos/*
 settings-production.toml
-startRecord-production.sh
-src/utils/cookies.json
+startRecord-production.sh

.gitignore

@@ -381,6 +381,4 @@ cookies.json
 src/subtitle/models/base.pt
 src/subtitle/models/small.pt
 src/burn/mergevideo.txt
-src/upload/upload.yaml
-src/upload/uploadVideoQueue.txt
 src/db/data.db

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "src/autoslice/auto_slice_video"]
 	path = src/autoslice/auto_slice_video
 	url = https://github.com/timerring/auto-slice-video.git
+[submodule "src/upload/bilitool"]
+	path = src/upload/bilitool
+	url = https://github.com/timerring/bilitool.git

src/burn/render_then_merge.py

@@ -10,8 +10,6 @@ from src.burn.render_command import render_command
 from src.upload.extract_video_info import get_video_info
 from src.log.logger import scan_log
 from db.conn import insert_upload_queue
-from src.upload.generate_yaml import generate_yaml_template
-from uuid import uuid4
 
 def normalize_video_path(filepath):
     """Normalize the video path to upload
@@ -92,11 +90,6 @@ def render_then_merge(video_path_list):
         
     merge_command(output_video_path, title, artist, date, merge_list)
     subprocess.run(['rm', '-r', tmp])
-
-    yaml_template = generate_yaml_template(output_video_path)
-    template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml')
-    with open(template_path, 'w', encoding='utf-8') as f:
-        f.write(yaml_template)
         
-    if not insert_upload_queue(output_video_path, template_path):
-        scan_log('插入待上传条目失败')
+    if not insert_upload_queue(output_video_path):
+        scan_log.error('Cannot insert the video to the upload queue')

src/burn/render_video.py

@@ -13,8 +13,6 @@ from src.autoslice.zhipu_sdk import zhipu_glm_4v_plus_generate_title
 from src.upload.extract_video_info import get_video_info
 from src.log.logger import scan_log
 from db.conn import insert_upload_queue
-from src.upload.generate_yaml import generate_yaml_template, generate_slice_yaml_template
-from uuid import uuid4
 
 def normalize_video_path(filepath):
     """Normalize the video path to upload
@@ -72,13 +70,8 @@ def render_video(video_path):
                     slice_video_flv_path = slice_path[:-4] + '.flv'
                     inject_metadata(slice_path, glm_title, slice_video_flv_path)
                     os.remove(slice_path)
-                    slice_yaml_template = generate_slice_yaml_template(slice_video_flv_path)
-                    slice_template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml')
-                    with open(slice_template_path, 'w', encoding='utf-8') as f:
-                        f.write(slice_yaml_template)
-
-                    if not insert_upload_queue(slice_video_flv_path, slice_template_path):
-                        scan_log('插入待上传条目失败')
+                    if not insert_upload_queue(slice_video_flv_path):
+                        scan_log.error('Cannot insert the video to the upload queue')
                 except Exception as e:
                     scan_log.error(f"Error in {slice_path}: {e}")
 
@@ -91,10 +84,5 @@ def render_video(video_path):
     # test_path = original_video_path[:-4]
     # os.rename(original_video_path, test_path)
     
-    yaml_template = generate_yaml_template(format_video_path)
-    template_path = os.path.join(VIDEOS_DIR, f'upload_conf/{uuid4()}.yaml')
-    with open(template_path, 'w', encoding='utf-8') as f:
-        f.write(yaml_template)
-        
-    if not insert_upload_queue(format_video_path, template_path):
-        scan_log('插入待上传条目失败')
+    if not insert_upload_queue(format_video_path):
+        scan_log.error('Cannot insert the video to the upload queue')

src/config.py

@@ -36,7 +36,7 @@ VIDEOS_DIR = os.path.join(BILIVE_DIR, 'Videos')
 
 
 if not os.path.exists(SRC_DIR + '/db/data.db'):
-    print("初始化数据库")
+    print("Initialize the database")
     create_table()
 
 if not os.path.exists(VIDEOS_DIR):

src/db/conn.py

@@ -13,7 +13,7 @@ def create_table():
         db = connect()
         cursor = db.cursor()
         sql = [
-            "create table upload_queue (id integer primary key autoincrement, video_path text, config_path text, locked integer default 0);",
+            "create table upload_queue (id integer primary key autoincrement, video_path text, locked integer default 0);",
             "create unique index idx_video_path on upload_queue(video_path);",
         ]
         for s in sql:
@@ -28,17 +28,26 @@ def create_table():
 def get_single_upload_queue():
     db = connect()
     cursor = db.cursor()
-    cursor.execute("select video_path, config_path from upload_queue where locked = 0 limit 1;")
+    cursor.execute("select video_path from upload_queue where locked = 0 limit 1;")
     row = cursor.fetchone()
-    result = {'video_path': row[0], 'config_path': row[1]} if row else None
+    result = {'video_path': row[0]} if row else None
     db.close()
     return result
 
-def insert_upload_queue(video_path: str, config_path: str):
+def get_all_upload_queue():
+    db = connect()
+    cursor = db.cursor()
+    cursor.execute("select video_path from upload_queue;")
+    rows = cursor.fetchall()
+    result = [{'video_path': row[0]} for row in rows]
+    db.close()
+    return result
+
+def insert_upload_queue(video_path: str):
     try:
         db = connect()
         cursor = db.cursor()
-        cursor.execute("insert into upload_queue (video_path, config_path) values (?, ?);", (video_path, config_path))
+        cursor.execute("insert into upload_queue (video_path) values (?);", (video_path,))
         db.commit()
         db.close()
         return True
@@ -76,12 +85,16 @@ if __name__ == "__main__":
     # Create Table
     create_table()
     # Insert Test Data
-    insert_upload_queue('test.mp4', 'config.yaml')
+    insert_upload_queue('')
     # Insert again to check the unique index
-    print(insert_upload_queue('test.mp4', 'config.yaml'))
-    # Get the single upload queue, shold be {'video_path': 'test.mp4', 'config_path': 'config.yaml'}
-    print(get_single_upload_queue())
+    # print(insert_upload_queue(''))
+    # Get the single upload queue, shold be {'video_path': 'test.mp4'}
+    # print(get_single_upload_queue())
+    # Get all upload queue
+    print(get_all_upload_queue())
+    # unlock the upload queue
+    update_upload_queue_lock('test.mp4', 0)
     # Delete the upload queue
-    delete_upload_queue('test.mp4')
+    delete_upload_queue('')
     # Get the single upload queue after delete, should be None
     print(get_single_upload_queue())

src/upload/__init__.py

@@ -2,4 +2,7 @@
 
 import sys
 import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from .bilitool.bilitool import UploadController, FeedController
+
+__all__ = ['UploadController', 'FeedController']

src/upload/extract_video_info.py

@@ -7,6 +7,7 @@ import os
 from datetime import datetime
 from src.upload.query_search_suggestion import get_bilibili_suggestions
 from src.config import GPU_EXIST, TITLE, DESC
+from src.log.logger import upload_log
 
 def get_video_info(video_file_path):
     """get the title, artist and date of the video file via ffprobe
@@ -15,28 +16,35 @@ def get_video_info(video_file_path):
     Returns:
         str: the title of the video file, if failed, return None
     """
-
-    command = [
-        "ffprobe",
-        "-v", "quiet",
-        "-print_format", "json",
-        "-show_format",
-        video_file_path
-    ]
-    output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8')
-    parsed_output = json.loads(output)
-    title_value = parsed_output["format"]["tags"]["title"]
-    artist_value = parsed_output["format"]["tags"]["artist"]
-    date_value = parsed_output["format"]["tags"]["date"]
-    if len(date_value) > 8:
-        dt = datetime.fromisoformat(date_value)
-        new_date = dt.strftime('%Y%m%d')
-    else:
-        new_date = date_value
-    return title_value, artist_value, new_date
+    try:
+        command = [
+            "ffprobe",
+            "-v", "quiet",
+            "-print_format", "json",
+            "-show_format",
+            video_file_path
+        ]
+        output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8')
+        parsed_output = json.loads(output)
+        title_value = parsed_output["format"]["tags"]["title"]
+        artist_value = parsed_output["format"]["tags"]["artist"]
+        date_value = parsed_output["format"]["tags"]["date"]
+        if len(date_value) > 8:
+            dt = datetime.fromisoformat(date_value)
+            new_date = dt.strftime('%Y%m%d')
+        else:
+            new_date = date_value
+        return title_value, artist_value, new_date
+    except Exception as e:
+        # Log the exception if needed
+        upload_log.error(f"Error occurred in get_video_info: {e}")
+        return None, None, None
 
 def generate_title(video_path):
     title, artist, date = get_video_info(video_path)
+    if title is None:
+        upload_log.error(f"Error occurred in generate_title: {title}")
+        return None
     source_link = generate_source(video_path)
     prefix = "【弹幕+字幕】" if GPU_EXIST else "【弹幕】"
     formatted_title = TITLE.format(artist=artist, date=date, title=title, source_link=source_link)

src/upload/generate_upload_data.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2024 bilive.
+
+import os
+import time
+import codecs
+from datetime import datetime
+from src.upload.extract_video_info import generate_title, generate_desc, generate_tag, generate_source
+import subprocess
+import json
+
+def generate_video_data(video_path):
+    copyright = 1
+    title = generate_title(video_path)
+    desc = generate_desc(video_path)
+    tid = 138
+    tag = generate_tag(video_path)
+    source = generate_source(video_path)
+    cover = ""
+    dynamic = ""
+    return copyright, title, desc, tid, tag, source, cover, dynamic
+
+def generate_slice_data(video_path):
+    command = [
+        "ffprobe",
+        "-v", "quiet",
+        "-print_format", "json",
+        "-show_format",
+        video_path
+    ]
+    output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8')
+    parsed_output = json.loads(output)
+    title = parsed_output["format"]["tags"]["generate"]
+    copyright = 1
+    tid = 138
+    tag = "直播切片"
+    return copyright, title, tid, tag
+
+if __name__ == "__main__":    
+    pass

src/upload/generate_yaml.py

@@ -1,70 +0,0 @@
-# Copyright (c) 2024 bilive.
-
-import os
-import time
-import yaml
-import codecs
-from datetime import datetime
-from src.upload.extract_video_info import generate_title, generate_desc, generate_tag, generate_source
-import subprocess
-import json
-
-def generate_yaml_template(video_path):
-    source = generate_source(video_path)
-    title = generate_title(video_path)
-    desc = generate_desc(video_path)
-    tag = generate_tag(video_path)
-    data = {
-        "line": "bda2",
-        "limit": 5,
-        "streamers": {
-            video_path: {
-                "copyright": 1,
-                "source": source,
-                "tid": 138,
-                "cover": "",
-                "title": title,
-                "desc_format_id": 0,
-                "desc": desc,
-                "dynamic": "",
-                "tag": tag
-            }
-        }
-    }
-    return yaml.dump(data, default_flow_style=False, sort_keys=False)
-
-def generate_slice_yaml_template(video_path):
-    command = [
-        "ffprobe",
-        "-v", "quiet",
-        "-print_format", "json",
-        "-show_format",
-        video_path
-    ]
-    output = subprocess.check_output(command, stderr=subprocess.STDOUT).decode('utf-8')
-    parsed_output = json.loads(output)
-    title = parsed_output["format"]["tags"]["generate"]
-    data = {
-        "line": "bda2",
-        "limit": 5,
-        "streamers": {
-            video_path: {
-                "copyright": 1,
-                "source": "",
-                "tid": 138,
-                "cover": "",
-                "title": title,
-                "desc_format_id": 0,
-                "desc": "",
-                "dynamic": "",
-                "tag": "直播切片"
-            }
-        }
-    }
-    return yaml.dump(data, default_flow_style=False, sort_keys=False)
-
-if __name__ == "__main__":    
-    # read the queue and upload the video
-    yaml_template = generate_yaml_template("")
-    with open('upload.yaml', 'w', encoding='utf-8') as file:
-        file.write(yaml_template)

src/upload/upload.py

@@ -5,7 +5,7 @@ import os
 import sys
 from src.config import SRC_DIR, BILIVE_DIR
 from datetime import datetime
-from src.upload.generate_yaml import generate_yaml_template, generate_slice_yaml_template
+from src.upload.generate_upload_data import generate_video_data, generate_slice_data
 from src.upload.extract_video_info import generate_title
 from src.log.logger import upload_log
 import time
@@ -13,31 +13,21 @@ import fcntl
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from db.conn import get_single_upload_queue, delete_upload_queue, update_upload_queue_lock
 import threading
+from .bilitool.bilitool import UploadController, FeedController, LoginController
 
-read_lock = threading.Lock()
+# read_lock = threading.Lock()
 
-def upload_video(upload_path, yaml_file_path):
+def upload_video(upload_path):
     try:
-        # Construct the command
-        command = [
-            f"{SRC_DIR}/utils/biliup",
-            "-u",
-            f"{SRC_DIR}/utils/cookies.json",
-            "upload",
-            upload_path,
-            "--config",
-            yaml_file_path
-        ]
-        
-        # Execute the command
-        result = subprocess.run(command, check=True, capture_output=True, text=True)
-        upload_log.debug(f"Upload output:\nstdout: {result.stdout}\nstderr: {result.stderr}")
-        
-        # Check if the command was successful
-        if result.returncode == 0:
+        if upload_path.endswith('.flv'):
+            copyright, title, tid, tag = generate_slice_data(upload_path)
+        else:
+            copyright, title, desc, tid, tag, source, cover, dynamic = generate_video_data(upload_path)
+            yaml = ""
+        result = UploadController().upload_video_entry(upload_path, yaml, copyright, tid, title, desc, tag, source, cover, dynamic)
+        if result == True:
             upload_log.info("Upload successfully, then delete the video")
             os.remove(upload_path)
-            os.remove(yaml_file_path)
             delete_upload_queue(upload_path)
         else:
             upload_log.error("Fail to upload, the files will be reserved.")
@@ -60,22 +50,9 @@ def find_bv_number(target_str, my_list):
 
 def append_upload(upload_path, bv_result):
     try:
-        # Construct the command
-        command = [
-            f"{SRC_DIR}/utils/biliup",
-            "-u",
-            f"{SRC_DIR}/utils/cookies.json",
-            "append",
-            "--vid",
-            bv_result,
-            upload_path
-        ]
-        # Execute the command
-        result = subprocess.run(command, check=True, capture_output=True, text=True)
-        upload_log.debug(f"Append output:\nstdout: {result.stdout}\nstderr: {result.stderr}")
-
+        result = UploadController().append_video_entry(upload_path, bv_result)
         # Check if the command was successful
-        if result.returncode == 0:
+        if result == True:
             upload_log.info("Upload successfully, then delete the video")
             os.remove(upload_path)
             delete_upload_queue(upload_path)
@@ -92,51 +69,54 @@ def append_upload(upload_path, bv_result):
     
 def read_append_and_delete_lines():
     while True:
-        upload_queue = None
+        # upload_queue = None
         # read the queue and update lock status to prevent other threads from reading the same data
-        with read_lock:
-            upload_queue = get_single_upload_queue()
-            # if there is a task in the queue, try to lock the task
-            if upload_queue:
-                video_path, config_path = upload_queue.values()
-                # lock the task by updating the locked status to 1
-                update_result = update_upload_queue_lock(video_path, 1)
-                # if failed to lock, log the error and let the next thread to handle the task
-                if not update_result:
-                    upload_log.error(f"Failed to lock task for {video_path}, possibly already locked by another thread or database error.")
-                    upload_queue = None
-                    continue
-            else:
-                upload_log.info("Empty queue, wait 2 minutes and check again.")
-                time.sleep(120)
-                continue
-          
+        if LoginController().check_bilibili_login():
+            pass
+        else:
+            LoginController().login_bilibili(export=False)
+            continue
+        # with read_lock:
+        upload_queue = get_single_upload_queue()
+        # if there is a task in the queue, try to lock the task
         if upload_queue:  
-            video_path, config_path = upload_queue.values()
+            video_path = upload_queue['video_path']
+            time.sleep(3)
+            query = generate_title(video_path)
+            if query is None:
+                if not os.path.exists(video_path):
+                    delete_upload_queue(video_path)
+                    continue
+                else:
+                    upload_log.error(f"Error occurred in ffprobe: {video_path}")
+                    update_upload_queue_lock(video_path, 1)
+                    continue
             upload_log.info(f"deal with {video_path}")
             # check if the live is already uploaded
             if video_path.endswith('.flv'):
                 # upload slice video
-                upload_video(video_path, config_path)
-                return
+                upload_video(video_path)
             else:
-                query = generate_title(video_path)
-                result = subprocess.check_output("bilitool" + " list", shell=True)
-                # print(result.decode("utf-8"), flush=True)
-                upload_list = result.decode("utf-8").splitlines()
-                bv_result = find_bv_number(query, upload_list)
+                upload_dict = FeedController().get_video_dict_info(20, "pubed,not_pubed,is_pubing")
+                # result = subprocess.check_output("bilitool" + " list", shell=True)
+                # # print(result.decode("utf-8"), flush=True)
+                # upload_list = result.decode("utf-8").splitlines()
+                bv_result = upload_dict.get(query)
                 if bv_result:
                     upload_log.info(f"The series of videos has already been uploaded, the BV number is: {bv_result}")
                     append_upload(video_path, bv_result)
                 else:
                     upload_log.info("First upload this live")
-                    upload_video(video_path, config_path)
-                    return
-        time.sleep(20)
+                    upload_video(video_path)
+            time.sleep(20)
+        else:
+            upload_log.info("Empty queue, wait 2 minutes and check again.")
+            time.sleep(120)
         
 
 if __name__ == "__main__":    
-    max_workers = os.getenv("MAX_WORKERS", 5)
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        future_to_upload = {executor.submit(read_append_and_delete_lines) for _ in range(max_workers)}
+    # max_workers = os.getenv("MAX_WORKERS", 1)
+    # with ThreadPoolExecutor(max_workers=max_workers) as executor:
+    #     future_to_upload = {executor.submit(read_append_and_delete_lines) for _ in range(max_workers)}
+    read_append_and_delete_lines()