lywsvip/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-06-05 02:47:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Feature] - Merge reference.json and extension_map.json, plus documentation generator fixes (#6268)

Browse Source 
* Merge reference.json and extension_map.json

* feat: check all packages are built and installed

* rebuild

* fix

* small fix

* Update package_builder.py

* move core into info

* move extensions into info

* fix test

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
This commit is contained in:
montezdesousa
2024-04-04 10:56:37 +01:00
committed by GitHub
parent 57deb8eeab
commit daa4520fea
12 changed files with 26862 additions and 26895 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
openbb_platform/core/openbb_core/app/model/system_settings.py
View File

@@ -15,7 +15,7 @@ from openbb_core.app.constants import (
)
from openbb_core.app.model.abstract.tagged import Tagged
from openbb_core.app.model.fast_api_settings import FastAPISettings
from openbb_core.app.version import VERSION
from openbb_core.app.version import CORE_VERSION, VERSION
class SystemSettings(Tagged):
@@ -28,6 +28,7 @@ class SystemSettings(Tagged):
# OpenBB section
version: str = VERSION
core: str = CORE_VERSION
home_directory: str = str(HOME_DIRECTORY)
openbb_directory: str = str(OPENBB_DIRECTORY)
user_settings_path: str = str(USER_SETTINGS_PATH)

9
openbb_platform/core/openbb_core/app/static/app_factory.py
View File

@@ -1,6 +1,6 @@
"""App factory."""
from typing import Optional, Type, TypeVar
from typing import Dict, Optional, Type, TypeVar
from openbb_core.app.command_runner import CommandRunner
from openbb_core.app.model.system_settings import SystemSettings
@@ -8,6 +8,7 @@ from openbb_core.app.model.user_settings import UserSettings
from openbb_core.app.static.account import Account
from openbb_core.app.static.container import Container
from openbb_core.app.static.coverage import Coverage
from openbb_core.app.static.reference_loader import ReferenceLoader
from openbb_core.app.version import VERSION
E = TypeVar("E", bound=Type[Container])
@@ -29,6 +30,7 @@ class BaseApp:
self._command_runner = command_runner
self._account = Account(self)
self._coverage = Coverage(self)
self._reference = ReferenceLoader().reference
@property
def account(self) -> Account:
@@ -50,6 +52,11 @@ class BaseApp:
"""Coverage menu."""
return self._coverage
@property
def reference(self) -> Dict[str, Dict]:
"""Return reference data."""
return self._reference
def create_app(extensions: Optional[E] = None) -> Type[BaseApp]:
"""Create the app."""

7
openbb_platform/core/openbb_core/app/static/coverage.py
View File

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
from openbb_core.api.router.helpers.coverage_helpers import get_route_schema_map
from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.router import CommandMap
from openbb_core.app.static.reference_loader import ReferenceLoader
if TYPE_CHECKING:
from openbb_core.app.static.app_factory import BaseApp
@@ -28,7 +27,6 @@ class Coverage:
self._app = app
self._command_map = CommandMap(coverage_sep=".")
self._provider_interface = ProviderInterface()
self._reference_loader = ReferenceLoader()
def __repr__(self) -> str:
"""Return docstring."""
@@ -52,11 +50,6 @@ class Coverage:
for command, value in self._command_map.commands_model.items()
}
@property
def reference(self) -> Dict[str, Dict]:
"""Return reference data."""
return self._reference_loader.reference
def command_schemas(self, filter_by_provider: Optional[str] = None):
"""Return route schema for a command."""
return get_route_schema_map(

55
openbb_platform/core/openbb_core/app/static/package_builder.py
View File

@@ -45,6 +45,7 @@ from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.router import RouterLoader
from openbb_core.app.static.utils.console import Console
from openbb_core.app.static.utils.linters import Linters
from openbb_core.app.version import CORE_VERSION, VERSION
from openbb_core.env import Env
from openbb_core.provider.abstract.data import Data
@@ -90,9 +91,11 @@ class PackageBuilder:
def auto_build(self) -> None:
"""Trigger build if there are differences between built and installed extensions."""
if Env().AUTO_BUILD:
add, remove = PackageBuilder._diff(
self.directory / "assets" / "extension_map.json"
reference = PackageBuilder._read(
self.directory / "assets" / "reference.json"
)
ext_map = reference.get("info", {}).get("extensions", {})
add, remove = PackageBuilder._diff(ext_map)
if add:
a = ", ".join(sorted(add))
print(f"Extensions to add: {a}") # noqa: T201
@@ -113,10 +116,9 @@ class PackageBuilder:
self.console.log("\nBuilding extensions package...\n")
self._clean(modules)
ext_map = self._get_extension_map()
self._save_extension_map(ext_map)
self._save_modules(modules, ext_map)
self._save_package()
self._save_reference_file()
self._save_reference_file(ext_map)
if self.lint:
self._run_linters()
@@ -143,12 +145,6 @@ class PackageBuilder:
]
return ext_map
def _save_extension_map(self, ext_map: Dict[str, List[str]]) -> None:
"""Save the map of extensions available at build time."""
code = dumps(obj=dict(sorted(ext_map.items())), indent=4)
self.console.log("Writing extension map...")
self._write(code=code, name="extension_map", extension="json", folder="assets")
def _save_modules(
self,
modules: Optional[Union[str, List[str]]] = None,
@@ -185,11 +181,23 @@ class PackageBuilder:
code = "### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ###\n"
self._write(code=code, name="__init__")
def _save_reference_file(self):
def _save_reference_file(self, ext_map: Optional[Dict[str, List[str]]] = None):
"""Save the reference.json file."""
self.console.log("\nWriting reference file...")
data = ReferenceGenerator.get_reference_data()
code = dumps(obj=data, indent=4)
code = dumps(
obj={
"openbb": VERSION.replace("dev", ""),
"info": {
"title": "OpenBB Platform (Python)",
"description": "This is the OpenBB Platform (Python).",
"core": CORE_VERSION.replace("dev", ""),
"extensions": ext_map,
},
"paths": data,
},
indent=4,
)
self._write(code=code, name="reference", extension="json", folder="assets")
def _run_linters(self):
@@ -224,13 +232,28 @@ class PackageBuilder:
return content
@staticmethod
def _diff(path: Path) -> Tuple[Set[str], Set[str]]:
def _diff(ext_map: Dict[str, List[str]]) -> Tuple[Set[str], Set[str]]:
"""Check differences between built and installed extensions.
Parameters
----------
path: Path
The path to the folder where the extension map is stored.
ext_map: Dict[str, List[str]]
Dictionary containing the extensions.
Example:
{
"openbb_core_extension": [
"commodity@1.0.1",
...
],
"openbb_provider_extension": [
"benzinga@1.1.3",
...
],
"openbb_obbject_extension": [
"openbb_charting@1.0.0",
...
]
}
Returns
-------
@@ -238,8 +261,6 @@ class PackageBuilder:
First element: set of installed extensions that are not in the package.
Second element: set of extensions in the package that are not installed.
"""
ext_map = PackageBuilder._read(path)
add: Set[str] = set()
remove: Set[str] = set()
groups = OpenBBGroups.groups()

5
openbb_platform/core/openbb_core/app/version.py
View File

@@ -58,3 +58,8 @@ try:
VERSION = get_package_version(PACKAGE)
except pkg_resources.DistributionNotFound:
VERSION = "unknown"
try:
CORE_VERSION = get_package_version("openbb-core")
except pkg_resources.DistributionNotFound:
CORE_VERSION = "unknown"

7
openbb_platform/core/tests/app/static/test_app_factory.py
View File

@@ -47,3 +47,10 @@ def test_app_coverage(app_factory):
coverage = app_factory.coverage
assert coverage
assert isinstance(coverage, Coverage)
def test_app_reference(app_factory):
"""Test app reference."""
reference = app_factory.reference
assert reference
assert isinstance(reference, dict)

7
openbb_platform/core/tests/app/static/test_coverage.py
View File

@@ -37,10 +37,3 @@ def test_coverage_commands(coverage):
command_coverage = coverage.commands
assert command_coverage
assert isinstance(command_coverage, dict)
def test_coverage_reference(coverage):
"""Test coverage reference."""
reference = coverage.reference
assert reference
assert isinstance(reference, dict)

25
openbb_platform/core/tests/app/static/test_package_builder.py
View File

@@ -582,15 +582,15 @@ def test_generate(docstring_generator):
assert "Returns" in doc
def test_read_extension_map(package_builder, tmp_openbb_dir):
"""Test read extension map."""
def test__read(package_builder, tmp_openbb_dir):
"""Test read."""
PATH = "openbb_core.app.static.package_builder."
open_mock = mock_open()
with patch(PATH + "open", open_mock), patch(PATH + "load") as mock_load:
package_builder._read(Path(tmp_openbb_dir / "assets" / "extension_map.json"))
package_builder._read(Path(tmp_openbb_dir / "assets" / "reference.json"))
open_mock.assert_called_once_with(
Path(tmp_openbb_dir / "assets" / "extension_map.json")
Path(tmp_openbb_dir / "assets" / "reference.json")
)
mock_load.assert_called_once()
@@ -646,7 +646,6 @@ def test_read_extension_map(package_builder, tmp_openbb_dir):
)
def test_package_diff(
package_builder,
tmp_openbb_dir,
ext_built,
ext_installed,
ext_inst_version,
@@ -659,19 +658,16 @@ def test_package_diff(
return ext_installed.select(**{"group": group})
PATH = "openbb_core.app.static.package_builder."
with patch.object(PackageBuilder, "_read") as mock_read, patch(
PATH + "entry_points", mock_entry_points
), patch.object(EntryPoint, "dist", new_callable=PropertyMock) as mock_obj:
with patch(PATH + "entry_points", mock_entry_points), patch.object(
EntryPoint, "dist", new_callable=PropertyMock
) as mock_obj:
class MockPathDistribution:
version = ext_inst_version
mock_obj.return_value = MockPathDistribution()
mock_read.return_value = ext_built
add, remove = package_builder._diff(
Path(tmp_openbb_dir, "assets", "extension_map.json")
)
add, remove = package_builder._diff(ext_built)
# We add whatever is not built, but is installed
assert add == expected_add
@@ -689,7 +685,7 @@ def test_package_diff(
({"this"}, {"that"}, False),
],
)
def test_auto_build(package_builder, tmp_openbb_dir, add, remove, openbb_auto_build):
def test_auto_build(package_builder, add, remove, openbb_auto_build):
"""Test auto build."""
with patch.object(PackageBuilder, "_diff") as mock_assets_diff, patch.object(
@@ -700,9 +696,6 @@ def test_auto_build(package_builder, tmp_openbb_dir, add, remove, openbb_auto_bu
package_builder.auto_build()
if openbb_auto_build:
mock_assets_diff.assert_called_once_with(
Path(tmp_openbb_dir, "assets", "extension_map.json")
)
if add or remove:
mock_build.assert_called_once()
else:

29
openbb_platform/openbb/assets/extension_map.json vendored
View File

@@ -1,29 +0,0 @@
{
"openbb_core_extension": [
"commodity@1.0.2",
"crypto@1.1.4",
"currency@1.1.4",
"derivatives@1.1.4",
"economy@1.1.4",
"equity@1.1.4",
"etf@1.1.4",
"fixedincome@1.1.4",
"index@1.1.4",
"news@1.1.4",
"regulators@1.1.4"
],
"openbb_obbject_extension": [],
"openbb_provider_extension": [
"benzinga@1.1.4",
"federal_reserve@1.1.4",
"fmp@1.1.4",
"fred@1.1.4",
"intrinio@1.1.4",
"oecd@1.1.4",
"polygon@1.1.4",
"sec@1.1.4",
"tiingo@1.1.4",
"tradingeconomics@1.1.4",
"yfinance@1.1.4"
]
}

53197
openbb_platform/openbb/assets/reference.json vendored
View File

File diff suppressed because it is too large Load Diff

14
openbb_platform/tests/test_extension_map.py
View File

@@ -6,12 +6,10 @@ from poetry.core.constraints.version import Version, VersionConstraint, parse_co
from poetry.core.pyproject.toml import PyProjectTOML
def load_ext_map(file: Path) -> Dict[str, Version]:
"""Load the extension map from extension_map.json."""
def create_ext_map(extensions: dict) -> Dict[str, Version]:
"""Create the extension map from extension."""
ext_map = {}
with open(file) as f:
ext_map_json = json.load(f)
for _, v in ext_map_json.items():
for _, v in extensions.items():
for value in v:
name, version = value.split("@")
ext_map[name] = Version.parse(version)
@@ -36,9 +34,9 @@ def load_req_ext(file: Path) -> Dict[str, VersionConstraint]:
def test_extension_map():
"""Ensure only required extensions are built and versions respect pyproject.toml"""
this_dir = Path(__file__).parent
ext_map = load_ext_map(
Path(this_dir, "..", "openbb", "assets", "extension_map.json")
)
with open(Path(this_dir, "..", "openbb", "assets", "reference.json")) as f:
reference = json.load(f)
ext_map = create_ext_map(reference.get("info", {}).get("extensions", {}))
req_ext = load_req_ext(Path(this_dir, "..", "pyproject.toml"))
for ext in req_ext:

399
website/generate_platform_v4_markdown.py
View File

@@ -1,17 +1,15 @@
"""Platform V4 Markdown Generator Script."""
import argparse
import json
import re
import shutil
import subprocess
from pathlib import Path
from typing import Dict, List, Union
from typing import Dict, List
import toml
from openbb_core.app.static.utils.console import Console
from openbb_core.provider import standard_models
from packaging import specifiers
from poetry.core.constraints.version import Version, VersionConstraint, parse_constraint
from poetry.core.pyproject.toml import PyProjectTOML
# Number of spaces to substitute tabs for indentation
TAB_WIDTH = 4
@@ -19,157 +17,38 @@ TAB_WIDTH = 4
# Maximum number of commands to display on the cards
MAX_COMMANDS = 8
# Path to the Platform directory and the reference.json file
# Input paths
PLATFORM_PATH = Path(__file__).parent.parent / "openbb_platform"
PLATFORM_PYPROJECT_PATH = Path(PLATFORM_PATH / "pyproject.toml")
REFERENCE_FILE_PATH = Path(PLATFORM_PATH / "openbb/assets/reference.json")
# Paths to use for generating and storing the markdown files
# Output paths
WEBSITE_PATH = Path(__file__).parent.absolute()
SEO_METADATA_PATH = Path(WEBSITE_PATH / "metadata/platform_v4_seo_metadata.json")
PLATFORM_CONTENT_PATH = Path(WEBSITE_PATH / "content/platform")
PLATFORM_REFERENCE_PATH = Path(WEBSITE_PATH / "content/platform/reference")
PLATFORM_DATA_MODELS_PATH = Path(WEBSITE_PATH / "content/platform/data_models")
# Imports used in the generated markdown files
# Markdown imports and elements
PLATFORM_REFERENCE_IMPORT = "import ReferenceCard from '@site/src/components/General/NewReferenceCard';" # fmt: skip
PLATFORM_REFERENCE_UL_ELEMENT = '<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">' # noqa: E501
# pylint: disable=redefined-outer-name
def check_installed_packages(
console: Console,
debug: bool,
) -> None:
"""Checks if the installed packages are the same as those on the platform pyproject.toml file.
Compares the versions of the installed packages with the versions specified in the pyproject.toml file.
The source of truth for the package versions is the pyproject.toml file, and the installed packages are
checked against the specified versions. If the installed packages do not satisfy the version requirements,
an error is raised.
Parameters
----------
console (Console): Console object to display messages and save logs
debug (bool): Flag to enable debug mode
"""
class Console:
"""Console class to log messages to the console."""
def convert_poetry_version_specifier(
poetry_version: Union[str, Dict[str, str]]
) -> str:
"""
Convert a Poetry version specifier to a format compatible with the packaging library.
Handles both simple string specifiers and dictionary specifiers, extracting only the version value if it's a dict.
def __init__(self, verbose: bool = False):
self.verbose = verbose
Parameters
----------
poetry_version (Union[str, Dict[str, str]]):
Poetry version specifier
def log(self, message: str) -> None:
if self.verbose:
print(message)
Returns
-------
str:
Version specifier compatible with the packaging library
"""
if isinstance(poetry_version, dict):
poetry_version = poetry_version.get("version", "")
if isinstance(poetry_version, str):
if poetry_version.startswith("^"):
base_version = poetry_version[1:]
# Use regex to split the version and convert to integers only if they are purely numeric
parts = re.split(r"\.|\-", base_version)
try:
major, minor = (int(x) for x in parts[:2])
except ValueError:
# If conversion fails, return the original version specifier
return poetry_version
next_major_version = major + 1
# Construct a version specifier that represents the range.
return f">={base_version},<{next_major_version}.0.0"
if poetry_version.startswith("~"):
base_version = poetry_version[1:]
parts = re.split(r"\.|\-", base_version)
try:
major, minor = (int(x) for x in parts[:2])
except ValueError:
# If conversion fails, return the original version specifier
return poetry_version
next_minor_version = minor + 1
# Construct a version specifier that represents the range.
return f">={base_version},<{major}.{next_minor_version}.0"
# No need to modify other specifiers, as they are compatible with packaging library
return poetry_version
def check_dependency(
package_name: str, version_spec: str, installed_packages_dict: Dict[str, str]
) -> None:
"""
Check if the installed package version satisfies the required version specifier.
Raises DependencyCheckError if the package is not installed or does not satisfy the version requirements.
Parameters
----------
package_name (str):
Name of the package to check
version_spec (str):
Version specifier to check against
installed_packages_dict (Dict[str, str]):
Dictionary of installed packages and their versions
"""
installed_version = installed_packages_dict.get(package_name.lower())
if not installed_version:
raise Exception(f"{package_name} is not installed.")
converted_version_spec = convert_poetry_version_specifier(version_spec)
specifier_set = specifiers.SpecifierSet(converted_version_spec)
if not specifier_set.contains(installed_version, prereleases=True):
message = f"{package_name} version {installed_version} does not satisfy the specified version {converted_version_spec}." # noqa: E501, pylint: disable=line-too-long
raise Exception(message)
console.log("\n[CRITICAL] Ensuring all the extensions are installed before the script runs...") # fmt: skip
# Execute the pip list command once and store the output
pip_list_output = subprocess.run(
"pip list | grep openbb", # noqa: S607
shell=True, # noqa: S602
capture_output=True,
text=True,
check=False,
)
installed_packages = pip_list_output.stdout.splitlines()
installed_packages_dict = {
line.split()[0].lower(): line.split()[1] for line in installed_packages
}
# Load the pyproject.toml file once
with open(PLATFORM_PATH / "pyproject.toml") as f:
toml_dict = toml.load(f)
# Extract the openbb dependencies, excluding the python dependency
dependencies = toml_dict["tool"]["poetry"]["dependencies"]
dependencies.pop("python", None)
# Compare versions and check dependencies
for package, version_spec in dependencies.items():
normalized_package_name = package.replace("_", "-").lower()
try:
# Convert the version specifier before checking
converted_version_spec = convert_poetry_version_specifier(version_spec)
check_dependency(
normalized_package_name, converted_version_spec, installed_packages_dict
)
# Ensure debug_mode output shows the processed version specifier
if debug:
installed_version = installed_packages_dict.get(normalized_package_name)
console.log(
f"{normalized_package_name}: Specified version {converted_version_spec}, Installed version {installed_version}" # noqa: E501, pylint: disable=line-too-long
)
except Exception as e:
raise e
console = Console(verbose=True)
def create_reference_markdown_seo(path: str, description: str) -> str:
@@ -177,15 +56,15 @@ def create_reference_markdown_seo(path: str, description: str) -> str:
Parameters
----------
path (str):
Command path relative to the obb class
description (str):
Description of the command
path: str
Command path relative to the obb class
description: str
Description of the command
Returns
-------
str:
SEO section for the markdown file
str
SEO section for the markdown file
"""
with open(SEO_METADATA_PATH) as f:
@@ -228,17 +107,17 @@ def create_reference_markdown_intro(
Parameters
----------
path (str):
Command path relative to the obb class
description (str):
Description of the command
deprecated (Dict[str, str]):
Deprecated flag and message
path: str
Command path relative to the obb class
description: str
Description of the command
deprecated: Dict[str, str]
Deprecated flag and message
Returns
-------
str:
Introduction section for the markdown file
str
Introduction section for the markdown file
"""
deprecation_message = (
@@ -268,15 +147,15 @@ def create_reference_markdown_tabular_section(
Parameters
----------
parameters (Dict[str, List[Dict[str, str]]]):
Dictionary of providers and their corresponding parameters
heading (str):
Section heading for the tabular section
parameters: Dict[str, List[Dict[str, str]]]
Dictionary of providers and their corresponding parameters
heading: str
Section heading for the tabular section
Returns
-------
str:
Tabular section for the markdown file
str
Tabular section for the markdown file
"""
standard_params_list = []
@@ -339,14 +218,15 @@ def create_reference_markdown_tabular_section(
def create_reference_markdown_returns_section(returns: List[Dict[str, str]]) -> str:
"""Create the returns section for the markdown file.
Args
----
returns (List[Dict[str, str]]):
List of dictionaries containing the name, type and description of the returns
Parameters
----------
returns: List[Dict[str, str]]
List of dictionaries containing the name, type and description of the returns
Returns
-------
str:
Returns section for the markdown file
str
Returns section for the markdown file
"""
returns_str = ""
@@ -371,17 +251,17 @@ def create_data_model_markdown(title: str, description: str, model: str) -> str:
Parameters
----------
title (str):
Title of the data model
description (str):
Description of the data model
model (str):
Model name
title: str
Title of the data model
description: str
Description of the data model
model: str
Model name
Returns
-------
str:
Basic markdown file content for the data model
str
Basic markdown file content for the data model
"""
# File name is used in the import statement
@@ -431,13 +311,13 @@ def find_data_model_implementation_file(data_model: str) -> str:
Parameters
----------
data_model (str):
Data model name
data_model: str
Data model name
Returns
-------
str:
File name containing the data model class
str
File name containing the data model class
"""
# Function to search for the data model class in the file
@@ -465,8 +345,8 @@ def generate_reference_index_files(reference_content: Dict[str, str]) -> None:
Parameters
----------
reference_content (Dict[str, str]):
Endpoints and their corresponding descriptions.
reference_content: Dict[str, str]
Endpoints and their corresponding descriptions.
"""
def generate_index_and_category(
@@ -599,17 +479,17 @@ def create_data_models_index(title: str, description: str, model: str) -> str:
Parameters
----------
title (str):
Title of the data model
description (str):
Description of the data model
model (str):
Model name
title: str
Title of the data model
description: str
Description of the data model
model: str
Model name
Returns
-------
str:
Index content for the data models
str
Index content for the data models
"""
# Get the first sentence of the description
@@ -632,8 +512,8 @@ def generate_data_models_index_files(content: str) -> None:
Parameters
----------
content (str):
Content for the data models index file
content: str
Content for the data models index file
"""
index_content = (
@@ -661,17 +541,17 @@ def generate_markdown_file(path: str, markdown_content: str, directory: str) ->
Parameters
----------
path (str):
Path to the markdown file
markdown_content (str):
Content for the markdown file
directory (str):
Directory to save the markdown file
path: str
Path to the markdown file
markdown_content: str
Content for the markdown file
directory: str
Directory to save the markdown file
Raises
------
ValueError:
If the content type is invalid
ValueError:
If the content type is invalid
"""
# For reference, split the path to separate the
@@ -697,24 +577,12 @@ def generate_markdown_file(path: str, markdown_content: str, directory: str) ->
# pylint: disable=redefined-outer-name
def generate_platform_markdown(
console: Console,
) -> None:
def generate_platform_markdown(paths: Dict) -> None:
"""Generate markdown files for OpenBB Docusaurus website."""
data_models_index_content = []
reference_index_content_dict = {}
console.log(f"\n[INFO] Reading the {REFERENCE_FILE_PATH} file...")
# Load the reference.json file
try:
with open(REFERENCE_FILE_PATH) as f:
reference = json.load(f)
except FileNotFoundError as exc:
raise FileNotFoundError(
"File not found! Please ensure the file exists."
) from exc
# Clear the platform/reference folder
console.log(f"\n[INFO] Clearing the {PLATFORM_REFERENCE_PATH} folder...")
shutil.rmtree(PLATFORM_REFERENCE_PATH, ignore_errors=True)
@@ -728,7 +596,7 @@ def generate_platform_markdown(
) # noqa: E501
console.log(f"\n[INFO] Generating the markdown files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
for path, path_data in reference.items():
for path, path_data in paths.items():
reference_markdown_content = ""
data_markdown_content = ""
@@ -808,28 +676,101 @@ def generate_platform_markdown(
console.log("\n[INFO] Markdown files generated successfully!")
def read_reference() -> dict:
"""Read the reference.json file."""
console.log(f"\n[INFO] Reading the {REFERENCE_FILE_PATH} file...")
# Load the reference.json file
try:
with open(REFERENCE_FILE_PATH) as f:
reference = json.load(f)
except FileNotFoundError as exc:
raise FileNotFoundError(
"File not found! Please ensure the file exists."
) from exc
return reference
def get_openbb_versions() -> Dict[str, VersionConstraint]:
"""Get the openbb package version constraints from pyproject.toml."""
pyproject = PyProjectTOML(PLATFORM_PYPROJECT_PATH)
deps = pyproject.data["tool"]["poetry"]["dependencies"]
dep_spec = {}
for p, v in deps.items():
if p.startswith("openbb"):
if isinstance(v, str):
dep_spec[p] = parse_constraint(v)
elif isinstance(v, dict):
dep_spec[p] = parse_constraint(v["version"])
return dep_spec
def check_installed(openbb_versions: Dict[str, VersionConstraint]) -> None:
"""Check all the openbb packages are installed and have the correct version."""
console.log("\n[INFO] Ensuring all packages installed...")
pip_list_output = subprocess.run(
"pip list | grep openbb", # noqa: S607
shell=True, # noqa: S602
capture_output=True,
text=True,
check=False,
)
result = pip_list_output.stdout.splitlines()
installed = {
line.split()[0].lower(): Version.parse(line.split()[1]) for line in result
}
failures = set()
for o, v in openbb_versions.items():
if o not in installed:
console.log(f"[INFO] Package '{o}' not installed.")
failures.add(o)
elif not v.allows(installed[o]):
console.log(
f"[INFO] Version '{installed[o]}' of '{o}' not compatible. Expected '{v}'."
)
failures.add(o)
if failures:
raise ValueError(f"Failures: {failures}")
def check_built(openbb_versions: Dict[str, VersionConstraint], reference: dict) -> None:
"""Check all the openbb packages installed are in the reference file."""
console.log("\n[INFO] Ensuring all packages built...")
core_version = reference.get("info", {}).get("core", "")
extensions = reference.get("info", {}).get("extensions", {})
built = {}
built["openbb-core"] = Version.parse(core_version)
for value in extensions.values():
for v in value:
name, version = v.split("@")
if name.startswith("openbb_"):
name = name[7:]
name = "openbb-" + name.replace("_", "-")
built[name] = Version.parse(version)
failures = set()
for o, v in openbb_versions.items():
if o not in built:
console.log(f"[INFO] Package '{o}' not in reference file.")
failures.add(o)
elif not v.allows(built[o]):
console.log(
f"[INFO] Version '{built[o]}' of '{o}' not compatible. Expected '{v}'."
)
failures.add(o)
if failures:
raise ValueError(f"Failures: {failures}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="Platform Markdown Generator V2",
description="Generate markdown files for the Platform website docs.",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Enable verbose output for debugging.",
)
openbb_versions = get_openbb_versions()
check_installed(openbb_versions)
args = parser.parse_args()
console = Console(True)
verbose = False
reference = read_reference()
check_built(openbb_versions, reference)
if args.verbose:
verbose = True
check_installed_packages(
console=console,
debug=verbose,
)
generate_platform_markdown(console=console)
generate_platform_markdown(reference.get("paths", {}))