Enhance economy menu with custom math functions (#2559)

* Initial test commit

* Trying to improve

* Handle a duplicate column

* Temp solution for mismatched timeframe indices

* Cleanup + website

* tests

* Fixed black

* Update error handling + fix concat bug

* spelling error

* pred was still haning around

* unit tests

* blackformat

Co-authored-by: Jeroen Bouma <jer.bouma@gmail.com>
Co-authored-by: Theodore Aptekarev <aptekarev@gmail.com>
Co-authored-by: Colin Delahunty <72827203+colin99d@users.noreply.github.com>
Co-authored-by: colin99d <colin99delahunty@gmail.com>

openbb_terminal/economy/economy_controller.py

@@ -54,6 +54,7 @@ class EconomyController(BaseController):
     """Economy Controller class"""
 
     CHOICES_COMMANDS = [
+        "eval",
         "overview",
         "futures",
         "macro",
@@ -404,6 +405,7 @@ class EconomyController(BaseController):
         mt.add_raw("\n")
         mt.add_param("_stored", self.stored_datasets)
         mt.add_raw("\n")
+        mt.add_cmd("eval")
         mt.add_cmd("plot")
         mt.add_raw("\n")
         mt.add_menu("qa")
@@ -1708,6 +1710,37 @@ class EconomyController(BaseController):
             # # after saving it and displaying it to the user
             os.remove(self.d_GROUPS[ns_group] + ".jpg")
 
+    @log_start_end(log=logger)
+    def call_eval(self, other_args):
+        parser = argparse.ArgumentParser(
+            add_help=False,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            prog="eval",
+            description="""Create custom data column from loaded datasets.""",
+        )
+        parser.add_argument(
+            "-q",
+            "--query",
+            type=str,
+            nargs="+",
+            dest="query",
+        )
+        if other_args and "-" not in other_args[0][0]:
+            other_args.insert(0, "-q")
+
+        ns_parser = self.parse_known_args_and_warn(
+            parser, other_args, export_allowed=EXPORT_ONLY_RAW_DATA_ALLOWED
+        )
+        if ns_parser:
+            self.DATASETS = economy_helpers.create_new_entry(
+                self.DATASETS, " ".join(ns_parser.query)
+            )
+
+            self.stored_datasets = economy_helpers.update_stored_datasets_string(
+                self.DATASETS
+            )
+        console.print()
+
     @log_start_end(log=logger)
     def call_qa(self, _):
         """Process qa command"""

openbb_terminal/economy/economy_helpers.py

@@ -1,6 +1,80 @@
 """Economy helpers"""
 __docformat__ = "numpy"
 
+from typing import Dict
+
+import pandas as pd
+
+from openbb_terminal.rich_config import console
+
+
+def create_new_entry(dataset: Dict[str, pd.DataFrame], query: str) -> Dict:
+    """Create a new series based off previously loaded columns
+
+    Parameters
+    ----------
+    dataset: Dict[str,pd.DataFrame]
+        Economy datasets that are loaded
+    query: str
+        Query to execute
+
+    Returns
+    -------
+    Dict[str, pd.DataFrame]
+    """
+    # Create a single dataframe from dictionary of dataframes
+    columns = []
+    data = pd.DataFrame()
+    for _, df in dataset.items():
+        if not df.empty:
+            columns.extend(df.columns)
+            data = pd.concat([data, df], axis=1)
+    # In order to account for potentially different index time steps, lets dropNans here.
+    # Potentially problematic down the road
+    data = data.dropna(axis=0)
+
+    # Eval the query to generate new sequence
+    # if there is an = in the query, then there will be a new named column
+    if "=" in query:
+        new_column = query.split("=")[0].replace(" ", "")
+        if new_column in data.columns:
+            query = query.replace(new_column, new_column + "_duplicate")
+            new_column += "_duplicate"
+        # Wrap the eval in a syntax error in case the user does something not allowed
+        try:
+            new_df = data.eval(query)
+        except SyntaxError:
+            console.print(
+                "[red]Invalid syntax in query.  Please enter something of the form `newcol=col1 + col2`[/red]\n"
+            )
+            return dataset
+        except pd.errors.UndefinedVariableError as e:
+            console.print(f"[red]{e}[/red]")
+            return dataset
+
+        # If custom exists in the dictionary, we need to append the current dataframe
+        if "custom" in dataset:
+            dataset["custom"] = pd.concat([dataset["custom"], new_df[[new_column]]])
+        else:
+            dataset["custom"] = new_df[[new_column]]
+        return dataset
+
+    # If there is not an equal (namely  .eval(colA + colB), the result will be a series
+    # and not a dataframe.  We can just call this custom_exp
+
+    try:
+        data = pd.DataFrame(data.eval(query), columns=["custom_exp"])
+        dataset["custom"] = data
+    except SyntaxError:
+        console.print(
+            "Invalid syntax in query.  Please enter something of the form `newcol=col1 + col2`"
+        )
+        return dataset
+    except pd.errors.UndefinedVariableError as e:
+        console.print(f"[red]{e}[/red]")
+        return dataset
+    return dataset
+
 
 def update_stored_datasets_string(datasets) -> str:
     stored_datasets_string = ""

openbb_terminal/miscellaneous/i18n/en.yml

@@ -762,6 +762,7 @@ en:
   economy/valuation: valuation of sectors, industry, country
   economy/performance: performance of sectors, industry, country
   economy/spectrum: spectrum of sectors, industry, country
+  economy/eval: create new series by performing operations on loaded data
   economy/forecast: forecast techniques; rnn, nbeats, transformer, block rnn
   economy/qa: Open quantitative analysis menu with stored data
   economy/qa/pick: pick new series from stored economy data

tests/openbb_terminal/economy/test_economy_helpers.py

@@ -0,0 +1,35 @@
+import pandas as pd
+import pytest
+from openbb_terminal.economy import economy_helpers
+
+
+def test_create_new_entry():
+    df_dict = {
+        "key1": pd.DataFrame([0, 1, 2, 3, 4], columns=["A"]),
+        "key2": pd.DataFrame([10, 20, 30, 40, 50], columns=["Z"]),
+    }
+    new_dict = economy_helpers.create_new_entry(df_dict, "New=A**2 + Z")
+    assert "custom" in new_dict
+    assert new_dict["custom"].equals(
+        pd.DataFrame([10, 21, 34, 49, 66], columns=["New"])
+    )
+
+
+@pytest.mark.vcr
+@pytest.mark.record_stdout
+def test_bad_create_new_entry():
+    df_dict = {
+        "key1": pd.DataFrame([0, 1, 2, 3, 4], columns=["A"]),
+        "key2": pd.DataFrame([10, 20, 30, 40, 50], columns=["Z"]),
+    }
+    economy_helpers.create_new_entry(df_dict, "New=A**2 + PSKJ  + hfjijfiefj")
+
+
+@pytest.mark.vcr
+@pytest.mark.record_stdout
+def test_bad_create_new_entry_bad_query():
+    df_dict = {
+        "key1": pd.DataFrame([0, 1, 2, 3, 4], columns=["A"]),
+        "key2": pd.DataFrame([10, 20, 30, 40, 50], columns=["Z"]),
+    }
+    economy_helpers.create_new_entry(df_dict, "New=A +")

tests/openbb_terminal/economy/txt/test_economy_controller/test_print_help.txt

@@ -20,6 +20,7 @@ Databases:
 
 Stored datasets: 
 
+    eval               create new series by performing operations on loaded data
     plot               plot data from the above commands together
 
 >   qa                 Open quantitative analysis menu with stored data

tests/openbb_terminal/economy/txt/test_economy_helpers/test_bad_create_new_entry.txt

@@ -0,0 +1 @@
+[red]name 'PSKJ' is not defined[/red]

tests/openbb_terminal/economy/txt/test_economy_helpers/test_bad_create_new_entry_bad_query.txt

@@ -0,0 +1,2 @@
+[red]Invalid syntax in query.  Please enter something of the form `newcol=col1 + col2`[/red]
+

website/content/terminal/economy/eval/_index.md

@@ -0,0 +1,13 @@
+```
+usage: eval [-q QUERY [QUERY ...]] [-h] [--export EXPORT]
+```
+Create custom data column from loaded datasets.
+
+Note that for division, the / operator will cause issues, so one should do `*N**-1` where N is the number you are dividing by 
+```
+options:
+  -q QUERY [QUERY ...], --query QUERY [QUERY ...]
+  -h, --help            show this help message (default: False)
+  --export EXPORT       Export raw data into csv, json, xlsx (default: )
+```
+For more information and examples, use 'about eval' to access the related guide.

website/data/menu/main.yml

@@ -590,7 +590,11 @@ main:
           - name: overview
             ref: /terminal/economy/overview
           - name: performance
-            ref: /terminal/economy/performance
+            ref: "/terminal/economy/performance"
+          - name: spectrum
+            ref: "/terminal/economy/spectrum"
+          - name: eval
+            ref: "/terminal/economy/eval"
           - name: plot
             ref: /terminal/economy/plot
           - name: quantitative analysis