diff --git a/README.md b/README.md index 37b3309..5807064 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ The Skypro reporting tool collates data to documents and analyse real-world perf This tool is run on the command line using `skypro report`. See `src/skypro/commands/report/README.md` for more information. +## Reporting web app - DEPRECATED +The Skypro reporting web app makes reporting results accessible to non-cli users. It is now deprecated. +This is run using Streamlit. +See `src/skypro/reporting_webapp/README.md` for more information. + ## Rates and energy flows Information about the costs and revenues associated with using power are fundamental to the codebase and a high-level understanding of how they are modelled is important for interpreting results. See `src/skypro/common/rates/README.md` for a background on how rates and energy flows are modelled in the codebase. diff --git a/docs/reporting_webapp_screenshot.png b/docs/reporting_webapp_screenshot.png new file mode 100644 index 0000000..0be3d58 Binary files /dev/null and b/docs/reporting_webapp_screenshot.png differ diff --git a/pyproject.toml b/pyproject.toml index f071687..c37c3fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ sqlalchemy = "^2.0.37" psycopg2-binary = "^2.9.10" requests = "^2.32.3" arrow = "^1.3.0" +streamlit = "^1.45.1" [[tool.poetry.source]] name = "testpypi" diff --git a/src/skypro/reporting_webapp/README.md b/src/skypro/reporting_webapp/README.md new file mode 100644 index 0000000..6b0acfb --- /dev/null +++ b/src/skypro/reporting_webapp/README.md @@ -0,0 +1,21 @@ +# DEPRECATED - Skypro reporting web app + +This is a Streamlit web app that makes the reporting functionality available over a web front end. + +The app supports different 'scenarios'- each of which have an associated reporting configuration file. +This is useful because it allows multiple microgrids to be reviewed from the same app. +It also means you can compare multiple Supplier arrangements, for example, you could have a scenario where P395 is active, and one where it's not. + +The app calls into the main reporting code (using the appropriate configuration file) and visualises the results. + +![screenshot](../../../docs/reporting_webapp_screenshot.png) + +## Usage + +The Streamlit app can be launched with: +``` +SKIP_PASSWORD=true CONFIG_FILE=src/skypro/reporting_webapp/example_config.yaml streamlit run src/skypro/reporting_webapp/main.py +``` +This should open a window in your default browser. +This example configuration uses the integration testing fixtures which only have data available for August 2024 - so that is the only month that works in this example! + \ No newline at end of file diff --git a/src/skypro/reporting_webapp/__init__.py b/src/skypro/reporting_webapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/skypro/reporting_webapp/example_config.yaml b/src/skypro/reporting_webapp/example_config.yaml new file mode 100644 index 0000000..a92b484 --- /dev/null +++ b/src/skypro/reporting_webapp/example_config.yaml @@ -0,0 +1,5 @@ +vars: {} + +reportScenarios: + "Some scenario": + config: "./src/tests/integration/fixtures/reporting/config.yaml" diff --git a/src/skypro/reporting_webapp/main.py b/src/skypro/reporting_webapp/main.py new file mode 100644 index 0000000..71683b3 --- /dev/null +++ b/src/skypro/reporting_webapp/main.py @@ -0,0 +1,357 @@ +import calendar +import logging +import os +from datetime import datetime, time, timedelta + +import hmac +from typing import List + +import pandas as pd +import pytz +import streamlit as st +import plotly.graph_objects as go +from dateutil.relativedelta import relativedelta +from skypro.common.cli_utils.cli_utils import read_yaml_file +from skypro.commands.report.config.config import parse_config +from skypro.commands.report.main import report, Report +from skypro.common.cli_utils.cli_utils import substitute_vars +from skypro.common.microgrid_analysis.output import generate_output_df +from skypro.common.microgrid_analysis.bill_match import bill_match +from skypro.common.notice.notice import Notice + +TIMEZONE = pytz.timezone("Europe/London") + +# This is a Streamlit web app that uses the reporting functionality from `skypro/commands/report` + + +def main(): + logging.basicConfig(level=logging.INFO) # Set to logging.INFO for non-debug mode + + skip_password = os.environ.get("SKIP_PASSWORD") + if skip_password == "true": + logging.info("Skipping password gate...") + else: + password = os.environ.get("PASSWORD") + if password is None: + raise ValueError("No password defined") + if not check_password(password): + st.stop() # Do not continue if check_password is not True. + + flows_db_url = get_db_url("FLOWS_DB_URL", "~/.simt/env.json", "flows") + rates_db_url = get_db_url("RATES_DB_URL", "~/.simt/env.json", "rates") + + config_path = os.environ.get("CONFIG_FILE", "./config/config.yaml") + config = read_yaml_file(config_path) + + st.title("Simtricity Reporting") + + # Draw the first row, which contains three columns: + # 1. A scenario selection (the scenarios are defined in the config file) + # 2. A month selector + # 3. A custom date range selector + # Either the month selector or custom date range selector will be used to set the time range of the reporting + now = datetime.now() + _, num_days = calendar.monthrange(now.year, now.month-1) + col1, col2, col3 = st.columns(3) + with col1: + scenario_name = st.selectbox( + "Select Scenario", + config["reportScenarios"].keys(), + ) + with col2: + months_df = get_previous_months_df(24) + month_select_start = st.selectbox( + label="Select month", + options=months_df["start"], + index=None, # select nothing by default + placeholder="Choose an option", + format_func=lambda x: x.strftime("%Y %B") + ) + if month_select_start is None: + month_select_end = None + else: + i = months_df.index[months_df["start"] == month_select_start].to_list()[0] + month_select_end = months_df.loc[i, "end"] + + with col3: + custom_selected = st.date_input( + label="Or select a custom time range", + value=[], + format="DD/MM/YYYY", + ) + if isinstance(custom_selected, tuple) and len(custom_selected) == 2: + custom_selected_start = custom_selected[0] + custom_selected_end = custom_selected[1] + else: + custom_selected_start = None + custom_selected_end = None + + start = month_select_start + end = month_select_end + if custom_selected_start is not None: + start = custom_selected_start + end = custom_selected_end + + if start is None: + st.stop() + + # Convert the start and end dates to aware datetimes + step_size = timedelta(minutes=5) + start = TIMEZONE.localize(datetime.combine(date=start, time=time())) + end = TIMEZONE.localize(datetime.combine(date=end, time=time())) + timedelta(days=1) - step_size + st.write(f"Reporting for period: {start} -> {end}") + + # Read in the reporting configuration for this scenario + report_config_path = config["reportScenarios"][scenario_name]["config"] + report_config = parse_config(report_config_path, env_vars=config["vars"]) + + with st.spinner("Running report..."): + result = report( + config=report_config, + flows_db_url=flows_db_url, + rates_db_url=rates_db_url, + start=start, + end=end, + step_size=step_size, + file_path_resolver_func=lambda file: os.path.expanduser(substitute_vars(file, config["vars"])), # Substitutes env vars and resolves `~` in file paths. This captures the `config` variable. + ) + + # The reporting run has produced a set of 'notices' for the user - display them in a table, highlighting any severe notices + notice_df = get_notice_df(result.notices) + num_important_notices = len(notice_df[notice_df["level_number"] >= 2]) + num_notices = len(notice_df) + with st.expander(f"{num_important_notices} important notices ({num_notices} in total)", icon="⚠️" if num_important_notices > 0 else None): + st.dataframe(notice_df[["Level", "Description"]], hide_index=True) + + # Present the import and export invoice estimates using 'bill matching' so that they are formatted like we see from our suppliers + import_bill = bill_match( + grid_energy_flow=result.df["grid_import"], + mkt_vol_grid_rates_df=result.mkt_vol_rates_dfs["grid_to_batt"], # use the grid rates for grid_to_batt as these include info about any OSAM rates + mkt_fixed_costs_df=result.mkt_fixed_cost_dfs["import"], + osam_rates=result.osam_rates, + osam_df=result.osam_df, + cepro_mkt_vol_bill_total_expected=result.breakdown.total_mkt_vol_costs["grid_to_batt"] + result.breakdown.total_mkt_vol_costs["grid_to_load"], + context="import", + line_items=report_config.reporting.bill_match.import_direction.line_items, + ) + + export_bill = bill_match( + grid_energy_flow=result.df["grid_export"], + mkt_vol_grid_rates_df=result.mkt_vol_rates_dfs["batt_to_grid"], # we have to pick one set of rates for all exports, so use batt_to_grid here, although solar_to_grid should also be the same + mkt_fixed_costs_df=result.mkt_fixed_cost_dfs["export"], + osam_rates=result.osam_rates, + osam_df=result.osam_df, + cepro_mkt_vol_bill_total_expected=result.breakdown.total_mkt_vol_costs["batt_to_grid"] + result.breakdown.total_mkt_vol_costs["solar_to_grid"], + context="export", + line_items=report_config.reporting.bill_match.export_direction.line_items, + ) + + with st.expander(f"Import Bill: £{import_bill.bill_total / 100:.2f}"): + st.write("Note that TNUoS, Ripple, and Capacity Market are not currently accounted for.") + st.dataframe(import_bill.bill_by_line_items_df, hide_index=False) + + with st.expander(f"Export Bill: £{export_bill.bill_total / 100:.2f}"): + st.dataframe(export_bill.bill_by_line_items_df, hide_index=False) + + with st.expander("Remote Generator: £N/A"): + st.write("Remote generators (e.g. Ripple) are not currently accounted for") + + with st.expander("Customer Income: £N/A"): + st.write("Detailed breakdown not yet available") + + st.write(f"Average cycles per day: {result.total_cycles/result.num_days:.1f}") + + draw_sankey(result) + + # Provide an option to download a CSV file for further analysis + output_df = generate_output_df( + df=result.df, + int_final_vol_rates_dfs=result.int_vol_rates_dfs, + mkt_final_vol_rates_dfs=result.mkt_vol_rates_dfs, + int_live_vol_rates_dfs=None, + mkt_live_vol_rates_dfs=None, + mkt_fixed_costs_dfs=result.mkt_fixed_cost_dfs, + customer_fixed_cost_dfs=result.customer_fixed_cost_dfs, + customer_vol_rates_dfs=result.customer_vol_rates_dfs, + load_energy_breakdown_df=None, + aggregate_timebase="30min", + rate_detail="all", + config_entries=[], + ) + output_csv = output_df.to_csv( + index_label="utctime" + ).encode("utf-8") + + st.download_button( + "Download CSV", + output_csv, + f"{scenario_name.lower().replace(' ', '_')}_report_{start.date().isoformat().replace('-', '_')}_{end.date().isoformat().replace('-', '_')}.csv", + "text/csv", + on_click="ignore", + key='download-csv' + ) + + +def get_db_url(env_var_name: str, env_config_path: str, env_config_section: str): + """ + Get database URL from environment variable or environment config file. + """ + db_url = os.environ.get(env_var_name) + if db_url is None: + logging.info(f"No {env_var_name} defined, trying {env_config_path}...") + try: + env_config = read_yaml_file(env_config_path) + except FileNotFoundError: + raise ValueError(f"Failed to find {env_var_name} in either environment variables or {env_config_path}") + + db_url = env_config[env_config_section]["dbUrl"] + if db_url is None: + raise ValueError(f"Failed to find {env_var_name} in either environment variables or {env_config_path}") + + return db_url + + +def get_previous_months_df(num_months: int) -> pd.DataFrame: + """ + Returns a dataframe containing the start and end dates of the previous months. + """ + starts = [] + ends = [] + current_date = datetime.now() + + # Generate the previous months + for i in range(1, num_months): + # Subtract i months from current date + past_date = current_date - relativedelta(months=i) + + # Create first day of the month + start_of_month = datetime(past_date.year, past_date.month, 1) + + # Create last day of the month (first day of next month - 1 day) + if past_date.month == 12: + end_of_month = datetime(past_date.year + 1, 1, 1) - timedelta(days=1) + else: + end_of_month = datetime(past_date.year, past_date.month + 1, 1) - timedelta(days=1) + + starts.append(start_of_month.date()) + ends.append(end_of_month.date()) + + # Create and return DataFrame + df = pd.DataFrame({ + 'start': starts, + 'end': ends + }) + + return df + + +def check_password(password: str): + """Returns `True` if the user had the correct password.""" + + def password_entered(): + """Checks whether a password entered by the user is correct.""" + if hmac.compare_digest(st.session_state["password"], password): + st.session_state["password_correct"] = True + del st.session_state["password"] # Don't store the password. + else: + st.session_state["password_correct"] = False + + # Return True if the password is validated. + if st.session_state.get("password_correct", False): + return True + + # Show input for password. + st.text_input( + "Password", type="password", on_change=password_entered, key="password" + ) + if "password_correct" in st.session_state: + st.error("😕 Password incorrect") + return False + + +def draw_sankey(result: Report): + """ + Plots a Sankey diagram of the energy flows in the microgrid + """ + color_blue = "rgba(38, 70, 83, 0.5)" + color_orange = "rgba(231, 111, 81, 0.5)" + color_teal = "rgba(42, 157, 143, 0.5)" + color_yellow = "rgba(233, 196, 106, 0.5)" + color_grey = "rgba(224, 225, 221, 0.7)" + sankey_df = pd.DataFrame.from_dict( + { + 0: ["Imports", "Battery", result.breakdown.total_flows["grid_to_batt"], color_grey, f"{result.breakdown.avg_mkt_vol_rates['grid_to_batt']:.2f} p/kWh"], + 1: ["Imports", "Loads", result.breakdown.total_flows["grid_to_load"], color_grey, f"{result.breakdown.avg_mkt_vol_rates['grid_to_load']:.2f} p/kWh"], + 2: ["Solar", "Battery", result.breakdown.total_flows["solar_to_batt"], color_yellow, f"{result.breakdown.avg_int_vol_rates['solar_to_batt']:.2f} p/kWh (displaced export rate)"], + 3: ["Solar", "Exports", result.breakdown.total_flows["solar_to_grid"], color_yellow, f"{result.breakdown.avg_mkt_vol_rates['solar_to_grid']:.2f} p/kWh"], + 4: ["Solar", "Loads", result.breakdown.total_flows["solar_to_load"], color_yellow, f"{result.breakdown.avg_int_vol_rates['solar_to_load']:.2f} p/kWh (displaced import rate)"], + 5: ["Battery", "Exports", result.breakdown.total_flows["batt_to_grid"], color_blue, f"{result.breakdown.avg_mkt_vol_rates['batt_to_grid']:.2f} p/kWh"], + 6: ["Battery", "Loads", result.breakdown.total_flows["batt_to_load"], color_blue, f"{result.breakdown.avg_int_vol_rates['batt_to_load']:.2f} p/kWh (displaced import rate)"], + 7: ["Battery", "Losses", result.breakdown.total_flows["bess_losses"], color_orange, ""], + # 8: ["Microgrid", "Power", 1, color_teal, ""], + # 9: ["Microgrid", "EVs", 1, color_teal, ""], + # 10: ["Microgrid", "Heat", 1, color_teal, ""], + }, + orient="index" + ) + sankey_df = sankey_df.rename(columns={ + 0: "source", + 1: "target", + 2: "value", + 3: "color", + 4: "label" + }) + + unique_source_target = list(pd.unique(sankey_df[['source', 'target']].values.ravel('K'))) + mapping_dict = {k: v for v, k in enumerate(unique_source_target)} + sankey_df['source'] = sankey_df['source'].map(mapping_dict) + sankey_df['target'] = sankey_df['target'].map(mapping_dict) + links_dict = sankey_df.to_dict(orient='list') + + fig = go.Figure(data=[go.Sankey( + arrangement="snap", + node=dict( + pad=15, + thickness=15, + line=dict(color="black", width=0.5), + label=unique_source_target, + color=[color_grey, color_yellow, color_blue, color_teal, color_grey, color_orange, color_teal, color_teal, + color_teal] + ), + link=dict( + source=links_dict["source"], + target=links_dict["target"], + value=links_dict["value"], + label=links_dict["label"], + color=links_dict["color"] + ))]) + + fig.update_layout(font_size=10) + + st.write("") + st.write("") + st.subheader("Energy Flows") + st.plotly_chart(fig) + + +def get_notice_df(notices: List[Notice]) -> pd.DataFrame: + """ + Returns a dataframe summarising the Notices, which can then be presented to the user. + """ + df = pd.DataFrame(columns=["level_number", "Description"]) + for notice in notices: + df.loc[len(df)] = [notice.level.value, notice.detail] + df = df.sort_values("level_number", ascending=False) + + df["Level"] = "N/A" + df.loc[df["level_number"] == 1, "Level"] = "Info" + df.loc[df["level_number"] == 2, "Level"] = "⚠️ Noteworthy" + df.loc[df["level_number"] == 3, "Level"] = "⚠️ Serious" + + df = df.reset_index(drop=True) + + return df + + +if __name__ == '__main__': + main()