From 2d9ca9187c06f99843b50c1f756d70f143e6ef9b Mon Sep 17 00:00:00 2001 From: Guilly Kolkman Date: Wed, 17 Dec 2025 16:30:53 +0100 Subject: [PATCH 1/5] Implemented the plotting of ensemble methods with model contributions. changed slight naming. Currently hardcoded results path --- examples/benchmarks/liander_2024_ensemble.py | 41 +- examples/model_contributions_plot.py | 443 ++++++++++++++++++ .../openstef_beam/evaluation/models/report.py | 12 +- 3 files changed, 477 insertions(+), 19 deletions(-) create mode 100644 examples/model_contributions_plot.py diff --git a/examples/benchmarks/liander_2024_ensemble.py b/examples/benchmarks/liander_2024_ensemble.py index f5ca1c5a6..f0e57147a 100644 --- a/examples/benchmarks/liander_2024_ensemble.py +++ b/examples/benchmarks/liander_2024_ensemble.py @@ -25,12 +25,22 @@ from pydantic_extra_types.coordinate import Coordinate from pydantic_extra_types.country import CountryAlpha2 -from openstef_beam.backtesting.backtest_forecaster import BacktestForecasterConfig, OpenSTEF4BacktestForecaster +from openstef_beam.backtesting.backtest_forecaster import ( + BacktestForecasterConfig, + OpenSTEF4BacktestForecaster, +) from openstef_beam.benchmarking.benchmark_pipeline import BenchmarkContext -from openstef_beam.benchmarking.benchmarks.liander2024 import Liander2024Category, create_liander2024_benchmark_runner -from openstef_beam.benchmarking.callbacks.strict_execution_callback import StrictExecutionCallback +from openstef_beam.benchmarking.benchmarks.liander2024 import ( + Liander2024Category, + create_liander2024_benchmark_runner, +) +from openstef_beam.benchmarking.callbacks.strict_execution_callback import ( + StrictExecutionCallback, +) from openstef_beam.benchmarking.models.benchmark_target import BenchmarkTarget -from openstef_beam.benchmarking.storage.local_storage import LocalBenchmarkStorage +from openstef_beam.benchmarking.storage.local_storage import ( + LocalBenchmarkStorage, +) from openstef_core.types import LeadTime, Q from openstef_meta.presets import ( EnsembleWorkflowConfig, @@ -47,12 +57,15 @@ N_PROCESSES = 11 if True else multiprocessing.cpu_count() # Amount of parallel processes to use for the benchmark ensemble_type = "learned_weights" # "stacking", "learned_weights" or "rules" -base_models = ["lgbm", "gblinear"] # combination of "lgbm", "gblinear", "xgboost" and "lgbm_linear" +base_models = [ + "lgbm", + "gblinear", +] # combination of "lgbm", "gblinear", "xgboost" and "lgbm_linear" combiner_model = ( "lgbm" # "lgbm", "xgboost", "rf" or "logistic" for learned weights combiner, gblinear for stacking combiner ) -model = "Ensemble_" + "_".join(base_models) + "_" + ensemble_type + "_" + combiner_model +model = "Ensemble_contributions_" + "_".join(base_models) + "_" + ensemble_type + "_" + combiner_model # Model configuration FORECAST_HORIZONS = [LeadTime.from_string("PT36H")] # Forecast horizon(s) @@ -95,7 +108,12 @@ relative_humidity_column="relative_humidity_2m", energy_price_column="EPEX_NL", forecast_combiner_sample_weight_exponent=0, - forecaster_sample_weight_exponent={"gblinear": 1, "lgbm": 0, "xgboost": 0, "lgbm_linear": 0}, + forecaster_sample_weight_exponent={ + "gblinear": 1, + "lgbm": 0, + "xgboost": 0, + "lgbm_linear": 0, + }, ) @@ -112,10 +130,7 @@ ) -def _target_forecaster_factory( - context: BenchmarkContext, - target: BenchmarkTarget, -) -> OpenSTEF4BacktestForecaster: +def _target_forecaster_factory(context: BenchmarkContext, target: BenchmarkTarget) -> OpenSTEF4BacktestForecaster: # Factory function that creates a forecaster for a given target. prefix = context.run_name base_config = common_config @@ -143,7 +158,7 @@ def _create_workflow() -> CustomForecastingWorkflow: config=backtest_config, workflow_factory=_create_workflow, debug=False, - contributions=False, + contributions=True, cache_dir=OUTPUT_PATH / "cache" / f"{context.run_name}_{target.name}", ) @@ -152,7 +167,7 @@ def _create_workflow() -> CustomForecastingWorkflow: start_time = time.time() create_liander2024_benchmark_runner( storage=LocalBenchmarkStorage(base_path=OUTPUT_PATH / model), - data_dir=Path("../data/liander2024-energy-forecasting-benchmark"), + data_dir=Path("data/liander2024-energy-forecasting-benchmark"), callbacks=[StrictExecutionCallback()], ).run( forecaster_factory=_target_forecaster_factory, diff --git a/examples/model_contributions_plot.py b/examples/model_contributions_plot.py new file mode 100644 index 000000000..6790723b4 --- /dev/null +++ b/examples/model_contributions_plot.py @@ -0,0 +1,443 @@ +# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project +# SPDX-License-Identifier: MPL-2.0 +"""Model contribution visualization for ensemble forecasting. + +This module provides functionality to visualize and analyze model contributions +in ensemble forecasting, specifically comparing GBLinear and LGBM base models +across different quantiles. +""" + +from pathlib import Path + +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from tqdm import tqdm + +from openstef_beam.analysis.plots import ForecastTimeSeriesPlotter + +try: + from openstef_beam.analysis.plots import ForecastTimeSeriesPlotter +except ImportError: + ForecastTimeSeriesPlotter = None + + +def load_contribution_data(folder_path: Path, n_rows: int = 24) -> pd.DataFrame: + """Load and concatenate contribution data from parquet files. + + Args: + folder_path: Path to folder containing contribution parquet files. + n_rows: Number of rows to take from each file (default: 24 for 6 hours + at 15-minute intervals). + + Returns: + Concatenated DataFrame with contribution data. + + Raises: + ValueError: If no parquet files are found in the specified folder. + """ + parquet_files = sorted(folder_path.glob("contrib_*_predict.parquet")) + + if not parquet_files: + msg = f"No contribution files found in {folder_path}" + raise ValueError(msg) + + print(f"Found {len(parquet_files)} contribution files") + + df_list = [] + for file in tqdm(parquet_files, desc="Loading contribution data"): + df_temp = pd.read_parquet(file) + df_subset = df_temp.head(n_rows) + df_list.append(df_subset) + + df_combined = pd.concat(df_list, axis=0, ignore_index=False) + print(f"Combined dataframe shape: {df_combined.shape}") + if not df_combined.empty: + start_ts = df_combined.index.min() + end_ts = df_combined.index.max() + print(f"Dataframe start: {start_ts}, end: {end_ts}") + else: + print("Combined dataframe is empty; no timestamps available.") + + return df_combined + + +def _filter_by_date_range( + df: pd.DataFrame, + start_date: str | pd.Timestamp | None = None, + end_date: str | pd.Timestamp | None = None, +) -> pd.DataFrame: + """Filter dataframe by date range, handling timezone-aware indices. + + Args: + df: DataFrame with DatetimeIndex. + start_date: Start date for filtering. If None, no start filtering. + end_date: End date for filtering. If None, no end filtering. + + Returns: + Filtered DataFrame. + """ + df_filtered = df.copy() + + if start_date is not None: + start_dt = pd.to_datetime(start_date) + if df_filtered.index.tz is not None and start_dt.tz is None: + start_dt = start_dt.tz_localize("UTC") + df_filtered = df_filtered[df_filtered.index >= start_dt] + + if end_date is not None: + end_dt = pd.to_datetime(end_date) + if df_filtered.index.tz is not None and end_dt.tz is None: + end_dt = end_dt.tz_localize("UTC") + df_filtered = df_filtered[df_filtered.index <= end_dt] + + return df_filtered + + +def _create_model_traces(df: pd.DataFrame, quantiles: list[str], default_quantile: str = "P50") -> go.Figure: + """Create Plotly traces for each model and quantile. + + Args: + df: DataFrame with model contribution columns. + quantiles: List of quantile names (e.g., ['P05', 'P50']). + default_quantile: Quantile to show by default. + + Returns: + Plotly Figure with traces for all models and quantiles. + """ + fig = go.Figure() + + for quantile in quantiles: + gblinear_col = f"gblinear_quantile_{quantile}" + lgbm_col = f"lgbm_quantile_{quantile}" + + is_visible = quantile == default_quantile + + # Add GBLinear trace + fig.add_trace( + go.Scatter( + x=df.index, + y=df[gblinear_col], + name=f"GBLinear - {quantile}", + mode="lines", + line={"width": 2, "color": "#1f77b4"}, + opacity=0.85, + visible=is_visible, + ) + ) + + # Add LGBM trace + fig.add_trace( + go.Scatter( + x=df.index, + y=df[lgbm_col], + name=f"LGBM - {quantile}", + mode="lines", + line={"width": 2, "color": "#ff7f0e"}, + opacity=0.85, + visible=is_visible, + ) + ) + + return fig + + +def _create_quantile_buttons(quantiles: list[str]) -> list[dict]: + """Create button controls for quantile selection. + + Args: + quantiles: List of quantile names. + + Returns: + List of button configuration dictionaries. + """ + buttons = [] + for i, quantile in enumerate(quantiles): + # Two traces per quantile (GBLinear and LGBM) + visible = [False] * (len(quantiles) * 2) + visible[i * 2] = True # GBLinear + visible[i * 2 + 1] = True # LGBM + + buttons.append({ + "label": quantile, + "method": "update", + "args": [ + {"visible": visible}, + {"title.text": (f"Model Contributions - Quantile {quantile}")}, + ], + }) + + return buttons + + +def plot_model_contributions( + df: pd.DataFrame, + start_date: str | pd.Timestamp | None = None, + end_date: str | pd.Timestamp | None = None, + quantiles: list[str] | None = None, +) -> go.Figure: + """Plot model contributions with interactive quantile selection. + + Creates an interactive Plotly visualization comparing GBLinear and LGBM + model contributions across different quantiles, with button controls to + switch between quantiles. + + Args: + df: DataFrame with model contribution columns. Expected columns: + 'gblinear_quantile_{Q}' and 'lgbm_quantile_{Q}' for each quantile. + start_date: Start date for filtering. If None, uses all data. + end_date: End date for filtering. If None, uses all data. + quantiles: List of quantile names (e.g., ['P05', 'P50']). If None, + defaults to ['P05', 'P10', 'P30', 'P50', 'P70', 'P90', 'P95']. + + Returns: + Plotly Figure object with interactive visualization. + + Example: + >>> df = load_contribution_data(Path("path/to/contributions")) + >>> fig = plot_model_contributions( + ... df, + ... start_date='2024-03-01', + ... end_date='2024-03-31', + ... quantiles=['P50', 'P90'] + ... ) + >>> fig.show() + """ + if quantiles is None: + quantiles = ["P05", "P10", "P30", "P50", "P70", "P90", "P95"] + + # Filter data by date range + df_plot = _filter_by_date_range(df, start_date, end_date) + + # Create traces for all models and quantiles + default_quantile = "P50" if "P50" in quantiles else quantiles[0] + fig = _create_model_traces(df_plot, quantiles, default_quantile) + + # Create quantile selection buttons + buttons = _create_quantile_buttons(quantiles) + active_idx = quantiles.index(default_quantile) + + # Update layout with controls and styling + fig.update_layout( + legend={ + "orientation": "v", + "x": 1.02, + "xanchor": "left", + "y": 1.0, + "yanchor": "top", + }, + updatemenus=[ + { + "type": "buttons", + "buttons": buttons, + "direction": "down", + "showactive": True, + "active": active_idx, + "x": 1.02, + "xanchor": "left", + "y": 0.5, + "yanchor": "middle", + "pad": {"r": 10, "t": 10}, + "bgcolor": "lightgray", + "bordercolor": "gray", + "borderwidth": 1, + } + ], + title=f"Model Contributions - Quantile {quantiles[active_idx]}", + xaxis_title="Timestamp", + yaxis_title="Contribution", + hovermode="x unified", + height=600, + showlegend=True, + ) + + return fig + + +def plot_combined_visualization( + df: pd.DataFrame, + start_date: str | pd.Timestamp | None = None, + end_date: str | pd.Timestamp | None = None, + quantiles: list[str] | None = None, +) -> go.Figure: + """Plot model contributions and forecast time series in subplots. + + Creates a combined visualization with two subplots: + 1. Top: Model contributions comparing GBLinear and LGBM + 2. Bottom: Forecast time series using ForecastTimeSeriesPlotter showing + ensemble forecast, measurements, and uncertainty bands + + Args: + df: DataFrame with model contribution columns, ensemble forecasts, and load. + Expected columns: 'gblinear_quantile_{Q}', 'lgbm_quantile_{Q}', + 'quantile_{Q}', and 'load'. + start_date: Start date for filtering. If None, uses all data. + end_date: End date for filtering. If None, uses all data. + quantiles: List of quantile names (e.g., ['P05', 'P50']). If None, + defaults to ['P05', 'P10', 'P30', 'P50', 'P70', 'P90', 'P95']. + + Returns: + Plotly Figure object with combined subplots. + """ + if quantiles is None: + quantiles = ["P05", "P10", "P30", "P50", "P70", "P90", "P95"] + + # Filter data by date range + df_plot = _filter_by_date_range(df, start_date, end_date) + + # Create forecast plot using ForecastTimeSeriesPlotter + # Prepare quantiles DataFrame for the plotter + quantile_cols = [f"quantile_{q}" for q in quantiles] + quantiles_df = df_plot[quantile_cols].copy() + + # Create measurements series + measurements = df_plot["load"] + + # Use ForecastTimeSeriesPlotter to create the forecast visualization + forecast_plotter = ForecastTimeSeriesPlotter() + forecast_plotter.add_measurements(measurements) + forecast_plotter.add_model( + model_name="Ensemble", + forecast=df_plot["quantile_P50"], + quantiles=quantiles_df, + ) + forecast_fig = forecast_plotter.plot(title="Ensemble Forecast vs Measurements") + + # Create subplot figure with 2 rows + fig = make_subplots( + rows=2, + cols=1, + row_heights=[0.5, 0.5], + subplot_titles=("Ensemble Forecast vs Measurements", "Model Contributions"), + vertical_spacing=0.12, + shared_xaxes=True, + ) + + # Add forecast traces to top subplot (row=1) + for trace in forecast_fig.data: + fig.add_trace(trace, row=1, col=1) + + # Count the number of forecast traces for visibility calculation + num_forecast_traces = len(forecast_fig.data) + + # Add contribution traces to bottom subplot (row=2) + default_quantile = "P50" if "P50" in quantiles else quantiles[0] + + for quantile in quantiles: + gblinear_col = f"gblinear_quantile_{quantile}" + lgbm_col = f"lgbm_quantile_{quantile}" + + is_visible = quantile == default_quantile + + # Add GBLinear trace + fig.add_trace( + go.Scatter( + x=df_plot.index, + y=df_plot[gblinear_col], + name=f"GBLinear - {quantile}", + mode="lines", + line={"width": 2, "color": "#1f77b4"}, + opacity=0.85, + visible=is_visible, + legendgroup="contributions", + ), + row=2, + col=1, + ) + + # Add LGBM trace + fig.add_trace( + go.Scatter( + x=df_plot.index, + y=df_plot[lgbm_col], + name=f"LGBM - {quantile}", + mode="lines", + line={"width": 2, "color": "#ff7f0e"}, + opacity=0.85, + visible=is_visible, + legendgroup="contributions", + ), + row=2, + col=1, + ) + + # Create quantile selection buttons (only affects contribution subplot) + buttons = [] + for i, quantile in enumerate(quantiles): + # Calculate visibility for all traces + # First set for forecast traces (always visible) + visible = [True] * num_forecast_traces + + # Add visibility for contribution traces (2 traces per quantile) + for j, _ in enumerate(quantiles): + visible.extend([j == i, j == i]) # GBLinear and LGBM visibility + + buttons.append({ + "label": quantile, + "method": "update", + "args": [ + {"visible": visible}, + {"title.text": (f"Model Contributions & Forecast - Quantile {quantile}")}, + ], + }) + + active_idx = quantiles.index(default_quantile) + + # Update layout + fig.update_layout( + title=f"Model Contributions & Forecast - Quantile {quantiles[active_idx]}", + height=1000, + hovermode="x unified", + showlegend=True, + updatemenus=[ + { + "type": "buttons", + "buttons": buttons, + "direction": "down", + "showactive": True, + "active": active_idx, + "x": 1.02, + "xanchor": "left", + "y": 0.5, + "yanchor": "middle", + "pad": {"r": 10, "t": 10}, + "bgcolor": "lightgray", + "bordercolor": "gray", + "borderwidth": 1, + } + ], + ) + + # Update axes labels + fig.update_xaxes(title_text="Timestamp", row=2, col=1) + fig.update_yaxes(title_text="Load (MW)", row=1, col=1) + fig.update_yaxes(title_text="Contribution", row=2, col=1) + + return fig + + +def main() -> None: + """Main function to demonstrate contribution visualization.""" + # Load contribution data + # Get the project root (two levels up from this file) + project_root = Path(__file__).parent.parent + folder_path = ( + project_root + / "benchmark_results" + / "cache" + / "Ensemble_contributions_lgbm_gblinear_learned_weights_lgbm_OS Apeldoorn" + ) + + df_combined = load_contribution_data(folder_path, n_rows=24) + + # Create combined visualization with contributions and forecast subplots + combined_fig = plot_combined_visualization(df_combined) + combined_output = project_root / "benchmark_results" / "model_contributions_combined_plot.html" + combined_fig.write_html(combined_output) + print(f"Combined plot saved to: {combined_output}") + + combined_fig.show() + + +if __name__ == "__main__": + main() diff --git a/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py b/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py index 7766d5947..e56124876 100644 --- a/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py +++ b/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py @@ -47,6 +47,8 @@ def to_parquet(self, path: Path): Args: path: Directory where to save the report data. """ + # Sanitize path by replacing colons (invalid on Windows) + path = Path(str(path).replace(":", "_")) path.mkdir(parents=True, exist_ok=True) (path / "metrics.json").write_bytes(TypeAdapter(list[SubsetMetric]).dump_json(self.metrics)) self.subset.to_parquet(path / "subset.parquet") @@ -66,11 +68,7 @@ def read_parquet(cls, path: Path) -> Self: ) filtering = TypeAdapter[Filtering](Filtering).validate_python(path.name) subset = ForecastDataset.read_parquet(path / "subset.parquet") - return cls( - filtering=filtering, - subset=subset, - metrics=metrics, - ) + return cls(filtering=filtering, subset=subset, metrics=metrics) def get_global_metric(self) -> SubsetMetric | None: """Returns the SubsetMetric with window='global', or None if not found.""" @@ -129,7 +127,9 @@ def to_parquet(self, path: Path): """ path.mkdir(parents=True, exist_ok=True) for subset_report in self.subset_reports: - subset_report.to_parquet(path / str(subset_report.filtering)) + # Sanitize filtering name for Windows compatibility (replace colons) + filtering_name = str(subset_report.filtering).replace(":", "_") + subset_report.to_parquet(path / filtering_name) @classmethod def read_parquet(cls, path: Path) -> Self: From cea9038d2d79ddffa3a3cd8bd24ac971e59c7ab0 Mon Sep 17 00:00:00 2001 From: Guilly Kolkman Date: Wed, 17 Dec 2025 17:04:35 +0100 Subject: [PATCH 2/5] Fixed typing issues in model_contributions_plot.py Signed-off-by: Guilly Kolkman --- examples/model_contributions_plot.py | 61 +++++++++++++--------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/examples/model_contributions_plot.py b/examples/model_contributions_plot.py index 6790723b4..a0398cca1 100644 --- a/examples/model_contributions_plot.py +++ b/examples/model_contributions_plot.py @@ -11,7 +11,7 @@ import pandas as pd import plotly.graph_objects as go -from plotly.subplots import make_subplots +from plotly.subplots import make_subplots # type: ignore[attr-defined] from tqdm import tqdm from openstef_beam.analysis.plots import ForecastTimeSeriesPlotter @@ -46,11 +46,11 @@ def load_contribution_data(folder_path: Path, n_rows: int = 24) -> pd.DataFrame: df_list = [] for file in tqdm(parquet_files, desc="Loading contribution data"): - df_temp = pd.read_parquet(file) + df_temp = pd.read_parquet(file) # type: ignore[call-overload] df_subset = df_temp.head(n_rows) - df_list.append(df_subset) + df_list.append(df_subset) # type: ignore[arg-type] - df_combined = pd.concat(df_list, axis=0, ignore_index=False) + df_combined = pd.concat(df_list, axis=0, ignore_index=False) # type: ignore[arg-type] print(f"Combined dataframe shape: {df_combined.shape}") if not df_combined.empty: start_ts = df_combined.index.min() @@ -81,13 +81,13 @@ def _filter_by_date_range( if start_date is not None: start_dt = pd.to_datetime(start_date) - if df_filtered.index.tz is not None and start_dt.tz is None: + if df_filtered.index.tz is not None and start_dt.tz is None: # type: ignore[attr-defined] start_dt = start_dt.tz_localize("UTC") df_filtered = df_filtered[df_filtered.index >= start_dt] if end_date is not None: end_dt = pd.to_datetime(end_date) - if df_filtered.index.tz is not None and end_dt.tz is None: + if df_filtered.index.tz is not None and end_dt.tz is None: # type: ignore[attr-defined] end_dt = end_dt.tz_localize("UTC") df_filtered = df_filtered[df_filtered.index <= end_dt] @@ -114,7 +114,7 @@ def _create_model_traces(df: pd.DataFrame, quantiles: list[str], default_quantil is_visible = quantile == default_quantile # Add GBLinear trace - fig.add_trace( + fig.add_trace( # type: ignore[call-overload] go.Scatter( x=df.index, y=df[gblinear_col], @@ -127,7 +127,7 @@ def _create_model_traces(df: pd.DataFrame, quantiles: list[str], default_quantil ) # Add LGBM trace - fig.add_trace( + fig.add_trace( # type: ignore[call-overload] go.Scatter( x=df.index, y=df[lgbm_col], @@ -142,7 +142,7 @@ def _create_model_traces(df: pd.DataFrame, quantiles: list[str], default_quantil return fig -def _create_quantile_buttons(quantiles: list[str]) -> list[dict]: +def _create_quantile_buttons(quantiles: list[str]) -> list[dict]: # type: ignore[type-arg] """Create button controls for quantile selection. Args: @@ -158,7 +158,7 @@ def _create_quantile_buttons(quantiles: list[str]) -> list[dict]: visible[i * 2] = True # GBLinear visible[i * 2 + 1] = True # LGBM - buttons.append({ + buttons.append({ # type: ignore[arg-type] "label": quantile, "method": "update", "args": [ @@ -167,7 +167,7 @@ def _create_quantile_buttons(quantiles: list[str]) -> list[dict]: ], }) - return buttons + return buttons # type: ignore[return-value] def plot_model_contributions( @@ -214,11 +214,11 @@ def plot_model_contributions( fig = _create_model_traces(df_plot, quantiles, default_quantile) # Create quantile selection buttons - buttons = _create_quantile_buttons(quantiles) + buttons = _create_quantile_buttons(quantiles) # type: ignore[assignment] active_idx = quantiles.index(default_quantile) # Update layout with controls and styling - fig.update_layout( + fig.update_layout( # type: ignore[call-overload] legend={ "orientation": "v", "x": 1.02, @@ -286,20 +286,15 @@ def plot_combined_visualization( df_plot = _filter_by_date_range(df, start_date, end_date) # Create forecast plot using ForecastTimeSeriesPlotter - # Prepare quantiles DataFrame for the plotter quantile_cols = [f"quantile_{q}" for q in quantiles] - quantiles_df = df_plot[quantile_cols].copy() - - # Create measurements series - measurements = df_plot["load"] # Use ForecastTimeSeriesPlotter to create the forecast visualization - forecast_plotter = ForecastTimeSeriesPlotter() - forecast_plotter.add_measurements(measurements) + forecast_plotter = ForecastTimeSeriesPlotter() # type: ignore[misc] + forecast_plotter.add_measurements(df_plot["load"]) forecast_plotter.add_model( model_name="Ensemble", forecast=df_plot["quantile_P50"], - quantiles=quantiles_df, + quantiles=df_plot[quantile_cols].copy(), ) forecast_fig = forecast_plotter.plot(title="Ensemble Forecast vs Measurements") @@ -314,11 +309,11 @@ def plot_combined_visualization( ) # Add forecast traces to top subplot (row=1) - for trace in forecast_fig.data: - fig.add_trace(trace, row=1, col=1) + for trace in forecast_fig.data: # type: ignore[attr-defined] + fig.add_trace(trace, row=1, col=1) # type: ignore[arg-type] # Count the number of forecast traces for visibility calculation - num_forecast_traces = len(forecast_fig.data) + num_forecast_traces = len(forecast_fig.data) # type: ignore[arg-type] # Add contribution traces to bottom subplot (row=2) default_quantile = "P50" if "P50" in quantiles else quantiles[0] @@ -330,7 +325,7 @@ def plot_combined_visualization( is_visible = quantile == default_quantile # Add GBLinear trace - fig.add_trace( + fig.add_trace( # type: ignore[call-overload] go.Scatter( x=df_plot.index, y=df_plot[gblinear_col], @@ -346,7 +341,7 @@ def plot_combined_visualization( ) # Add LGBM trace - fig.add_trace( + fig.add_trace( # type: ignore[call-overload] go.Scatter( x=df_plot.index, y=df_plot[lgbm_col], @@ -372,7 +367,7 @@ def plot_combined_visualization( for j, _ in enumerate(quantiles): visible.extend([j == i, j == i]) # GBLinear and LGBM visibility - buttons.append({ + buttons.append({ # type: ignore[arg-type] "label": quantile, "method": "update", "args": [ @@ -384,7 +379,7 @@ def plot_combined_visualization( active_idx = quantiles.index(default_quantile) # Update layout - fig.update_layout( + fig.update_layout( # type: ignore[call-overload] title=f"Model Contributions & Forecast - Quantile {quantiles[active_idx]}", height=1000, hovermode="x unified", @@ -409,9 +404,9 @@ def plot_combined_visualization( ) # Update axes labels - fig.update_xaxes(title_text="Timestamp", row=2, col=1) - fig.update_yaxes(title_text="Load (MW)", row=1, col=1) - fig.update_yaxes(title_text="Contribution", row=2, col=1) + fig.update_xaxes(title_text="Timestamp", row=2, col=1) # type: ignore[call-overload] + fig.update_yaxes(title_text="Load (MW)", row=1, col=1) # type: ignore[call-overload] + fig.update_yaxes(title_text="Contribution", row=2, col=1) # type: ignore[call-overload] return fig @@ -433,10 +428,10 @@ def main() -> None: # Create combined visualization with contributions and forecast subplots combined_fig = plot_combined_visualization(df_combined) combined_output = project_root / "benchmark_results" / "model_contributions_combined_plot.html" - combined_fig.write_html(combined_output) + combined_fig.write_html(combined_output) # type: ignore[call-overload] print(f"Combined plot saved to: {combined_output}") - combined_fig.show() + combined_fig.show() # type: ignore[call-overload] if __name__ == "__main__": From 2d61aaf559db4ffa7c89f02b7a325eb9022dd395 Mon Sep 17 00:00:00 2001 From: Guilly Kolkman Date: Wed, 17 Dec 2025 17:05:57 +0100 Subject: [PATCH 3/5] Fixed typing issue --- data/liander2024-energy-forecasting-benchmark | 1 + examples/model_contributions.ipynb | 1751 +++++++++++++++++ 2 files changed, 1752 insertions(+) create mode 160000 data/liander2024-energy-forecasting-benchmark create mode 100644 examples/model_contributions.ipynb diff --git a/data/liander2024-energy-forecasting-benchmark b/data/liander2024-energy-forecasting-benchmark new file mode 160000 index 000000000..ec28635e2 --- /dev/null +++ b/data/liander2024-energy-forecasting-benchmark @@ -0,0 +1 @@ +Subproject commit ec28635e20a12927f3323a2dc18a0719a361b56d diff --git a/examples/model_contributions.ipynb b/examples/model_contributions.ipynb new file mode 100644 index 000000000..d94d67eec --- /dev/null +++ b/examples/model_contributions.ipynb @@ -0,0 +1,1751 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "b52202e3", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import plotly.graph_objects as go\n", + "from plotly.subplots import make_subplots\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2ca96a4f", + "metadata": {}, + "outputs": [], + "source": [ + "# df = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/Ensemble_lgbm_gblinear_learned_weights_lgbm/backtest/transformer/OS Apeldoorn/predictions.parquet\")\n", + "# print(df.head())\n", + "# df_focus = df[[\"quantile_P05\"]]\n", + "\n", + "# start = \"2024-03-16\"\n", + "# end = \"2024-03-23\"\n", + "# df_week = df_focus.loc[start:end]\n", + "\n", + "# df_week.plot(figsize=(14, 6))\n", + "\n", + "# plt.title(\"Model Contributions\")\n", + "# plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "52e7d666", + "metadata": {}, + "outputs": [], + "source": [ + "# display(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e4dfe0a3", + "metadata": {}, + "outputs": [], + "source": [ + "df_contribution = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn/contrib_20240301060000_predict.parquet\")\n", + "df_contribution2 = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn/contrib_20240301000000_predict.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "659ade83", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 00:00:00+00:000.2488250.7511750.8114570.1885430.7757460.2242540.7967020.2032980.7373460.262654...0.1355130.864487-2.137667e+07-2.317569e+07-2.248770e+07-2.196953e+07-2.158473e+07-2.100265e+07-2.091442e+07-2.065124e+07
2024-03-01 00:15:00+00:000.0800040.9199960.5297130.4702870.7817720.2182280.7753140.2246860.7757290.224271...0.1477200.852280NaN-2.209275e+07-2.208461e+07-2.157246e+07-2.118342e+07-2.092382e+07-2.073507e+07-2.063586e+07
2024-03-01 00:30:00+00:000.4708160.5291840.1918370.8081630.8281760.1718240.7753140.2246860.7757290.224271...0.2423170.757683NaN-2.250475e+07-2.174137e+07-2.138064e+07-2.100023e+07-2.053325e+07-2.041650e+07-1.982233e+07
2024-03-01 00:45:00+00:000.0900270.9099730.1650050.8349950.5156460.4843540.8316680.1683320.6291620.370838...0.6789810.321019NaN-2.125884e+07-2.111755e+07-2.109466e+07-2.071542e+07-2.019039e+07-1.965634e+07-1.874860e+07
2024-03-01 01:00:00+00:000.0966000.9034000.6431520.3568480.8340690.1659310.8375540.1624460.5731100.426890...0.0898150.910185NaN-2.126173e+07-2.106740e+07-2.063920e+07-2.017966e+07-1.971798e+07-1.952271e+07-1.944001e+07
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.248825 0.751175 \n", + "2024-03-01 00:15:00+00:00 0.080004 0.919996 \n", + "2024-03-01 00:30:00+00:00 0.470816 0.529184 \n", + "2024-03-01 00:45:00+00:00 0.090027 0.909973 \n", + "2024-03-01 01:00:00+00:00 0.096600 0.903400 \n", + "\n", + " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.811457 0.188543 \n", + "2024-03-01 00:15:00+00:00 0.529713 0.470287 \n", + "2024-03-01 00:30:00+00:00 0.191837 0.808163 \n", + "2024-03-01 00:45:00+00:00 0.165005 0.834995 \n", + "2024-03-01 01:00:00+00:00 0.643152 0.356848 \n", + "\n", + " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.775746 0.224254 \n", + "2024-03-01 00:15:00+00:00 0.781772 0.218228 \n", + "2024-03-01 00:30:00+00:00 0.828176 0.171824 \n", + "2024-03-01 00:45:00+00:00 0.515646 0.484354 \n", + "2024-03-01 01:00:00+00:00 0.834069 0.165931 \n", + "\n", + " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.796702 0.203298 \n", + "2024-03-01 00:15:00+00:00 0.775314 0.224686 \n", + "2024-03-01 00:30:00+00:00 0.775314 0.224686 \n", + "2024-03-01 00:45:00+00:00 0.831668 0.168332 \n", + "2024-03-01 01:00:00+00:00 0.837554 0.162446 \n", + "\n", + " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", + "timestamp ... \n", + "2024-03-01 00:00:00+00:00 0.737346 0.262654 ... \n", + "2024-03-01 00:15:00+00:00 0.775729 0.224271 ... \n", + "2024-03-01 00:30:00+00:00 0.775729 0.224271 ... \n", + "2024-03-01 00:45:00+00:00 0.629162 0.370838 ... \n", + "2024-03-01 01:00:00+00:00 0.573110 0.426890 ... \n", + "\n", + " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.135513 0.864487 \n", + "2024-03-01 00:15:00+00:00 0.147720 0.852280 \n", + "2024-03-01 00:30:00+00:00 0.242317 0.757683 \n", + "2024-03-01 00:45:00+00:00 0.678981 0.321019 \n", + "2024-03-01 01:00:00+00:00 0.089815 0.910185 \n", + "\n", + " load quantile_P05 quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.137667e+07 -2.317569e+07 -2.248770e+07 \n", + "2024-03-01 00:15:00+00:00 NaN -2.209275e+07 -2.208461e+07 \n", + "2024-03-01 00:30:00+00:00 NaN -2.250475e+07 -2.174137e+07 \n", + "2024-03-01 00:45:00+00:00 NaN -2.125884e+07 -2.111755e+07 \n", + "2024-03-01 01:00:00+00:00 NaN -2.126173e+07 -2.106740e+07 \n", + "\n", + " quantile_P30 quantile_P50 quantile_P70 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.196953e+07 -2.158473e+07 -2.100265e+07 \n", + "2024-03-01 00:15:00+00:00 -2.157246e+07 -2.118342e+07 -2.092382e+07 \n", + "2024-03-01 00:30:00+00:00 -2.138064e+07 -2.100023e+07 -2.053325e+07 \n", + "2024-03-01 00:45:00+00:00 -2.109466e+07 -2.071542e+07 -2.019039e+07 \n", + "2024-03-01 01:00:00+00:00 -2.063920e+07 -2.017966e+07 -1.971798e+07 \n", + "\n", + " quantile_P90 quantile_P95 \n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.091442e+07 -2.065124e+07 \n", + "2024-03-01 00:15:00+00:00 -2.073507e+07 -2.063586e+07 \n", + "2024-03-01 00:30:00+00:00 -2.041650e+07 -1.982233e+07 \n", + "2024-03-01 00:45:00+00:00 -1.965634e+07 -1.874860e+07 \n", + "2024-03-01 01:00:00+00:00 -1.952271e+07 -1.944001e+07 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 06:00:00+00:000.6830180.3169820.9307990.0692010.8481510.1518490.7703280.2296720.8539480.146052...0.4385020.561498-23820000.0-2.635957e+07-2.558408e+07-2.448471e+07-2.390916e+07-2.351279e+07-2.248671e+07-2.211594e+07
2024-03-01 06:15:00+00:000.5213240.4786760.4236980.5763020.6382140.3617860.7932140.2067860.8550170.144983...0.1272800.872720NaN-2.763479e+07-2.622804e+07-2.546636e+07-2.520980e+07-2.481184e+07-2.364225e+07-2.321880e+07
2024-03-01 06:30:00+00:000.8591640.1408360.7824160.2175840.8953740.1046260.8339640.1660360.8416750.158325...0.1098940.890106NaN-2.860721e+07-2.760050e+07-2.660407e+07-2.602701e+07-2.557144e+07-2.470169e+07-2.462576e+07
2024-03-01 06:45:00+00:000.9488980.0511020.9099140.0900860.8268100.1731900.8627440.1372560.4236880.576312...0.1178930.882107NaN-2.972855e+07-2.872363e+07-2.763870e+07-2.716232e+07-2.711700e+07-2.652464e+07-2.651125e+07
2024-03-01 07:00:00+00:000.9489840.0510160.9250050.0749950.8517400.1482600.8582440.1417560.8283320.171668...0.1156830.884317NaN-3.082963e+07-2.989013e+07-2.899229e+07-2.831311e+07-2.772480e+07-2.708928e+07-2.703521e+07
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 0.683018 0.316982 \n", + "2024-03-01 06:15:00+00:00 0.521324 0.478676 \n", + "2024-03-01 06:30:00+00:00 0.859164 0.140836 \n", + "2024-03-01 06:45:00+00:00 0.948898 0.051102 \n", + "2024-03-01 07:00:00+00:00 0.948984 0.051016 \n", + "\n", + " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 0.930799 0.069201 \n", + "2024-03-01 06:15:00+00:00 0.423698 0.576302 \n", + "2024-03-01 06:30:00+00:00 0.782416 0.217584 \n", + "2024-03-01 06:45:00+00:00 0.909914 0.090086 \n", + "2024-03-01 07:00:00+00:00 0.925005 0.074995 \n", + "\n", + " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 0.848151 0.151849 \n", + "2024-03-01 06:15:00+00:00 0.638214 0.361786 \n", + "2024-03-01 06:30:00+00:00 0.895374 0.104626 \n", + "2024-03-01 06:45:00+00:00 0.826810 0.173190 \n", + "2024-03-01 07:00:00+00:00 0.851740 0.148260 \n", + "\n", + " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 0.770328 0.229672 \n", + "2024-03-01 06:15:00+00:00 0.793214 0.206786 \n", + "2024-03-01 06:30:00+00:00 0.833964 0.166036 \n", + "2024-03-01 06:45:00+00:00 0.862744 0.137256 \n", + "2024-03-01 07:00:00+00:00 0.858244 0.141756 \n", + "\n", + " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", + "timestamp ... \n", + "2024-03-01 06:00:00+00:00 0.853948 0.146052 ... \n", + "2024-03-01 06:15:00+00:00 0.855017 0.144983 ... \n", + "2024-03-01 06:30:00+00:00 0.841675 0.158325 ... \n", + "2024-03-01 06:45:00+00:00 0.423688 0.576312 ... \n", + "2024-03-01 07:00:00+00:00 0.828332 0.171668 ... \n", + "\n", + " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 0.438502 0.561498 \n", + "2024-03-01 06:15:00+00:00 0.127280 0.872720 \n", + "2024-03-01 06:30:00+00:00 0.109894 0.890106 \n", + "2024-03-01 06:45:00+00:00 0.117893 0.882107 \n", + "2024-03-01 07:00:00+00:00 0.115683 0.884317 \n", + "\n", + " load quantile_P05 quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 -23820000.0 -2.635957e+07 -2.558408e+07 \n", + "2024-03-01 06:15:00+00:00 NaN -2.763479e+07 -2.622804e+07 \n", + "2024-03-01 06:30:00+00:00 NaN -2.860721e+07 -2.760050e+07 \n", + "2024-03-01 06:45:00+00:00 NaN -2.972855e+07 -2.872363e+07 \n", + "2024-03-01 07:00:00+00:00 NaN -3.082963e+07 -2.989013e+07 \n", + "\n", + " quantile_P30 quantile_P50 quantile_P70 \\\n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 -2.448471e+07 -2.390916e+07 -2.351279e+07 \n", + "2024-03-01 06:15:00+00:00 -2.546636e+07 -2.520980e+07 -2.481184e+07 \n", + "2024-03-01 06:30:00+00:00 -2.660407e+07 -2.602701e+07 -2.557144e+07 \n", + "2024-03-01 06:45:00+00:00 -2.763870e+07 -2.716232e+07 -2.711700e+07 \n", + "2024-03-01 07:00:00+00:00 -2.899229e+07 -2.831311e+07 -2.772480e+07 \n", + "\n", + " quantile_P90 quantile_P95 \n", + "timestamp \n", + "2024-03-01 06:00:00+00:00 -2.248671e+07 -2.211594e+07 \n", + "2024-03-01 06:15:00+00:00 -2.364225e+07 -2.321880e+07 \n", + "2024-03-01 06:30:00+00:00 -2.470169e+07 -2.462576e+07 \n", + "2024-03-01 06:45:00+00:00 -2.652464e+07 -2.651125e+07 \n", + "2024-03-01 07:00:00+00:00 -2.708928e+07 -2.703521e+07 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(df_contribution2.head())\n", + "display(df_contribution.head())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5cc2844a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6b014bc8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 1224 files\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1224/1224 [00:46<00:00, 26.45it/s]\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Combined dataframe shape: (29376, 22)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 00:00:00+00:000.2488250.7511750.8114570.1885430.7757460.2242540.7967020.2032980.7373460.262654...0.1355130.864487-2.137667e+07-2.317569e+07-2.248770e+07-2.196953e+07-2.158473e+07-2.100265e+07-2.091442e+07-2.065124e+07
2024-03-01 00:15:00+00:000.0800040.9199960.5297130.4702870.7817720.2182280.7753140.2246860.7757290.224271...0.1477200.852280NaN-2.209275e+07-2.208461e+07-2.157246e+07-2.118342e+07-2.092382e+07-2.073507e+07-2.063586e+07
2024-03-01 00:30:00+00:000.4708160.5291840.1918370.8081630.8281760.1718240.7753140.2246860.7757290.224271...0.2423170.757683NaN-2.250475e+07-2.174137e+07-2.138064e+07-2.100023e+07-2.053325e+07-2.041650e+07-1.982233e+07
2024-03-01 00:45:00+00:000.0900270.9099730.1650050.8349950.5156460.4843540.8316680.1683320.6291620.370838...0.6789810.321019NaN-2.125884e+07-2.111755e+07-2.109466e+07-2.071542e+07-2.019039e+07-1.965634e+07-1.874860e+07
2024-03-01 01:00:00+00:000.0966000.9034000.6431520.3568480.8340690.1659310.8375540.1624460.5731100.426890...0.0898150.910185NaN-2.126173e+07-2.106740e+07-2.063920e+07-2.017966e+07-1.971798e+07-1.952271e+07-1.944001e+07
2024-03-01 01:15:00+00:000.0966000.9034000.4294090.5705910.8340690.1659310.8375540.1624460.6355960.364404...0.1724030.827597NaN-2.101299e+07-2.072796e+07-2.055497e+07-2.009874e+07-1.958135e+07-1.927191e+07-1.865684e+07
2024-03-01 01:30:00+00:000.0966000.9034000.4294090.5705910.8340690.1659310.8402650.1597350.4605110.539489...0.3052060.694794NaN-2.098769e+07-2.067334e+07-2.044858e+07-1.999029e+07-1.929075e+07-1.860416e+07-1.848513e+07
2024-03-01 01:45:00+00:000.0966000.9034000.1632830.8367170.2959130.7040870.8313440.1686560.3922390.607761...0.2253830.774617NaN-2.094209e+07-1.999452e+07-1.993032e+07-1.979889e+07-1.927314e+07-1.854875e+07-1.838815e+07
2024-03-01 02:00:00+00:000.0966000.9034000.4294090.5705910.3194310.6805690.8313440.1686560.3922390.607761...0.1484710.851529NaN-2.068070e+07-2.047176e+07-1.996984e+07-1.978139e+07-1.919273e+07-1.852332e+07-1.850626e+07
2024-03-01 02:15:00+00:000.0958050.9041950.4294090.5705910.6526790.3473210.8025930.1974070.6887270.311273...0.1484710.851529NaN-2.058423e+07-2.036005e+07-1.997906e+07-1.958525e+07-1.894833e+07-1.875233e+07-1.848577e+07
\n", + "

10 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.248825 0.751175 \n", + "2024-03-01 00:15:00+00:00 0.080004 0.919996 \n", + "2024-03-01 00:30:00+00:00 0.470816 0.529184 \n", + "2024-03-01 00:45:00+00:00 0.090027 0.909973 \n", + "2024-03-01 01:00:00+00:00 0.096600 0.903400 \n", + "2024-03-01 01:15:00+00:00 0.096600 0.903400 \n", + "2024-03-01 01:30:00+00:00 0.096600 0.903400 \n", + "2024-03-01 01:45:00+00:00 0.096600 0.903400 \n", + "2024-03-01 02:00:00+00:00 0.096600 0.903400 \n", + "2024-03-01 02:15:00+00:00 0.095805 0.904195 \n", + "\n", + " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.811457 0.188543 \n", + "2024-03-01 00:15:00+00:00 0.529713 0.470287 \n", + "2024-03-01 00:30:00+00:00 0.191837 0.808163 \n", + "2024-03-01 00:45:00+00:00 0.165005 0.834995 \n", + "2024-03-01 01:00:00+00:00 0.643152 0.356848 \n", + "2024-03-01 01:15:00+00:00 0.429409 0.570591 \n", + "2024-03-01 01:30:00+00:00 0.429409 0.570591 \n", + "2024-03-01 01:45:00+00:00 0.163283 0.836717 \n", + "2024-03-01 02:00:00+00:00 0.429409 0.570591 \n", + "2024-03-01 02:15:00+00:00 0.429409 0.570591 \n", + "\n", + " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.775746 0.224254 \n", + "2024-03-01 00:15:00+00:00 0.781772 0.218228 \n", + "2024-03-01 00:30:00+00:00 0.828176 0.171824 \n", + "2024-03-01 00:45:00+00:00 0.515646 0.484354 \n", + "2024-03-01 01:00:00+00:00 0.834069 0.165931 \n", + "2024-03-01 01:15:00+00:00 0.834069 0.165931 \n", + "2024-03-01 01:30:00+00:00 0.834069 0.165931 \n", + "2024-03-01 01:45:00+00:00 0.295913 0.704087 \n", + "2024-03-01 02:00:00+00:00 0.319431 0.680569 \n", + "2024-03-01 02:15:00+00:00 0.652679 0.347321 \n", + "\n", + " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.796702 0.203298 \n", + "2024-03-01 00:15:00+00:00 0.775314 0.224686 \n", + "2024-03-01 00:30:00+00:00 0.775314 0.224686 \n", + "2024-03-01 00:45:00+00:00 0.831668 0.168332 \n", + "2024-03-01 01:00:00+00:00 0.837554 0.162446 \n", + "2024-03-01 01:15:00+00:00 0.837554 0.162446 \n", + "2024-03-01 01:30:00+00:00 0.840265 0.159735 \n", + "2024-03-01 01:45:00+00:00 0.831344 0.168656 \n", + "2024-03-01 02:00:00+00:00 0.831344 0.168656 \n", + "2024-03-01 02:15:00+00:00 0.802593 0.197407 \n", + "\n", + " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", + "timestamp ... \n", + "2024-03-01 00:00:00+00:00 0.737346 0.262654 ... \n", + "2024-03-01 00:15:00+00:00 0.775729 0.224271 ... \n", + "2024-03-01 00:30:00+00:00 0.775729 0.224271 ... \n", + "2024-03-01 00:45:00+00:00 0.629162 0.370838 ... \n", + "2024-03-01 01:00:00+00:00 0.573110 0.426890 ... \n", + "2024-03-01 01:15:00+00:00 0.635596 0.364404 ... \n", + "2024-03-01 01:30:00+00:00 0.460511 0.539489 ... \n", + "2024-03-01 01:45:00+00:00 0.392239 0.607761 ... \n", + "2024-03-01 02:00:00+00:00 0.392239 0.607761 ... \n", + "2024-03-01 02:15:00+00:00 0.688727 0.311273 ... \n", + "\n", + " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 0.135513 0.864487 \n", + "2024-03-01 00:15:00+00:00 0.147720 0.852280 \n", + "2024-03-01 00:30:00+00:00 0.242317 0.757683 \n", + "2024-03-01 00:45:00+00:00 0.678981 0.321019 \n", + "2024-03-01 01:00:00+00:00 0.089815 0.910185 \n", + "2024-03-01 01:15:00+00:00 0.172403 0.827597 \n", + "2024-03-01 01:30:00+00:00 0.305206 0.694794 \n", + "2024-03-01 01:45:00+00:00 0.225383 0.774617 \n", + "2024-03-01 02:00:00+00:00 0.148471 0.851529 \n", + "2024-03-01 02:15:00+00:00 0.148471 0.851529 \n", + "\n", + " load quantile_P05 quantile_P10 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.137667e+07 -2.317569e+07 -2.248770e+07 \n", + "2024-03-01 00:15:00+00:00 NaN -2.209275e+07 -2.208461e+07 \n", + "2024-03-01 00:30:00+00:00 NaN -2.250475e+07 -2.174137e+07 \n", + "2024-03-01 00:45:00+00:00 NaN -2.125884e+07 -2.111755e+07 \n", + "2024-03-01 01:00:00+00:00 NaN -2.126173e+07 -2.106740e+07 \n", + "2024-03-01 01:15:00+00:00 NaN -2.101299e+07 -2.072796e+07 \n", + "2024-03-01 01:30:00+00:00 NaN -2.098769e+07 -2.067334e+07 \n", + "2024-03-01 01:45:00+00:00 NaN -2.094209e+07 -1.999452e+07 \n", + "2024-03-01 02:00:00+00:00 NaN -2.068070e+07 -2.047176e+07 \n", + "2024-03-01 02:15:00+00:00 NaN -2.058423e+07 -2.036005e+07 \n", + "\n", + " quantile_P30 quantile_P50 quantile_P70 \\\n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.196953e+07 -2.158473e+07 -2.100265e+07 \n", + "2024-03-01 00:15:00+00:00 -2.157246e+07 -2.118342e+07 -2.092382e+07 \n", + "2024-03-01 00:30:00+00:00 -2.138064e+07 -2.100023e+07 -2.053325e+07 \n", + "2024-03-01 00:45:00+00:00 -2.109466e+07 -2.071542e+07 -2.019039e+07 \n", + "2024-03-01 01:00:00+00:00 -2.063920e+07 -2.017966e+07 -1.971798e+07 \n", + "2024-03-01 01:15:00+00:00 -2.055497e+07 -2.009874e+07 -1.958135e+07 \n", + "2024-03-01 01:30:00+00:00 -2.044858e+07 -1.999029e+07 -1.929075e+07 \n", + "2024-03-01 01:45:00+00:00 -1.993032e+07 -1.979889e+07 -1.927314e+07 \n", + "2024-03-01 02:00:00+00:00 -1.996984e+07 -1.978139e+07 -1.919273e+07 \n", + "2024-03-01 02:15:00+00:00 -1.997906e+07 -1.958525e+07 -1.894833e+07 \n", + "\n", + " quantile_P90 quantile_P95 \n", + "timestamp \n", + "2024-03-01 00:00:00+00:00 -2.091442e+07 -2.065124e+07 \n", + "2024-03-01 00:15:00+00:00 -2.073507e+07 -2.063586e+07 \n", + "2024-03-01 00:30:00+00:00 -2.041650e+07 -1.982233e+07 \n", + "2024-03-01 00:45:00+00:00 -1.965634e+07 -1.874860e+07 \n", + "2024-03-01 01:00:00+00:00 -1.952271e+07 -1.944001e+07 \n", + "2024-03-01 01:15:00+00:00 -1.927191e+07 -1.865684e+07 \n", + "2024-03-01 01:30:00+00:00 -1.860416e+07 -1.848513e+07 \n", + "2024-03-01 01:45:00+00:00 -1.854875e+07 -1.838815e+07 \n", + "2024-03-01 02:00:00+00:00 -1.852332e+07 -1.850626e+07 \n", + "2024-03-01 02:15:00+00:00 -1.875233e+07 -1.848577e+07 \n", + "\n", + "[10 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-12-31 21:30:00+00:000.9395260.0604740.9155380.0844620.7959210.2040790.7454350.2545650.4425740.557426...0.8867570.113243NaN-2.273370e+07-2.227941e+07-2.161480e+07-2.095055e+07-2.082273e+07-2.000162e+07-1.894876e+07
2024-12-31 21:45:00+00:000.9425230.0574770.9259880.0740120.7961140.2038860.7454350.2545650.5447490.455251...0.8908110.109189NaN-2.230843e+07-2.182914e+07-2.125863e+07-2.061431e+07-2.021932e+07-1.937308e+07-1.861693e+07
2024-12-31 22:00:00+00:000.1249570.8750430.2699510.7300490.6745740.3254260.7502970.2497030.5589390.441061...0.8867570.113243NaN-2.282250e+07-2.207064e+07-2.154538e+07-2.100245e+07-2.062644e+07-1.952722e+07-1.922988e+07
2024-12-31 22:15:00+00:000.1236610.8763390.3958110.6041890.7635740.2364260.7502970.2497030.6103800.389620...0.8867570.113243NaN-2.277797e+07-2.202767e+07-2.129449e+07-2.074317e+07-2.036547e+07-1.922563e+07-1.895426e+07
2024-12-31 22:30:00+00:000.1690190.8309810.3958110.6041890.7997180.2002820.7502970.2497030.6103800.389620...0.8908110.109189NaN-2.288190e+07-2.198203e+07-2.113597e+07-2.059848e+07-2.022853e+07-1.928894e+07-1.879247e+07
2024-12-31 22:45:00+00:000.5734140.4265860.6919970.3080030.8364110.1635890.7502970.2497030.6638710.336129...0.8887060.111294NaN-2.297925e+07-2.195887e+07-2.092103e+07-2.037274e+07-1.998042e+07-1.879439e+07-1.855492e+07
2024-12-31 23:00:00+00:000.1366740.8633260.5835700.4164300.8317760.1682240.7502970.2497030.6103800.389620...0.8909380.109062NaN-2.278653e+07-2.227599e+07-2.117868e+07-2.064982e+07-2.022001e+07-1.903018e+07-1.886584e+07
2024-12-31 23:15:00+00:000.3345920.6654080.6919970.3080030.8364110.1635890.7019710.2980290.7219740.278026...0.8887060.111294NaN-2.290655e+07-2.199052e+07-2.079653e+07-2.032965e+07-1.923286e+07-1.873750e+07-1.853917e+07
2024-12-31 23:30:00+00:000.7203120.2796880.8990150.1009850.8021260.1978740.5171450.4828550.7116600.288340...0.8795430.120457NaN-2.297595e+07-2.184417e+07-2.069840e+07-2.034597e+07-1.889343e+07-1.838063e+07-1.830427e+07
2024-12-31 23:45:00+00:000.6176670.3823330.8438620.1561380.7960460.2039540.5437440.4562560.7116600.288340...0.8662030.133797NaN-2.255517e+07-2.142616e+07-2.021394e+07-1.989196e+07-1.843501e+07-1.807584e+07-1.765399e+07
\n", + "

10 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 0.939526 0.060474 \n", + "2024-12-31 21:45:00+00:00 0.942523 0.057477 \n", + "2024-12-31 22:00:00+00:00 0.124957 0.875043 \n", + "2024-12-31 22:15:00+00:00 0.123661 0.876339 \n", + "2024-12-31 22:30:00+00:00 0.169019 0.830981 \n", + "2024-12-31 22:45:00+00:00 0.573414 0.426586 \n", + "2024-12-31 23:00:00+00:00 0.136674 0.863326 \n", + "2024-12-31 23:15:00+00:00 0.334592 0.665408 \n", + "2024-12-31 23:30:00+00:00 0.720312 0.279688 \n", + "2024-12-31 23:45:00+00:00 0.617667 0.382333 \n", + "\n", + " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 0.915538 0.084462 \n", + "2024-12-31 21:45:00+00:00 0.925988 0.074012 \n", + "2024-12-31 22:00:00+00:00 0.269951 0.730049 \n", + "2024-12-31 22:15:00+00:00 0.395811 0.604189 \n", + "2024-12-31 22:30:00+00:00 0.395811 0.604189 \n", + "2024-12-31 22:45:00+00:00 0.691997 0.308003 \n", + "2024-12-31 23:00:00+00:00 0.583570 0.416430 \n", + "2024-12-31 23:15:00+00:00 0.691997 0.308003 \n", + "2024-12-31 23:30:00+00:00 0.899015 0.100985 \n", + "2024-12-31 23:45:00+00:00 0.843862 0.156138 \n", + "\n", + " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 0.795921 0.204079 \n", + "2024-12-31 21:45:00+00:00 0.796114 0.203886 \n", + "2024-12-31 22:00:00+00:00 0.674574 0.325426 \n", + "2024-12-31 22:15:00+00:00 0.763574 0.236426 \n", + "2024-12-31 22:30:00+00:00 0.799718 0.200282 \n", + "2024-12-31 22:45:00+00:00 0.836411 0.163589 \n", + "2024-12-31 23:00:00+00:00 0.831776 0.168224 \n", + "2024-12-31 23:15:00+00:00 0.836411 0.163589 \n", + "2024-12-31 23:30:00+00:00 0.802126 0.197874 \n", + "2024-12-31 23:45:00+00:00 0.796046 0.203954 \n", + "\n", + " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 0.745435 0.254565 \n", + "2024-12-31 21:45:00+00:00 0.745435 0.254565 \n", + "2024-12-31 22:00:00+00:00 0.750297 0.249703 \n", + "2024-12-31 22:15:00+00:00 0.750297 0.249703 \n", + "2024-12-31 22:30:00+00:00 0.750297 0.249703 \n", + "2024-12-31 22:45:00+00:00 0.750297 0.249703 \n", + "2024-12-31 23:00:00+00:00 0.750297 0.249703 \n", + "2024-12-31 23:15:00+00:00 0.701971 0.298029 \n", + "2024-12-31 23:30:00+00:00 0.517145 0.482855 \n", + "2024-12-31 23:45:00+00:00 0.543744 0.456256 \n", + "\n", + " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", + "timestamp ... \n", + "2024-12-31 21:30:00+00:00 0.442574 0.557426 ... \n", + "2024-12-31 21:45:00+00:00 0.544749 0.455251 ... \n", + "2024-12-31 22:00:00+00:00 0.558939 0.441061 ... \n", + "2024-12-31 22:15:00+00:00 0.610380 0.389620 ... \n", + "2024-12-31 22:30:00+00:00 0.610380 0.389620 ... \n", + "2024-12-31 22:45:00+00:00 0.663871 0.336129 ... \n", + "2024-12-31 23:00:00+00:00 0.610380 0.389620 ... \n", + "2024-12-31 23:15:00+00:00 0.721974 0.278026 ... \n", + "2024-12-31 23:30:00+00:00 0.711660 0.288340 ... \n", + "2024-12-31 23:45:00+00:00 0.711660 0.288340 ... \n", + "\n", + " gblinear_quantile_P95 lgbm_quantile_P95 load \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 0.886757 0.113243 NaN \n", + "2024-12-31 21:45:00+00:00 0.890811 0.109189 NaN \n", + "2024-12-31 22:00:00+00:00 0.886757 0.113243 NaN \n", + "2024-12-31 22:15:00+00:00 0.886757 0.113243 NaN \n", + "2024-12-31 22:30:00+00:00 0.890811 0.109189 NaN \n", + "2024-12-31 22:45:00+00:00 0.888706 0.111294 NaN \n", + "2024-12-31 23:00:00+00:00 0.890938 0.109062 NaN \n", + "2024-12-31 23:15:00+00:00 0.888706 0.111294 NaN \n", + "2024-12-31 23:30:00+00:00 0.879543 0.120457 NaN \n", + "2024-12-31 23:45:00+00:00 0.866203 0.133797 NaN \n", + "\n", + " quantile_P05 quantile_P10 quantile_P30 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 -2.273370e+07 -2.227941e+07 -2.161480e+07 \n", + "2024-12-31 21:45:00+00:00 -2.230843e+07 -2.182914e+07 -2.125863e+07 \n", + "2024-12-31 22:00:00+00:00 -2.282250e+07 -2.207064e+07 -2.154538e+07 \n", + "2024-12-31 22:15:00+00:00 -2.277797e+07 -2.202767e+07 -2.129449e+07 \n", + "2024-12-31 22:30:00+00:00 -2.288190e+07 -2.198203e+07 -2.113597e+07 \n", + "2024-12-31 22:45:00+00:00 -2.297925e+07 -2.195887e+07 -2.092103e+07 \n", + "2024-12-31 23:00:00+00:00 -2.278653e+07 -2.227599e+07 -2.117868e+07 \n", + "2024-12-31 23:15:00+00:00 -2.290655e+07 -2.199052e+07 -2.079653e+07 \n", + "2024-12-31 23:30:00+00:00 -2.297595e+07 -2.184417e+07 -2.069840e+07 \n", + "2024-12-31 23:45:00+00:00 -2.255517e+07 -2.142616e+07 -2.021394e+07 \n", + "\n", + " quantile_P50 quantile_P70 quantile_P90 \\\n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 -2.095055e+07 -2.082273e+07 -2.000162e+07 \n", + "2024-12-31 21:45:00+00:00 -2.061431e+07 -2.021932e+07 -1.937308e+07 \n", + "2024-12-31 22:00:00+00:00 -2.100245e+07 -2.062644e+07 -1.952722e+07 \n", + "2024-12-31 22:15:00+00:00 -2.074317e+07 -2.036547e+07 -1.922563e+07 \n", + "2024-12-31 22:30:00+00:00 -2.059848e+07 -2.022853e+07 -1.928894e+07 \n", + "2024-12-31 22:45:00+00:00 -2.037274e+07 -1.998042e+07 -1.879439e+07 \n", + "2024-12-31 23:00:00+00:00 -2.064982e+07 -2.022001e+07 -1.903018e+07 \n", + "2024-12-31 23:15:00+00:00 -2.032965e+07 -1.923286e+07 -1.873750e+07 \n", + "2024-12-31 23:30:00+00:00 -2.034597e+07 -1.889343e+07 -1.838063e+07 \n", + "2024-12-31 23:45:00+00:00 -1.989196e+07 -1.843501e+07 -1.807584e+07 \n", + "\n", + " quantile_P95 \n", + "timestamp \n", + "2024-12-31 21:30:00+00:00 -1.894876e+07 \n", + "2024-12-31 21:45:00+00:00 -1.861693e+07 \n", + "2024-12-31 22:00:00+00:00 -1.922988e+07 \n", + "2024-12-31 22:15:00+00:00 -1.895426e+07 \n", + "2024-12-31 22:30:00+00:00 -1.879247e+07 \n", + "2024-12-31 22:45:00+00:00 -1.855492e+07 \n", + "2024-12-31 23:00:00+00:00 -1.886584e+07 \n", + "2024-12-31 23:15:00+00:00 -1.853917e+07 \n", + "2024-12-31 23:30:00+00:00 -1.830427e+07 \n", + "2024-12-31 23:45:00+00:00 -1.765399e+07 \n", + "\n", + "[10 rows x 22 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pathlib import Path\n", + "from tqdm import tqdm\n", + "# Define the folder path\n", + "folder_path = Path(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn\")\n", + "\n", + "# Get all parquet files in the folder\n", + "parquet_files = sorted(folder_path.glob(\"contrib_*_predict.parquet\"))\n", + "\n", + "print(f\"Found {len(parquet_files)} files\")\n", + "\n", + "# Read and concatenate first 6 hours (24 rows) from each file\n", + "df_list = []\n", + "for file in tqdm(parquet_files):\n", + " df_temp = pd.read_parquet(file)\n", + " df_6h = df_temp.head(24) # First 6 hours at 15-min intervals\n", + " df_list.append(df_6h)\n", + "\n", + "# Concatenate all into one dataframe\n", + "df_combined = pd.concat(df_list, axis=0, ignore_index=False)\n", + "\n", + "print(f\"Combined dataframe shape: {df_combined.shape}\")\n", + "display(df_combined.head(10))\n", + "display(df_combined.tail(10))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5febfec5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "992a7f53", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_model_contributions(df, start_date=None, end_date=None, quantiles=None):\n", + " \"\"\"\n", + " Plot comparison of GBLinear and LGBM contributions with quantile dropdown filter using Plotly.\n", + "\n", + " Args:\n", + " df: DataFrame with model contribution columns\n", + " start_date: Start date for filtering (str or datetime). If None, uses all data.\n", + " end_date: End date for filtering (str or datetime). If None, uses all data.\n", + " quantiles: List of quantile names to include in filter (e.g., ['P05', 'P50']). If None, uses all available.\n", + " \"\"\"\n", + " # Filter dataframe by date range if specified\n", + " df_plot = df.copy()\n", + " if start_date is not None:\n", + " start_dt = pd.to_datetime(start_date)\n", + " if df_plot.index.tz is not None and start_dt.tz is None:\n", + " start_dt = start_dt.tz_localize('UTC')\n", + " df_plot = df_plot[df_plot.index >= start_dt]\n", + " if end_date is not None:\n", + " end_dt = pd.to_datetime(end_date)\n", + " if df_plot.index.tz is not None and end_dt.tz is None:\n", + " end_dt = end_dt.tz_localize('UTC')\n", + " df_plot = df_plot[df_plot.index <= end_dt]\n", + "\n", + " # Get quantiles to plot\n", + " if quantiles is None:\n", + " quantiles = ['P05', 'P10', 'P30', 'P50', 'P70', 'P90', 'P95']\n", + "\n", + " # Create figure\n", + " fig = go.Figure()\n", + "\n", + " # Add traces for each quantile\n", + " for quantile in quantiles:\n", + " gblinear_col = f'gblinear_quantile_{quantile}'\n", + " lgbm_col = f'lgbm_quantile_{quantile}'\n", + "\n", + " # Add GBLinear trace\n", + " fig.add_trace(\n", + " go.Scatter(\n", + " x=df_plot.index,\n", + " y=df_plot[gblinear_col],\n", + " name=f'GBLinear - {quantile}',\n", + " mode='lines',\n", + " line=dict(width=2, color='blue'),\n", + " opacity=0.7,\n", + " visible=(quantile == 'P50') # Only P50 visible by default\n", + " )\n", + " )\n", + "\n", + " # Add LGBM trace\n", + " fig.add_trace(\n", + " go.Scatter(\n", + " x=df_plot.index,\n", + " y=df_plot[lgbm_col],\n", + " name=f'LGBM - {quantile}',\n", + " mode='lines',\n", + " line=dict(width=2, color='#ff7f0e'), # high-contrast orange for LGBM\n", + " opacity=0.85,\n", + " visible=(quantile == 'P50') # Only P50 visible by default\n", + " )\n", + " )\n", + "\n", + " # Make the GBLinear trace (added just before) explicitly high-contrast too\n", + " # fig.data[-2] refers to the GBLinear trace we added immediately prior\n", + " if len(fig.data) >= 2:\n", + " fig.data[-2].update(line=dict(color='#1f77b4', width=2), opacity=0.85) # contrasting blue for GBLinear\n", + "\n", + " # Create dropdown buttons\n", + " buttons = []\n", + " for i, quantile in enumerate(quantiles):\n", + " # Create visibility list - show only the selected quantile (both GBLinear and LGBM)\n", + " visible = [False] * (len(quantiles) * 2)\n", + " visible[i * 2] = True # GBLinear for this quantile\n", + " visible[i * 2 + 1] = True # LGBM for this quantile\n", + "\n", + " buttons.append(\n", + " dict(\n", + " label=quantile,\n", + " method='update',\n", + " args=[\n", + " {'visible': visible},\n", + " {'title.text': f'Model Contributions - Quantile {quantile}'}\n", + " ]\n", + " )\n", + " )\n", + "\n", + " # Update layout with dropdown menu\n", + " # determine default active quantile (P50 if available)\n", + " active_idx = quantiles.index('P50') if 'P50' in quantiles else 0\n", + "\n", + " fig.update_layout(\n", + " legend=dict(\n", + " orientation='v',\n", + " x=1.02,\n", + " xanchor='left',\n", + " y=1.0,\n", + " yanchor='top'\n", + " ),\n", + " updatemenus=[\n", + " dict(\n", + " type='buttons',\n", + " buttons=buttons,\n", + " direction='down',\n", + " showactive=True,\n", + " active=active_idx, # Set the default active button\n", + " x=1.02,\n", + " xanchor='left',\n", + " y=0.88, # place beneath the legend\n", + " yanchor='top',\n", + " pad={'r': 10, 't': 10},\n", + " bgcolor='lightgray',\n", + " bordercolor='gray',\n", + " borderwidth=1,\n", + " )\n", + " ],\n", + " title=f\"Model Contributions - Quantile {quantiles[active_idx]}\",\n", + " xaxis_title='Timestamp',\n", + " yaxis_title='Contribution',\n", + " hovermode='x unified',\n", + " height=600,\n", + " showlegend=True\n", + " )\n", + "\n", + " fig.show()\n", + "\n", + "# Example usage - plot all data\n", + "# plot_model_contributions(df_combined)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "880c6150", + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Plot specific date range\n", + "# plot_model_contributions(df_combined, start_date='2024-04-21', end_date='2024-04-30')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "56c1846f", + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Plot specific quantiles only\n", + "# plot_model_contributions(df_combined, start_date='2024-11-17', end_date='2024-11-29')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abef9b04", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 9c6368d4c89cbcf3bc43e0497549c428bba5c1ce Mon Sep 17 00:00:00 2001 From: Guilly Kolkman Date: Thu, 18 Dec 2025 11:36:41 +0100 Subject: [PATCH 4/5] FixedD-1T06:00 sanitization for handling on windows paths Signed-off-by: Guilly Kolkman --- .../src/openstef_beam/evaluation/models/report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py b/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py index d16d66cef..f466dc713 100644 --- a/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py +++ b/packages/openstef-beam/src/openstef_beam/evaluation/models/report.py @@ -47,8 +47,6 @@ def to_parquet(self, path: Path): Args: path: Directory where to save the report data. """ - # Sanitize path by replacing colons (invalid on Windows) - path = Path(str(path).replace(":", "_")) path.mkdir(parents=True, exist_ok=True) (path / "metrics.json").write_bytes(TypeAdapter(list[SubsetMetric]).dump_json(self.metrics)) self.subset.to_parquet(path / "subset.parquet") @@ -66,7 +64,9 @@ def read_parquet(cls, path: Path) -> Self: metrics = TypeAdapter[list[SubsetMetric]](list[SubsetMetric]).validate_json( (path / "metrics.json").read_bytes() ) - filtering = TypeAdapter[Filtering](Filtering).validate_python(path.name) + # Reverse sanitization: convert underscores back to colons for parsing + filtering_str = path.name.replace("_", ":") + filtering = TypeAdapter[Filtering](Filtering).validate_python(filtering_str) subset = ForecastDataset.read_parquet(path / "subset.parquet") return cls(filtering=filtering, subset=subset, metrics=metrics) From 3cfc37cd8d4fd42ad6e9dc1890244a0abe3a4373 Mon Sep 17 00:00:00 2001 From: Guilly Kolkman Date: Thu, 18 Dec 2025 11:39:04 +0100 Subject: [PATCH 5/5] Deleted notebook for original plotting code Signed-off-by: Guilly Kolkman --- examples/model_contributions.ipynb | 1751 ---------------------------- 1 file changed, 1751 deletions(-) delete mode 100644 examples/model_contributions.ipynb diff --git a/examples/model_contributions.ipynb b/examples/model_contributions.ipynb deleted file mode 100644 index d94d67eec..000000000 --- a/examples/model_contributions.ipynb +++ /dev/null @@ -1,1751 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "b52202e3", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import plotly.graph_objects as go\n", - "from plotly.subplots import make_subplots\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2ca96a4f", - "metadata": {}, - "outputs": [], - "source": [ - "# df = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/Ensemble_lgbm_gblinear_learned_weights_lgbm/backtest/transformer/OS Apeldoorn/predictions.parquet\")\n", - "# print(df.head())\n", - "# df_focus = df[[\"quantile_P05\"]]\n", - "\n", - "# start = \"2024-03-16\"\n", - "# end = \"2024-03-23\"\n", - "# df_week = df_focus.loc[start:end]\n", - "\n", - "# df_week.plot(figsize=(14, 6))\n", - "\n", - "# plt.title(\"Model Contributions\")\n", - "# plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "52e7d666", - "metadata": {}, - "outputs": [], - "source": [ - "# display(df)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "e4dfe0a3", - "metadata": {}, - "outputs": [], - "source": [ - "df_contribution = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn/contrib_20240301060000_predict.parquet\")\n", - "df_contribution2 = pd.read_parquet(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn/contrib_20240301000000_predict.parquet\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "659ade83", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 00:00:00+00:000.2488250.7511750.8114570.1885430.7757460.2242540.7967020.2032980.7373460.262654...0.1355130.864487-2.137667e+07-2.317569e+07-2.248770e+07-2.196953e+07-2.158473e+07-2.100265e+07-2.091442e+07-2.065124e+07
2024-03-01 00:15:00+00:000.0800040.9199960.5297130.4702870.7817720.2182280.7753140.2246860.7757290.224271...0.1477200.852280NaN-2.209275e+07-2.208461e+07-2.157246e+07-2.118342e+07-2.092382e+07-2.073507e+07-2.063586e+07
2024-03-01 00:30:00+00:000.4708160.5291840.1918370.8081630.8281760.1718240.7753140.2246860.7757290.224271...0.2423170.757683NaN-2.250475e+07-2.174137e+07-2.138064e+07-2.100023e+07-2.053325e+07-2.041650e+07-1.982233e+07
2024-03-01 00:45:00+00:000.0900270.9099730.1650050.8349950.5156460.4843540.8316680.1683320.6291620.370838...0.6789810.321019NaN-2.125884e+07-2.111755e+07-2.109466e+07-2.071542e+07-2.019039e+07-1.965634e+07-1.874860e+07
2024-03-01 01:00:00+00:000.0966000.9034000.6431520.3568480.8340690.1659310.8375540.1624460.5731100.426890...0.0898150.910185NaN-2.126173e+07-2.106740e+07-2.063920e+07-2.017966e+07-1.971798e+07-1.952271e+07-1.944001e+07
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.248825 0.751175 \n", - "2024-03-01 00:15:00+00:00 0.080004 0.919996 \n", - "2024-03-01 00:30:00+00:00 0.470816 0.529184 \n", - "2024-03-01 00:45:00+00:00 0.090027 0.909973 \n", - "2024-03-01 01:00:00+00:00 0.096600 0.903400 \n", - "\n", - " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.811457 0.188543 \n", - "2024-03-01 00:15:00+00:00 0.529713 0.470287 \n", - "2024-03-01 00:30:00+00:00 0.191837 0.808163 \n", - "2024-03-01 00:45:00+00:00 0.165005 0.834995 \n", - "2024-03-01 01:00:00+00:00 0.643152 0.356848 \n", - "\n", - " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.775746 0.224254 \n", - "2024-03-01 00:15:00+00:00 0.781772 0.218228 \n", - "2024-03-01 00:30:00+00:00 0.828176 0.171824 \n", - "2024-03-01 00:45:00+00:00 0.515646 0.484354 \n", - "2024-03-01 01:00:00+00:00 0.834069 0.165931 \n", - "\n", - " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.796702 0.203298 \n", - "2024-03-01 00:15:00+00:00 0.775314 0.224686 \n", - "2024-03-01 00:30:00+00:00 0.775314 0.224686 \n", - "2024-03-01 00:45:00+00:00 0.831668 0.168332 \n", - "2024-03-01 01:00:00+00:00 0.837554 0.162446 \n", - "\n", - " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", - "timestamp ... \n", - "2024-03-01 00:00:00+00:00 0.737346 0.262654 ... \n", - "2024-03-01 00:15:00+00:00 0.775729 0.224271 ... \n", - "2024-03-01 00:30:00+00:00 0.775729 0.224271 ... \n", - "2024-03-01 00:45:00+00:00 0.629162 0.370838 ... \n", - "2024-03-01 01:00:00+00:00 0.573110 0.426890 ... \n", - "\n", - " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.135513 0.864487 \n", - "2024-03-01 00:15:00+00:00 0.147720 0.852280 \n", - "2024-03-01 00:30:00+00:00 0.242317 0.757683 \n", - "2024-03-01 00:45:00+00:00 0.678981 0.321019 \n", - "2024-03-01 01:00:00+00:00 0.089815 0.910185 \n", - "\n", - " load quantile_P05 quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.137667e+07 -2.317569e+07 -2.248770e+07 \n", - "2024-03-01 00:15:00+00:00 NaN -2.209275e+07 -2.208461e+07 \n", - "2024-03-01 00:30:00+00:00 NaN -2.250475e+07 -2.174137e+07 \n", - "2024-03-01 00:45:00+00:00 NaN -2.125884e+07 -2.111755e+07 \n", - "2024-03-01 01:00:00+00:00 NaN -2.126173e+07 -2.106740e+07 \n", - "\n", - " quantile_P30 quantile_P50 quantile_P70 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.196953e+07 -2.158473e+07 -2.100265e+07 \n", - "2024-03-01 00:15:00+00:00 -2.157246e+07 -2.118342e+07 -2.092382e+07 \n", - "2024-03-01 00:30:00+00:00 -2.138064e+07 -2.100023e+07 -2.053325e+07 \n", - "2024-03-01 00:45:00+00:00 -2.109466e+07 -2.071542e+07 -2.019039e+07 \n", - "2024-03-01 01:00:00+00:00 -2.063920e+07 -2.017966e+07 -1.971798e+07 \n", - "\n", - " quantile_P90 quantile_P95 \n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.091442e+07 -2.065124e+07 \n", - "2024-03-01 00:15:00+00:00 -2.073507e+07 -2.063586e+07 \n", - "2024-03-01 00:30:00+00:00 -2.041650e+07 -1.982233e+07 \n", - "2024-03-01 00:45:00+00:00 -1.965634e+07 -1.874860e+07 \n", - "2024-03-01 01:00:00+00:00 -1.952271e+07 -1.944001e+07 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 06:00:00+00:000.6830180.3169820.9307990.0692010.8481510.1518490.7703280.2296720.8539480.146052...0.4385020.561498-23820000.0-2.635957e+07-2.558408e+07-2.448471e+07-2.390916e+07-2.351279e+07-2.248671e+07-2.211594e+07
2024-03-01 06:15:00+00:000.5213240.4786760.4236980.5763020.6382140.3617860.7932140.2067860.8550170.144983...0.1272800.872720NaN-2.763479e+07-2.622804e+07-2.546636e+07-2.520980e+07-2.481184e+07-2.364225e+07-2.321880e+07
2024-03-01 06:30:00+00:000.8591640.1408360.7824160.2175840.8953740.1046260.8339640.1660360.8416750.158325...0.1098940.890106NaN-2.860721e+07-2.760050e+07-2.660407e+07-2.602701e+07-2.557144e+07-2.470169e+07-2.462576e+07
2024-03-01 06:45:00+00:000.9488980.0511020.9099140.0900860.8268100.1731900.8627440.1372560.4236880.576312...0.1178930.882107NaN-2.972855e+07-2.872363e+07-2.763870e+07-2.716232e+07-2.711700e+07-2.652464e+07-2.651125e+07
2024-03-01 07:00:00+00:000.9489840.0510160.9250050.0749950.8517400.1482600.8582440.1417560.8283320.171668...0.1156830.884317NaN-3.082963e+07-2.989013e+07-2.899229e+07-2.831311e+07-2.772480e+07-2.708928e+07-2.703521e+07
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 0.683018 0.316982 \n", - "2024-03-01 06:15:00+00:00 0.521324 0.478676 \n", - "2024-03-01 06:30:00+00:00 0.859164 0.140836 \n", - "2024-03-01 06:45:00+00:00 0.948898 0.051102 \n", - "2024-03-01 07:00:00+00:00 0.948984 0.051016 \n", - "\n", - " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 0.930799 0.069201 \n", - "2024-03-01 06:15:00+00:00 0.423698 0.576302 \n", - "2024-03-01 06:30:00+00:00 0.782416 0.217584 \n", - "2024-03-01 06:45:00+00:00 0.909914 0.090086 \n", - "2024-03-01 07:00:00+00:00 0.925005 0.074995 \n", - "\n", - " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 0.848151 0.151849 \n", - "2024-03-01 06:15:00+00:00 0.638214 0.361786 \n", - "2024-03-01 06:30:00+00:00 0.895374 0.104626 \n", - "2024-03-01 06:45:00+00:00 0.826810 0.173190 \n", - "2024-03-01 07:00:00+00:00 0.851740 0.148260 \n", - "\n", - " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 0.770328 0.229672 \n", - "2024-03-01 06:15:00+00:00 0.793214 0.206786 \n", - "2024-03-01 06:30:00+00:00 0.833964 0.166036 \n", - "2024-03-01 06:45:00+00:00 0.862744 0.137256 \n", - "2024-03-01 07:00:00+00:00 0.858244 0.141756 \n", - "\n", - " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", - "timestamp ... \n", - "2024-03-01 06:00:00+00:00 0.853948 0.146052 ... \n", - "2024-03-01 06:15:00+00:00 0.855017 0.144983 ... \n", - "2024-03-01 06:30:00+00:00 0.841675 0.158325 ... \n", - "2024-03-01 06:45:00+00:00 0.423688 0.576312 ... \n", - "2024-03-01 07:00:00+00:00 0.828332 0.171668 ... \n", - "\n", - " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 0.438502 0.561498 \n", - "2024-03-01 06:15:00+00:00 0.127280 0.872720 \n", - "2024-03-01 06:30:00+00:00 0.109894 0.890106 \n", - "2024-03-01 06:45:00+00:00 0.117893 0.882107 \n", - "2024-03-01 07:00:00+00:00 0.115683 0.884317 \n", - "\n", - " load quantile_P05 quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 -23820000.0 -2.635957e+07 -2.558408e+07 \n", - "2024-03-01 06:15:00+00:00 NaN -2.763479e+07 -2.622804e+07 \n", - "2024-03-01 06:30:00+00:00 NaN -2.860721e+07 -2.760050e+07 \n", - "2024-03-01 06:45:00+00:00 NaN -2.972855e+07 -2.872363e+07 \n", - "2024-03-01 07:00:00+00:00 NaN -3.082963e+07 -2.989013e+07 \n", - "\n", - " quantile_P30 quantile_P50 quantile_P70 \\\n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 -2.448471e+07 -2.390916e+07 -2.351279e+07 \n", - "2024-03-01 06:15:00+00:00 -2.546636e+07 -2.520980e+07 -2.481184e+07 \n", - "2024-03-01 06:30:00+00:00 -2.660407e+07 -2.602701e+07 -2.557144e+07 \n", - "2024-03-01 06:45:00+00:00 -2.763870e+07 -2.716232e+07 -2.711700e+07 \n", - "2024-03-01 07:00:00+00:00 -2.899229e+07 -2.831311e+07 -2.772480e+07 \n", - "\n", - " quantile_P90 quantile_P95 \n", - "timestamp \n", - "2024-03-01 06:00:00+00:00 -2.248671e+07 -2.211594e+07 \n", - "2024-03-01 06:15:00+00:00 -2.364225e+07 -2.321880e+07 \n", - "2024-03-01 06:30:00+00:00 -2.470169e+07 -2.462576e+07 \n", - "2024-03-01 06:45:00+00:00 -2.652464e+07 -2.651125e+07 \n", - "2024-03-01 07:00:00+00:00 -2.708928e+07 -2.703521e+07 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "display(df_contribution2.head())\n", - "display(df_contribution.head())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5cc2844a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "6b014bc8", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 1224 files\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 1224/1224 [00:46<00:00, 26.45it/s]\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Combined dataframe shape: (29376, 22)\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-03-01 00:00:00+00:000.2488250.7511750.8114570.1885430.7757460.2242540.7967020.2032980.7373460.262654...0.1355130.864487-2.137667e+07-2.317569e+07-2.248770e+07-2.196953e+07-2.158473e+07-2.100265e+07-2.091442e+07-2.065124e+07
2024-03-01 00:15:00+00:000.0800040.9199960.5297130.4702870.7817720.2182280.7753140.2246860.7757290.224271...0.1477200.852280NaN-2.209275e+07-2.208461e+07-2.157246e+07-2.118342e+07-2.092382e+07-2.073507e+07-2.063586e+07
2024-03-01 00:30:00+00:000.4708160.5291840.1918370.8081630.8281760.1718240.7753140.2246860.7757290.224271...0.2423170.757683NaN-2.250475e+07-2.174137e+07-2.138064e+07-2.100023e+07-2.053325e+07-2.041650e+07-1.982233e+07
2024-03-01 00:45:00+00:000.0900270.9099730.1650050.8349950.5156460.4843540.8316680.1683320.6291620.370838...0.6789810.321019NaN-2.125884e+07-2.111755e+07-2.109466e+07-2.071542e+07-2.019039e+07-1.965634e+07-1.874860e+07
2024-03-01 01:00:00+00:000.0966000.9034000.6431520.3568480.8340690.1659310.8375540.1624460.5731100.426890...0.0898150.910185NaN-2.126173e+07-2.106740e+07-2.063920e+07-2.017966e+07-1.971798e+07-1.952271e+07-1.944001e+07
2024-03-01 01:15:00+00:000.0966000.9034000.4294090.5705910.8340690.1659310.8375540.1624460.6355960.364404...0.1724030.827597NaN-2.101299e+07-2.072796e+07-2.055497e+07-2.009874e+07-1.958135e+07-1.927191e+07-1.865684e+07
2024-03-01 01:30:00+00:000.0966000.9034000.4294090.5705910.8340690.1659310.8402650.1597350.4605110.539489...0.3052060.694794NaN-2.098769e+07-2.067334e+07-2.044858e+07-1.999029e+07-1.929075e+07-1.860416e+07-1.848513e+07
2024-03-01 01:45:00+00:000.0966000.9034000.1632830.8367170.2959130.7040870.8313440.1686560.3922390.607761...0.2253830.774617NaN-2.094209e+07-1.999452e+07-1.993032e+07-1.979889e+07-1.927314e+07-1.854875e+07-1.838815e+07
2024-03-01 02:00:00+00:000.0966000.9034000.4294090.5705910.3194310.6805690.8313440.1686560.3922390.607761...0.1484710.851529NaN-2.068070e+07-2.047176e+07-1.996984e+07-1.978139e+07-1.919273e+07-1.852332e+07-1.850626e+07
2024-03-01 02:15:00+00:000.0958050.9041950.4294090.5705910.6526790.3473210.8025930.1974070.6887270.311273...0.1484710.851529NaN-2.058423e+07-2.036005e+07-1.997906e+07-1.958525e+07-1.894833e+07-1.875233e+07-1.848577e+07
\n", - "

10 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.248825 0.751175 \n", - "2024-03-01 00:15:00+00:00 0.080004 0.919996 \n", - "2024-03-01 00:30:00+00:00 0.470816 0.529184 \n", - "2024-03-01 00:45:00+00:00 0.090027 0.909973 \n", - "2024-03-01 01:00:00+00:00 0.096600 0.903400 \n", - "2024-03-01 01:15:00+00:00 0.096600 0.903400 \n", - "2024-03-01 01:30:00+00:00 0.096600 0.903400 \n", - "2024-03-01 01:45:00+00:00 0.096600 0.903400 \n", - "2024-03-01 02:00:00+00:00 0.096600 0.903400 \n", - "2024-03-01 02:15:00+00:00 0.095805 0.904195 \n", - "\n", - " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.811457 0.188543 \n", - "2024-03-01 00:15:00+00:00 0.529713 0.470287 \n", - "2024-03-01 00:30:00+00:00 0.191837 0.808163 \n", - "2024-03-01 00:45:00+00:00 0.165005 0.834995 \n", - "2024-03-01 01:00:00+00:00 0.643152 0.356848 \n", - "2024-03-01 01:15:00+00:00 0.429409 0.570591 \n", - "2024-03-01 01:30:00+00:00 0.429409 0.570591 \n", - "2024-03-01 01:45:00+00:00 0.163283 0.836717 \n", - "2024-03-01 02:00:00+00:00 0.429409 0.570591 \n", - "2024-03-01 02:15:00+00:00 0.429409 0.570591 \n", - "\n", - " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.775746 0.224254 \n", - "2024-03-01 00:15:00+00:00 0.781772 0.218228 \n", - "2024-03-01 00:30:00+00:00 0.828176 0.171824 \n", - "2024-03-01 00:45:00+00:00 0.515646 0.484354 \n", - "2024-03-01 01:00:00+00:00 0.834069 0.165931 \n", - "2024-03-01 01:15:00+00:00 0.834069 0.165931 \n", - "2024-03-01 01:30:00+00:00 0.834069 0.165931 \n", - "2024-03-01 01:45:00+00:00 0.295913 0.704087 \n", - "2024-03-01 02:00:00+00:00 0.319431 0.680569 \n", - "2024-03-01 02:15:00+00:00 0.652679 0.347321 \n", - "\n", - " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.796702 0.203298 \n", - "2024-03-01 00:15:00+00:00 0.775314 0.224686 \n", - "2024-03-01 00:30:00+00:00 0.775314 0.224686 \n", - "2024-03-01 00:45:00+00:00 0.831668 0.168332 \n", - "2024-03-01 01:00:00+00:00 0.837554 0.162446 \n", - "2024-03-01 01:15:00+00:00 0.837554 0.162446 \n", - "2024-03-01 01:30:00+00:00 0.840265 0.159735 \n", - "2024-03-01 01:45:00+00:00 0.831344 0.168656 \n", - "2024-03-01 02:00:00+00:00 0.831344 0.168656 \n", - "2024-03-01 02:15:00+00:00 0.802593 0.197407 \n", - "\n", - " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", - "timestamp ... \n", - "2024-03-01 00:00:00+00:00 0.737346 0.262654 ... \n", - "2024-03-01 00:15:00+00:00 0.775729 0.224271 ... \n", - "2024-03-01 00:30:00+00:00 0.775729 0.224271 ... \n", - "2024-03-01 00:45:00+00:00 0.629162 0.370838 ... \n", - "2024-03-01 01:00:00+00:00 0.573110 0.426890 ... \n", - "2024-03-01 01:15:00+00:00 0.635596 0.364404 ... \n", - "2024-03-01 01:30:00+00:00 0.460511 0.539489 ... \n", - "2024-03-01 01:45:00+00:00 0.392239 0.607761 ... \n", - "2024-03-01 02:00:00+00:00 0.392239 0.607761 ... \n", - "2024-03-01 02:15:00+00:00 0.688727 0.311273 ... \n", - "\n", - " gblinear_quantile_P95 lgbm_quantile_P95 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 0.135513 0.864487 \n", - "2024-03-01 00:15:00+00:00 0.147720 0.852280 \n", - "2024-03-01 00:30:00+00:00 0.242317 0.757683 \n", - "2024-03-01 00:45:00+00:00 0.678981 0.321019 \n", - "2024-03-01 01:00:00+00:00 0.089815 0.910185 \n", - "2024-03-01 01:15:00+00:00 0.172403 0.827597 \n", - "2024-03-01 01:30:00+00:00 0.305206 0.694794 \n", - "2024-03-01 01:45:00+00:00 0.225383 0.774617 \n", - "2024-03-01 02:00:00+00:00 0.148471 0.851529 \n", - "2024-03-01 02:15:00+00:00 0.148471 0.851529 \n", - "\n", - " load quantile_P05 quantile_P10 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.137667e+07 -2.317569e+07 -2.248770e+07 \n", - "2024-03-01 00:15:00+00:00 NaN -2.209275e+07 -2.208461e+07 \n", - "2024-03-01 00:30:00+00:00 NaN -2.250475e+07 -2.174137e+07 \n", - "2024-03-01 00:45:00+00:00 NaN -2.125884e+07 -2.111755e+07 \n", - "2024-03-01 01:00:00+00:00 NaN -2.126173e+07 -2.106740e+07 \n", - "2024-03-01 01:15:00+00:00 NaN -2.101299e+07 -2.072796e+07 \n", - "2024-03-01 01:30:00+00:00 NaN -2.098769e+07 -2.067334e+07 \n", - "2024-03-01 01:45:00+00:00 NaN -2.094209e+07 -1.999452e+07 \n", - "2024-03-01 02:00:00+00:00 NaN -2.068070e+07 -2.047176e+07 \n", - "2024-03-01 02:15:00+00:00 NaN -2.058423e+07 -2.036005e+07 \n", - "\n", - " quantile_P30 quantile_P50 quantile_P70 \\\n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.196953e+07 -2.158473e+07 -2.100265e+07 \n", - "2024-03-01 00:15:00+00:00 -2.157246e+07 -2.118342e+07 -2.092382e+07 \n", - "2024-03-01 00:30:00+00:00 -2.138064e+07 -2.100023e+07 -2.053325e+07 \n", - "2024-03-01 00:45:00+00:00 -2.109466e+07 -2.071542e+07 -2.019039e+07 \n", - "2024-03-01 01:00:00+00:00 -2.063920e+07 -2.017966e+07 -1.971798e+07 \n", - "2024-03-01 01:15:00+00:00 -2.055497e+07 -2.009874e+07 -1.958135e+07 \n", - "2024-03-01 01:30:00+00:00 -2.044858e+07 -1.999029e+07 -1.929075e+07 \n", - "2024-03-01 01:45:00+00:00 -1.993032e+07 -1.979889e+07 -1.927314e+07 \n", - "2024-03-01 02:00:00+00:00 -1.996984e+07 -1.978139e+07 -1.919273e+07 \n", - "2024-03-01 02:15:00+00:00 -1.997906e+07 -1.958525e+07 -1.894833e+07 \n", - "\n", - " quantile_P90 quantile_P95 \n", - "timestamp \n", - "2024-03-01 00:00:00+00:00 -2.091442e+07 -2.065124e+07 \n", - "2024-03-01 00:15:00+00:00 -2.073507e+07 -2.063586e+07 \n", - "2024-03-01 00:30:00+00:00 -2.041650e+07 -1.982233e+07 \n", - "2024-03-01 00:45:00+00:00 -1.965634e+07 -1.874860e+07 \n", - "2024-03-01 01:00:00+00:00 -1.952271e+07 -1.944001e+07 \n", - "2024-03-01 01:15:00+00:00 -1.927191e+07 -1.865684e+07 \n", - "2024-03-01 01:30:00+00:00 -1.860416e+07 -1.848513e+07 \n", - "2024-03-01 01:45:00+00:00 -1.854875e+07 -1.838815e+07 \n", - "2024-03-01 02:00:00+00:00 -1.852332e+07 -1.850626e+07 \n", - "2024-03-01 02:15:00+00:00 -1.875233e+07 -1.848577e+07 \n", - "\n", - "[10 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
gblinear_quantile_P05lgbm_quantile_P05gblinear_quantile_P10lgbm_quantile_P10gblinear_quantile_P30lgbm_quantile_P30gblinear_quantile_P50lgbm_quantile_P50gblinear_quantile_P70lgbm_quantile_P70...gblinear_quantile_P95lgbm_quantile_P95loadquantile_P05quantile_P10quantile_P30quantile_P50quantile_P70quantile_P90quantile_P95
timestamp
2024-12-31 21:30:00+00:000.9395260.0604740.9155380.0844620.7959210.2040790.7454350.2545650.4425740.557426...0.8867570.113243NaN-2.273370e+07-2.227941e+07-2.161480e+07-2.095055e+07-2.082273e+07-2.000162e+07-1.894876e+07
2024-12-31 21:45:00+00:000.9425230.0574770.9259880.0740120.7961140.2038860.7454350.2545650.5447490.455251...0.8908110.109189NaN-2.230843e+07-2.182914e+07-2.125863e+07-2.061431e+07-2.021932e+07-1.937308e+07-1.861693e+07
2024-12-31 22:00:00+00:000.1249570.8750430.2699510.7300490.6745740.3254260.7502970.2497030.5589390.441061...0.8867570.113243NaN-2.282250e+07-2.207064e+07-2.154538e+07-2.100245e+07-2.062644e+07-1.952722e+07-1.922988e+07
2024-12-31 22:15:00+00:000.1236610.8763390.3958110.6041890.7635740.2364260.7502970.2497030.6103800.389620...0.8867570.113243NaN-2.277797e+07-2.202767e+07-2.129449e+07-2.074317e+07-2.036547e+07-1.922563e+07-1.895426e+07
2024-12-31 22:30:00+00:000.1690190.8309810.3958110.6041890.7997180.2002820.7502970.2497030.6103800.389620...0.8908110.109189NaN-2.288190e+07-2.198203e+07-2.113597e+07-2.059848e+07-2.022853e+07-1.928894e+07-1.879247e+07
2024-12-31 22:45:00+00:000.5734140.4265860.6919970.3080030.8364110.1635890.7502970.2497030.6638710.336129...0.8887060.111294NaN-2.297925e+07-2.195887e+07-2.092103e+07-2.037274e+07-1.998042e+07-1.879439e+07-1.855492e+07
2024-12-31 23:00:00+00:000.1366740.8633260.5835700.4164300.8317760.1682240.7502970.2497030.6103800.389620...0.8909380.109062NaN-2.278653e+07-2.227599e+07-2.117868e+07-2.064982e+07-2.022001e+07-1.903018e+07-1.886584e+07
2024-12-31 23:15:00+00:000.3345920.6654080.6919970.3080030.8364110.1635890.7019710.2980290.7219740.278026...0.8887060.111294NaN-2.290655e+07-2.199052e+07-2.079653e+07-2.032965e+07-1.923286e+07-1.873750e+07-1.853917e+07
2024-12-31 23:30:00+00:000.7203120.2796880.8990150.1009850.8021260.1978740.5171450.4828550.7116600.288340...0.8795430.120457NaN-2.297595e+07-2.184417e+07-2.069840e+07-2.034597e+07-1.889343e+07-1.838063e+07-1.830427e+07
2024-12-31 23:45:00+00:000.6176670.3823330.8438620.1561380.7960460.2039540.5437440.4562560.7116600.288340...0.8662030.133797NaN-2.255517e+07-2.142616e+07-2.021394e+07-1.989196e+07-1.843501e+07-1.807584e+07-1.765399e+07
\n", - "

10 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " gblinear_quantile_P05 lgbm_quantile_P05 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 0.939526 0.060474 \n", - "2024-12-31 21:45:00+00:00 0.942523 0.057477 \n", - "2024-12-31 22:00:00+00:00 0.124957 0.875043 \n", - "2024-12-31 22:15:00+00:00 0.123661 0.876339 \n", - "2024-12-31 22:30:00+00:00 0.169019 0.830981 \n", - "2024-12-31 22:45:00+00:00 0.573414 0.426586 \n", - "2024-12-31 23:00:00+00:00 0.136674 0.863326 \n", - "2024-12-31 23:15:00+00:00 0.334592 0.665408 \n", - "2024-12-31 23:30:00+00:00 0.720312 0.279688 \n", - "2024-12-31 23:45:00+00:00 0.617667 0.382333 \n", - "\n", - " gblinear_quantile_P10 lgbm_quantile_P10 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 0.915538 0.084462 \n", - "2024-12-31 21:45:00+00:00 0.925988 0.074012 \n", - "2024-12-31 22:00:00+00:00 0.269951 0.730049 \n", - "2024-12-31 22:15:00+00:00 0.395811 0.604189 \n", - "2024-12-31 22:30:00+00:00 0.395811 0.604189 \n", - "2024-12-31 22:45:00+00:00 0.691997 0.308003 \n", - "2024-12-31 23:00:00+00:00 0.583570 0.416430 \n", - "2024-12-31 23:15:00+00:00 0.691997 0.308003 \n", - "2024-12-31 23:30:00+00:00 0.899015 0.100985 \n", - "2024-12-31 23:45:00+00:00 0.843862 0.156138 \n", - "\n", - " gblinear_quantile_P30 lgbm_quantile_P30 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 0.795921 0.204079 \n", - "2024-12-31 21:45:00+00:00 0.796114 0.203886 \n", - "2024-12-31 22:00:00+00:00 0.674574 0.325426 \n", - "2024-12-31 22:15:00+00:00 0.763574 0.236426 \n", - "2024-12-31 22:30:00+00:00 0.799718 0.200282 \n", - "2024-12-31 22:45:00+00:00 0.836411 0.163589 \n", - "2024-12-31 23:00:00+00:00 0.831776 0.168224 \n", - "2024-12-31 23:15:00+00:00 0.836411 0.163589 \n", - "2024-12-31 23:30:00+00:00 0.802126 0.197874 \n", - "2024-12-31 23:45:00+00:00 0.796046 0.203954 \n", - "\n", - " gblinear_quantile_P50 lgbm_quantile_P50 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 0.745435 0.254565 \n", - "2024-12-31 21:45:00+00:00 0.745435 0.254565 \n", - "2024-12-31 22:00:00+00:00 0.750297 0.249703 \n", - "2024-12-31 22:15:00+00:00 0.750297 0.249703 \n", - "2024-12-31 22:30:00+00:00 0.750297 0.249703 \n", - "2024-12-31 22:45:00+00:00 0.750297 0.249703 \n", - "2024-12-31 23:00:00+00:00 0.750297 0.249703 \n", - "2024-12-31 23:15:00+00:00 0.701971 0.298029 \n", - "2024-12-31 23:30:00+00:00 0.517145 0.482855 \n", - "2024-12-31 23:45:00+00:00 0.543744 0.456256 \n", - "\n", - " gblinear_quantile_P70 lgbm_quantile_P70 ... \\\n", - "timestamp ... \n", - "2024-12-31 21:30:00+00:00 0.442574 0.557426 ... \n", - "2024-12-31 21:45:00+00:00 0.544749 0.455251 ... \n", - "2024-12-31 22:00:00+00:00 0.558939 0.441061 ... \n", - "2024-12-31 22:15:00+00:00 0.610380 0.389620 ... \n", - "2024-12-31 22:30:00+00:00 0.610380 0.389620 ... \n", - "2024-12-31 22:45:00+00:00 0.663871 0.336129 ... \n", - "2024-12-31 23:00:00+00:00 0.610380 0.389620 ... \n", - "2024-12-31 23:15:00+00:00 0.721974 0.278026 ... \n", - "2024-12-31 23:30:00+00:00 0.711660 0.288340 ... \n", - "2024-12-31 23:45:00+00:00 0.711660 0.288340 ... \n", - "\n", - " gblinear_quantile_P95 lgbm_quantile_P95 load \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 0.886757 0.113243 NaN \n", - "2024-12-31 21:45:00+00:00 0.890811 0.109189 NaN \n", - "2024-12-31 22:00:00+00:00 0.886757 0.113243 NaN \n", - "2024-12-31 22:15:00+00:00 0.886757 0.113243 NaN \n", - "2024-12-31 22:30:00+00:00 0.890811 0.109189 NaN \n", - "2024-12-31 22:45:00+00:00 0.888706 0.111294 NaN \n", - "2024-12-31 23:00:00+00:00 0.890938 0.109062 NaN \n", - "2024-12-31 23:15:00+00:00 0.888706 0.111294 NaN \n", - "2024-12-31 23:30:00+00:00 0.879543 0.120457 NaN \n", - "2024-12-31 23:45:00+00:00 0.866203 0.133797 NaN \n", - "\n", - " quantile_P05 quantile_P10 quantile_P30 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 -2.273370e+07 -2.227941e+07 -2.161480e+07 \n", - "2024-12-31 21:45:00+00:00 -2.230843e+07 -2.182914e+07 -2.125863e+07 \n", - "2024-12-31 22:00:00+00:00 -2.282250e+07 -2.207064e+07 -2.154538e+07 \n", - "2024-12-31 22:15:00+00:00 -2.277797e+07 -2.202767e+07 -2.129449e+07 \n", - "2024-12-31 22:30:00+00:00 -2.288190e+07 -2.198203e+07 -2.113597e+07 \n", - "2024-12-31 22:45:00+00:00 -2.297925e+07 -2.195887e+07 -2.092103e+07 \n", - "2024-12-31 23:00:00+00:00 -2.278653e+07 -2.227599e+07 -2.117868e+07 \n", - "2024-12-31 23:15:00+00:00 -2.290655e+07 -2.199052e+07 -2.079653e+07 \n", - "2024-12-31 23:30:00+00:00 -2.297595e+07 -2.184417e+07 -2.069840e+07 \n", - "2024-12-31 23:45:00+00:00 -2.255517e+07 -2.142616e+07 -2.021394e+07 \n", - "\n", - " quantile_P50 quantile_P70 quantile_P90 \\\n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 -2.095055e+07 -2.082273e+07 -2.000162e+07 \n", - "2024-12-31 21:45:00+00:00 -2.061431e+07 -2.021932e+07 -1.937308e+07 \n", - "2024-12-31 22:00:00+00:00 -2.100245e+07 -2.062644e+07 -1.952722e+07 \n", - "2024-12-31 22:15:00+00:00 -2.074317e+07 -2.036547e+07 -1.922563e+07 \n", - "2024-12-31 22:30:00+00:00 -2.059848e+07 -2.022853e+07 -1.928894e+07 \n", - "2024-12-31 22:45:00+00:00 -2.037274e+07 -1.998042e+07 -1.879439e+07 \n", - "2024-12-31 23:00:00+00:00 -2.064982e+07 -2.022001e+07 -1.903018e+07 \n", - "2024-12-31 23:15:00+00:00 -2.032965e+07 -1.923286e+07 -1.873750e+07 \n", - "2024-12-31 23:30:00+00:00 -2.034597e+07 -1.889343e+07 -1.838063e+07 \n", - "2024-12-31 23:45:00+00:00 -1.989196e+07 -1.843501e+07 -1.807584e+07 \n", - "\n", - " quantile_P95 \n", - "timestamp \n", - "2024-12-31 21:30:00+00:00 -1.894876e+07 \n", - "2024-12-31 21:45:00+00:00 -1.861693e+07 \n", - "2024-12-31 22:00:00+00:00 -1.922988e+07 \n", - "2024-12-31 22:15:00+00:00 -1.895426e+07 \n", - "2024-12-31 22:30:00+00:00 -1.879247e+07 \n", - "2024-12-31 22:45:00+00:00 -1.855492e+07 \n", - "2024-12-31 23:00:00+00:00 -1.886584e+07 \n", - "2024-12-31 23:15:00+00:00 -1.853917e+07 \n", - "2024-12-31 23:30:00+00:00 -1.830427e+07 \n", - "2024-12-31 23:45:00+00:00 -1.765399e+07 \n", - "\n", - "[10 rows x 22 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "# Define the folder path\n", - "folder_path = Path(\"C:/Users/GuillyKOLKMAN/OneDrive - SIA PARTNERS/Desktop/Projects/Alliander/openstef/benchmark_results/cache/Ensemble_contributionslgbm_gblinear_learned_weights_lgbm_OS Apeldoorn\")\n", - "\n", - "# Get all parquet files in the folder\n", - "parquet_files = sorted(folder_path.glob(\"contrib_*_predict.parquet\"))\n", - "\n", - "print(f\"Found {len(parquet_files)} files\")\n", - "\n", - "# Read and concatenate first 6 hours (24 rows) from each file\n", - "df_list = []\n", - "for file in tqdm(parquet_files):\n", - " df_temp = pd.read_parquet(file)\n", - " df_6h = df_temp.head(24) # First 6 hours at 15-min intervals\n", - " df_list.append(df_6h)\n", - "\n", - "# Concatenate all into one dataframe\n", - "df_combined = pd.concat(df_list, axis=0, ignore_index=False)\n", - "\n", - "print(f\"Combined dataframe shape: {df_combined.shape}\")\n", - "display(df_combined.head(10))\n", - "display(df_combined.tail(10))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5febfec5", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "992a7f53", - "metadata": {}, - "outputs": [], - "source": [ - "def plot_model_contributions(df, start_date=None, end_date=None, quantiles=None):\n", - " \"\"\"\n", - " Plot comparison of GBLinear and LGBM contributions with quantile dropdown filter using Plotly.\n", - "\n", - " Args:\n", - " df: DataFrame with model contribution columns\n", - " start_date: Start date for filtering (str or datetime). If None, uses all data.\n", - " end_date: End date for filtering (str or datetime). If None, uses all data.\n", - " quantiles: List of quantile names to include in filter (e.g., ['P05', 'P50']). If None, uses all available.\n", - " \"\"\"\n", - " # Filter dataframe by date range if specified\n", - " df_plot = df.copy()\n", - " if start_date is not None:\n", - " start_dt = pd.to_datetime(start_date)\n", - " if df_plot.index.tz is not None and start_dt.tz is None:\n", - " start_dt = start_dt.tz_localize('UTC')\n", - " df_plot = df_plot[df_plot.index >= start_dt]\n", - " if end_date is not None:\n", - " end_dt = pd.to_datetime(end_date)\n", - " if df_plot.index.tz is not None and end_dt.tz is None:\n", - " end_dt = end_dt.tz_localize('UTC')\n", - " df_plot = df_plot[df_plot.index <= end_dt]\n", - "\n", - " # Get quantiles to plot\n", - " if quantiles is None:\n", - " quantiles = ['P05', 'P10', 'P30', 'P50', 'P70', 'P90', 'P95']\n", - "\n", - " # Create figure\n", - " fig = go.Figure()\n", - "\n", - " # Add traces for each quantile\n", - " for quantile in quantiles:\n", - " gblinear_col = f'gblinear_quantile_{quantile}'\n", - " lgbm_col = f'lgbm_quantile_{quantile}'\n", - "\n", - " # Add GBLinear trace\n", - " fig.add_trace(\n", - " go.Scatter(\n", - " x=df_plot.index,\n", - " y=df_plot[gblinear_col],\n", - " name=f'GBLinear - {quantile}',\n", - " mode='lines',\n", - " line=dict(width=2, color='blue'),\n", - " opacity=0.7,\n", - " visible=(quantile == 'P50') # Only P50 visible by default\n", - " )\n", - " )\n", - "\n", - " # Add LGBM trace\n", - " fig.add_trace(\n", - " go.Scatter(\n", - " x=df_plot.index,\n", - " y=df_plot[lgbm_col],\n", - " name=f'LGBM - {quantile}',\n", - " mode='lines',\n", - " line=dict(width=2, color='#ff7f0e'), # high-contrast orange for LGBM\n", - " opacity=0.85,\n", - " visible=(quantile == 'P50') # Only P50 visible by default\n", - " )\n", - " )\n", - "\n", - " # Make the GBLinear trace (added just before) explicitly high-contrast too\n", - " # fig.data[-2] refers to the GBLinear trace we added immediately prior\n", - " if len(fig.data) >= 2:\n", - " fig.data[-2].update(line=dict(color='#1f77b4', width=2), opacity=0.85) # contrasting blue for GBLinear\n", - "\n", - " # Create dropdown buttons\n", - " buttons = []\n", - " for i, quantile in enumerate(quantiles):\n", - " # Create visibility list - show only the selected quantile (both GBLinear and LGBM)\n", - " visible = [False] * (len(quantiles) * 2)\n", - " visible[i * 2] = True # GBLinear for this quantile\n", - " visible[i * 2 + 1] = True # LGBM for this quantile\n", - "\n", - " buttons.append(\n", - " dict(\n", - " label=quantile,\n", - " method='update',\n", - " args=[\n", - " {'visible': visible},\n", - " {'title.text': f'Model Contributions - Quantile {quantile}'}\n", - " ]\n", - " )\n", - " )\n", - "\n", - " # Update layout with dropdown menu\n", - " # determine default active quantile (P50 if available)\n", - " active_idx = quantiles.index('P50') if 'P50' in quantiles else 0\n", - "\n", - " fig.update_layout(\n", - " legend=dict(\n", - " orientation='v',\n", - " x=1.02,\n", - " xanchor='left',\n", - " y=1.0,\n", - " yanchor='top'\n", - " ),\n", - " updatemenus=[\n", - " dict(\n", - " type='buttons',\n", - " buttons=buttons,\n", - " direction='down',\n", - " showactive=True,\n", - " active=active_idx, # Set the default active button\n", - " x=1.02,\n", - " xanchor='left',\n", - " y=0.88, # place beneath the legend\n", - " yanchor='top',\n", - " pad={'r': 10, 't': 10},\n", - " bgcolor='lightgray',\n", - " bordercolor='gray',\n", - " borderwidth=1,\n", - " )\n", - " ],\n", - " title=f\"Model Contributions - Quantile {quantiles[active_idx]}\",\n", - " xaxis_title='Timestamp',\n", - " yaxis_title='Contribution',\n", - " hovermode='x unified',\n", - " height=600,\n", - " showlegend=True\n", - " )\n", - "\n", - " fig.show()\n", - "\n", - "# Example usage - plot all data\n", - "# plot_model_contributions(df_combined)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "880c6150", - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Plot specific date range\n", - "# plot_model_contributions(df_combined, start_date='2024-04-21', end_date='2024-04-30')" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "56c1846f", - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Plot specific quantiles only\n", - "# plot_model_contributions(df_combined, start_date='2024-11-17', end_date='2024-11-29')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "abef9b04", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}