From bd315bcead4cd64aa24f5150ba997cd81e9df2f3 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 2 Mar 2026 09:31:44 -0700 Subject: [PATCH 01/31] initialize branch From 3be7fde4f9d855da1fe6beb7c0f393f6f1d1fe1c Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 4 Mar 2026 12:58:38 -0700 Subject: [PATCH 02/31] move stages, reps, commit, etc config to data class --- gtep/gtep_data.py | 58 +++++++++++++++++++++++++++++++++++----------- gtep/gtep_model.py | 18 +++++--------- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 75baa4eb..c7f83369 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -27,28 +27,54 @@ class ExpansionPlanningData: """Standard data storage class for the IDAES GTEP model.""" - def __init__(self): - pass + def __init__( + self, + stages=2, + num_reps=4, + len_reps=1, + num_commit=24, + num_dispatch=1, + duration_dispatch=60, + ): - def load_prescient(self, data_path, options_dict=None): + self.stages = stages + self.num_reps = num_reps + self.len_reps = len_reps + self.num_commit = num_commit + self.num_dispatch = num_dispatch + self.duration_dispatch = duration_dispatch + + def load_prescient( + self, + data_path, + representative_dates, + representative_weights=None, + options_dict=None, + ): """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None """ self.data_type = "prescient" - options_dict = { - "data_path": data_path, - "input_format": "rts-gmlc", - "start_date": "01-01-2020", - "num_days": 365, - "sced_horizon": 1, - "sced_frequency_minutes": 60, - "ruc_horizon": 36, - } + # create prescient config object with defaults prescient_options = PrescientConfig() + + if options_dict is None: + # set basic configurations that do not match prescient defaults + options_dict = { + "data_path": data_path, + "num_days": 365, + "ruc_horizon": 36, + } + else: + # ensure data path is included in options dictionary + options_dict["data_path"] = data_path + + # update configuration values based on options dictionary prescient_options.set_value(options_dict) + # Use prescient data provider to load in sequential data for representative periods data_list = [] @@ -223,8 +249,12 @@ def load_default_data_settings(self): self.md.data["elements"]["generator"][gen]["max_operating_reserve"] = 1 self.md.data["elements"]["generator"][gen]["max_spinning_reserve"] = 1 self.md.data["elements"]["generator"][gen]["max_quickstart_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["ramp_up_rate"] = 0.1 - self.md.data["elements"]["generator"][gen]["ramp_down_rate"] = 0.1 + self.md.data["elements"]["generator"][gen][ + "ramp_up_rate" + ] = 0.1 # FIXME should we update to only add if necessary + self.md.data["elements"]["generator"][gen][ + "ramp_down_rate" + ] = 0.1 # FIXME should we update to only add if necessary self.md.data["elements"]["generator"][gen]["emissions_factor"] = 1 self.md.data["elements"]["generator"][gen]["start_fuel"] = 1 self.md.data["elements"]["generator"][gen]["investment_cost"] = 1 diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 80df49eb..72efceb9 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -72,15 +72,9 @@ class ExpansionPlanningModel: def __init__( self, config=None, - stages=1, formulation=None, data=None, cost_data=None, - num_reps=3, - len_reps=24, - num_commit=24, - num_dispatch=4, - duration_dispatch=15, ): """Initialize generation & expansion planning model object. @@ -96,15 +90,15 @@ def __init__( :return: Pyomo model for full GTEP """ - self.stages = stages + self.stages = data.stages self.formulation = formulation self.data = data self.cost_data = cost_data - self.num_reps = num_reps - self.len_reps = len_reps - self.num_commit = num_commit - self.num_dispatch = num_dispatch - self.duration_dispatch = duration_dispatch + self.num_reps = data.num_reps + self.len_reps = data.len_reps + self.num_commit = data.num_commit + self.num_dispatch = data.num_dispatch + self.duration_dispatch = data.duration_dispatch self.config = _get_model_config() self.timer = TicTocTimer() From 7d7bd16b08a1be4e73fc57d5691a264accd7c097 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 4 Mar 2026 13:11:15 -0700 Subject: [PATCH 03/31] update data object initialization in test_validation.py --- gtep/tests/unit/test_validation.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 5dc19b10..0683cc7a 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -32,17 +32,16 @@ def get_solution_object(): - data_object = ExpansionPlanningData() - data_object.load_prescient(input_data_source) - - mod_object = ExpansionPlanningModel( + data_object = ExpansionPlanningData( stages=2, - data=data_object, num_reps=2, len_reps=1, num_commit=6, num_dispatch=4, ) + data_object.load_prescient(input_data_source) + + mod_object = ExpansionPlanningModel(data=data_object) mod_object.create_model() TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) TransformationFactory("gdp.bigm").apply_to(mod_object.model) From ff6be6b256685d5e01636f7f7fa3f404e778d441 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 4 Mar 2026 14:07:19 -0700 Subject: [PATCH 04/31] update gtep_model tests to move stages, reps, etc to data object --- gtep/tests/unit/test_gtep_model.py | 69 +++++++++++++++++++----------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index e48150a3..09cf1a2d 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -65,24 +65,32 @@ def test_model_init(self): self.assertIsInstance(modObject, ExpansionPlanningModel) modObject.create_model() self.assertIsInstance(modObject.model, ConcreteModel) - self.assertEqual(modObject.stages, 1) + self.assertEqual(modObject.stages, 2) self.assertEqual(modObject.formulation, None) self.assertIsInstance(modObject.model.md, ModelData) - self.assertEqual(modObject.num_reps, 3) - self.assertEqual(modObject.len_reps, 24) + self.assertEqual(modObject.num_reps, 4) + self.assertEqual(modObject.len_reps, 1) self.assertEqual(modObject.num_commit, 24) self.assertEqual(modObject.num_dispatch, 4) + self.assertEqual(modObject.duration_dispatch, 60) # Test that the ExpansionPlanningModel object can read a default dataset and init # properly with non-default values - modObject = ExpansionPlanningModel( - data=data_object, + dataObject = ExpansionPlanningData( stages=2, num_reps=4, len_reps=16, num_commit=12, num_dispatch=12, + duration_dispatch=30, ) + + curr_dir = dirname(abspath(__file__)) + debug_data_path = abspath(join(curr_dir, "..", "..", "data", "5bus")) + dataObject.load_prescient(debug_data_path) + + modObject = ExpansionPlanningModel(data=data_object) + self.assertIsInstance(modObject, ExpansionPlanningModel) modObject.create_model() self.assertIsInstance(modObject.model, ConcreteModel) @@ -93,6 +101,7 @@ def test_model_init(self): self.assertEqual(modObject.len_reps, 16) self.assertEqual(modObject.num_commit, 12) self.assertEqual(modObject.num_dispatch, 12) + self.assertEqual(modObject.duration_dispatch, 30) # We have expansion blocks and they are where and what we think they are expansion_blocks = modObject.model.component("investmentStage") @@ -126,14 +135,14 @@ def test_model_unit_consistency(self): # Test that the ExpansionPlanningModel has consistent units and spot check that # components have their expected units data_object = read_debug_model() - modObject = ExpansionPlanningModel( - data=data_object, - stages=2, - num_reps=2, - len_reps=2, - num_commit=2, - num_dispatch=2, - ) + # update data object details + data_object.stages = 2 + data_object.num_reps = 2 + data_object.len_reps = 2 + data_object.num_commit = 2 + data_object.num_dispatch = 2 + + modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() m = modObject.model @@ -188,9 +197,14 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is data_object = read_debug_model() - modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 - ) + # update data object details + data_object.stages = 1 + data_object.num_reps = 1 + data_object.len_reps = 1 + data_object.num_commit = 1 + data_object.num_dispatch = 1 + + modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() # Check for consistent units @@ -215,9 +229,14 @@ def test_solve_bigm(self): def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model() - modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 - ) + # update data object details + data_object.stages = 1 + data_object.num_reps = 1 + data_object.len_reps = 1 + data_object.num_commit = 1 + data_object.num_dispatch = 1 + + modObject = ExpansionPlanningModel(data=data_object) modObject.config["include_investment"] = False modObject.create_model() @@ -245,6 +264,12 @@ def test_with_cost_data(self): # Test ExpansionPlanningModel with cost data # This model originated from driver_esr.py data_object = read_debug_model() + # update data object details + data_object.stages = 2 + data_object.num_reps = 2 + data_object.len_reps = 1 + data_object.num_commit = 6 + data_object.num_dispatch = 15 curr_dir = dirname(abspath(__file__)) data_path = abspath(join(curr_dir, "..", "..", "data", "costs")) @@ -271,14 +296,8 @@ def test_with_cost_data(self): # Populate and create GTEP model mod_object = ExpansionPlanningModel( - stages=2, data=data_object, cost_data=data_processing_object, - num_reps=2, - len_reps=1, - num_commit=6, - num_dispatch=4, - duration_dispatch=15, ) mod_object.config["include_investment"] = True From 32070ce76a5992dad9d90826c5f317ec9d73faaf Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 4 Mar 2026 14:35:36 -0700 Subject: [PATCH 05/31] update the gtep model tests for inputs in helper function --- gtep/tests/unit/test_gtep_model.py | 62 +++++++++++++----------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 09cf1a2d..29f15fe1 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -47,10 +47,20 @@ def patch_unit_handlers(): # Helper functions -def read_debug_model(): +def read_debug_model( + stages=2, + num_reps=4, + len_reps=1, + num_commit=24, + num_dispatch=1, + duration_dispatch=60, +): + curr_dir = dirname(abspath(__file__)) debug_data_path = abspath(join(curr_dir, "..", "..", "data", "5bus")) - dataObject = ExpansionPlanningData() + dataObject = ExpansionPlanningData( + stages, num_reps, len_reps, num_commit, num_dispatch, duration_dispatch + ) dataObject.load_prescient(debug_data_path) return dataObject @@ -71,12 +81,12 @@ def test_model_init(self): self.assertEqual(modObject.num_reps, 4) self.assertEqual(modObject.len_reps, 1) self.assertEqual(modObject.num_commit, 24) - self.assertEqual(modObject.num_dispatch, 4) + self.assertEqual(modObject.num_dispatch, 1) self.assertEqual(modObject.duration_dispatch, 60) # Test that the ExpansionPlanningModel object can read a default dataset and init # properly with non-default values - dataObject = ExpansionPlanningData( + data_object = read_debug_model( stages=2, num_reps=4, len_reps=16, @@ -85,10 +95,6 @@ def test_model_init(self): duration_dispatch=30, ) - curr_dir = dirname(abspath(__file__)) - debug_data_path = abspath(join(curr_dir, "..", "..", "data", "5bus")) - dataObject.load_prescient(debug_data_path) - modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) @@ -134,13 +140,9 @@ def test_model_init(self): def test_model_unit_consistency(self): # Test that the ExpansionPlanningModel has consistent units and spot check that # components have their expected units - data_object = read_debug_model() - # update data object details - data_object.stages = 2 - data_object.num_reps = 2 - data_object.len_reps = 2 - data_object.num_commit = 2 - data_object.num_dispatch = 2 + data_object = read_debug_model( + stages=2, num_reps=2, len_reps=2, num_commit=2, num_dispatch=2 + ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -196,13 +198,9 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is - data_object = read_debug_model() - # update data object details - data_object.stages = 1 - data_object.num_reps = 1 - data_object.len_reps = 1 - data_object.num_commit = 1 - data_object.num_dispatch = 1 + data_object = read_debug_model( + stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -228,13 +226,9 @@ def test_solve_bigm(self): def test_no_investment(self): # Solve the debug model with no investment - data_object = read_debug_model() - # update data object details - data_object.stages = 1 - data_object.num_reps = 1 - data_object.len_reps = 1 - data_object.num_commit = 1 - data_object.num_dispatch = 1 + data_object = read_debug_model( + stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + ) modObject = ExpansionPlanningModel(data=data_object) modObject.config["include_investment"] = False @@ -263,13 +257,9 @@ def test_no_investment(self): def test_with_cost_data(self): # Test ExpansionPlanningModel with cost data # This model originated from driver_esr.py - data_object = read_debug_model() - # update data object details - data_object.stages = 2 - data_object.num_reps = 2 - data_object.len_reps = 1 - data_object.num_commit = 6 - data_object.num_dispatch = 15 + data_object = read_debug_model( + stages=2, num_reps=2, len_reps=1, num_commit=6, num_dispatch=15 + ) curr_dir = dirname(abspath(__file__)) data_path = abspath(join(curr_dir, "..", "..", "data", "costs")) From ed258a1293d7526fe992dd1b5a8c35c454403fd8 Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 5 Mar 2026 14:05:54 -0700 Subject: [PATCH 06/31] update gtep_structure --- gtep/gtep_data.py | 79 +++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index c7f83369..137716cc 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -22,6 +22,7 @@ from prescient.data.providers import gmlc_data_provider import datetime import pandas as pd +import os class ExpansionPlanningData: @@ -48,13 +49,16 @@ def load_prescient( self, data_path, representative_dates, - representative_weights=None, + representative_weights={}, options_dict=None, ): """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded + :param representative_dates: List of time points to include Note: Change the last date for whatever extreme day is needed based on the given run(s) + :param representative_weights: dictionary of weights for each representative date, defaults to empty Dict :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None + """ self.data_type = "prescient" @@ -68,6 +72,7 @@ def load_prescient( "num_days": 365, "ruc_horizon": 36, } + else: # ensure data path is included in options dictionary options_dict["data_path"] = data_path @@ -78,18 +83,32 @@ def load_prescient( # Use prescient data provider to load in sequential data for representative periods data_list = [] - x = datetime.datetime(2020, 1, 1) data_provider = gmlc_data_provider.GmlcDataProvider(options=prescient_options) + + # grab details from simulation objects file (data provider above throws error if no simulation_objects.csv exists) + metadata_path = os.path.join(data_path, "simulation_objects.csv") + metadata_df = pd.read_csv(metadata_path, index_col=0) + + # save to variable for easy calling + sced_freq_min = prescient_options.sced_frequency_minutes + + # TODO double check that this is the value to check with total steps (the old hard code was 24*365 for num_time_steps) + period_per_step = int(metadata_df.loc["Periods_per_Step"]["REAL_TIME"]) + total_num_steps = prescient_options.num_days * period_per_step + # populate an egret model data with the basic stuff self.md = data_provider.get_initial_actuals_model( - options=prescient_options, num_time_steps=24 * 365, minutes_per_timestep=60 + options=prescient_options, + num_time_steps=total_num_steps, + minutes_per_timestep=sced_freq_min, ) + # fill in renewable actuals and maybe demand idk data_provider.populate_with_actuals( options=prescient_options, - num_time_periods=24 * 365, - time_period_length_minutes=60, - start_time=x, + num_time_periods=total_num_steps, + time_period_length_minutes=sced_freq_min, + start_time=data_provider._start_time, model=self.md, ) @@ -115,40 +134,43 @@ def load_prescient( ## of modelData objects, not just a single modelData object # Arbitrary time points and lengths picked for representative periods # default here allows up to 24 hours for periods + self.representative_dates = representative_dates - ## RMA: - ## Change the last date for whatever extreme day is needed based on the given run(s) + if not representative_weights: + # set the weight for each day to the total weight divided by number of days + total_weight = prescient_options.num_days * self.stages + weight_per_date = int(total_weight / (len(representative_dates))) + self.representative_weights = { + key: weight_per_date + for date, key in enumerate(self.representative_dates) + } time_keys = self.md.data["system"]["time_keys"] - self.representative_dates = [ - "2020-01-28 00:00", - "2020-04-23 00:00", - "2020-07-05 00:00", - "2020-10-14 00:00", - ] - ## FIXME: - ## RESIL WEEK ONLY - ## but we'll want something similar just less insane in the future - if len(self.representative_dates) == 5: - self.representative_weights = {1: 91, 2: 91, 3: 91, 4: 91, 5: 1} - else: - self.representative_weights = {1: 91, 2: 91, 3: 91, 4: 91} - # self.representative_weights = {1:1} for date in self.representative_dates: key_idx = time_keys.index(date) - time_key_set = time_keys[key_idx : key_idx + 24] + time_key_set = time_keys[key_idx : key_idx + period_per_step] data_list.append(self.md.clone_at_time_keys(time_key_set)) self.representative_data = data_list - def import_load_scaling(self, load_file_name): + def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]): adjusted_forecast = pd.read_excel(load_file_name) + + # check years are valid + if len(forecast_years) < self.stages: + raise ValueError( + "Not enough forecast years for the number of stages of investment" + ) + elif any(year < 2020 or year > 2050 for year in forecast_years): + raise ValueError( + "The list of years includes a year before 2020 or after 2050." + ) + adjusted_forecast_by_period = adjusted_forecast[ - (adjusted_forecast["year"] == 2025) - | (adjusted_forecast["year"] == 2030) - | (adjusted_forecast["year"] == 2035) + adjusted_forecast["year"].isin(forecast_years) ] + base_zones = [ "base_economic_coast", "base_economic_east", @@ -283,6 +305,9 @@ def load_storage_csv(self, data_path): self.md.data["elements"]["storage"] = {} def texas_case_study_updates(self, data_path): + if "Texas" or "Coal" not in data_path: + raise ValueError("The data path provided is not a Texas case study") + generator_update_path = data_path + "/gen.csv" generator_df = pd.read_csv(generator_update_path) bonus_feature_list = [ From cbf1d105e11f21dd16f58dc501cdc5703b48d45e Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 5 Mar 2026 14:53:45 -0700 Subject: [PATCH 07/31] add error raises and update imports --- gtep/gtep_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 137716cc..4cc116a1 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -20,7 +20,6 @@ from pyomo.environ import * from prescient.simulator.config import PrescientConfig from prescient.data.providers import gmlc_data_provider -import datetime import pandas as pd import os From 7d0a3e3343353a5514320618d4da9960f4d02d23 Mon Sep 17 00:00:00 2001 From: bstorm Date: Mon, 9 Mar 2026 16:42:23 -0600 Subject: [PATCH 08/31] update period --- gtep/gtep_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 4cc116a1..8c95f7b8 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -91,8 +91,9 @@ def load_prescient( # save to variable for easy calling sced_freq_min = prescient_options.sced_frequency_minutes - # TODO double check that this is the value to check with total steps (the old hard code was 24*365 for num_time_steps) - period_per_step = int(metadata_df.loc["Periods_per_Step"]["REAL_TIME"]) + # This step is grabbing DAY_AHEAD information for now + # (in the future we may want to update to grab the "REAL_TIME" data if the data has reliable data since the actuals model is looking for real time data info) + period_per_step = int(metadata_df.loc["Periods_per_Step"]["DAY_AHEAD"]) total_num_steps = prescient_options.num_days * period_per_step # populate an egret model data with the basic stuff @@ -252,7 +253,7 @@ def import_outage_data(self, load_file_name): self.bus_hours = self.bus_hours.astype(int) def load_default_data_settings(self): - ## TODO: too many of these are hard coded; everything should check if it exists too. + ## TODO: everything should check if it exists too. """Fills in necessary but unspecified data information.""" for gen in self.md.data["elements"]["generator"]: if self.md.data["elements"]["generator"][gen]["fuel"] == "C": From 2f1abde94ed88a3275633a1ac5fe610dcff3147b Mon Sep 17 00:00:00 2001 From: Belle Date: Mon, 16 Mar 2026 11:32:18 -0600 Subject: [PATCH 09/31] Update comments and add representative dates to prescient input in tests --- gtep/gtep_data.py | 116 +++++++++++++++++++++-------- gtep/tests/unit/test_gtep_model.py | 10 ++- gtep/tests/unit/test_validation.py | 10 ++- 3 files changed, 101 insertions(+), 35 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 8c95f7b8..56232ff2 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -155,6 +155,12 @@ def load_prescient( self.representative_data = data_list def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]): + """Imports load scaling data for forecast years. + + :param load_file_name: filepath for adjusted forecast excel file + :param forecast_years: list of years to forecast, defaults to [2025, 2030, 2035] + + """ adjusted_forecast = pd.read_excel(load_file_name) # check years are valid @@ -221,11 +227,15 @@ def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]) self.load_scaling = load_scaling_df def import_outage_data(self, load_file_name): + """Imports outage data. + + :param load_file_name: filepath for adjusted forecast excel file + + """ outage_list = pd.read_csv(load_file_name) percentile_threshold = 0.9 threshold_value = outage_list["case_4b_prob"].quantile(percentile_threshold) filtered_outages = outage_list[outage_list["case_4b_prob"] >= threshold_value] - import re filtered_outages["hour"] = filtered_outages["lim_timestamp"].str.extract( r" (\d+):" @@ -253,41 +263,75 @@ def import_outage_data(self, load_file_name): self.bus_hours = self.bus_hours.astype(int) def load_default_data_settings(self): - ## TODO: everything should check if it exists too. + # Many of these items are hardcoded for the time being """Fills in necessary but unspecified data information.""" - for gen in self.md.data["elements"]["generator"]: - if self.md.data["elements"]["generator"][gen]["fuel"] == "C": - if self.md.data["elements"]["generator"][gen]["in_service"] == False: - self.md.data["elements"]["generator"][gen]["lifetime"] = 1 - else: - self.md.data["elements"]["generator"][gen]["lifetime"] = 2 - else: - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 - self.md.data["elements"]["generator"][gen]["spinning_reserve_frac"] = 0.1 - self.md.data["elements"]["generator"][gen]["quickstart_reserve_frac"] = 0.1 - self.md.data["elements"]["generator"][gen]["capital_multiplier"] = 1 - self.md.data["elements"]["generator"][gen]["extension_multiplier"] = 0 - self.md.data["elements"]["generator"][gen]["max_operating_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["max_spinning_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["max_quickstart_reserve"] = 1 - self.md.data["elements"]["generator"][gen][ - "ramp_up_rate" - ] = 0.1 # FIXME should we update to only add if necessary - self.md.data["elements"]["generator"][gen][ - "ramp_down_rate" - ] = 0.1 # FIXME should we update to only add if necessary - self.md.data["elements"]["generator"][gen]["emissions_factor"] = 1 - self.md.data["elements"]["generator"][gen]["start_fuel"] = 1 - self.md.data["elements"]["generator"][gen]["investment_cost"] = 1 - for branch in self.md.data["elements"]["branch"]: - self.md.data["elements"]["branch"][branch]["loss_rate"] = 0 - self.md.data["elements"]["branch"][branch]["distance"] = 1 - self.md.data["elements"]["branch"][branch]["capital_cost"] = 10000000 - self.md.data["system"]["min_operating_reserve"] = 0.1 - self.md.data["system"]["min_spinning_reserve"] = 0.1 + if "elements" in self.md.data: + if "generator" in self.md.data["elements"]: + for gen in self.md.data["elements"]["generator"]: + if "fuel" in self.md.data["elements"]["generator"][gen]: + if self.md.data["elements"]["generator"][gen]["fuel"] == "C": + if ( + "in_service" + in self.md.data["elements"]["generator"][gen] + ): + if ( + self.md.data["elements"]["generator"][gen][ + "in_service" + ] + == False + ): + self.md.data["elements"]["generator"][gen][ + "lifetime" + ] = 1 + else: + self.md.data["elements"]["generator"][gen][ + "lifetime" + ] = 2 + else: + self.md.data["elements"]["generator"][gen]["lifetime"] = 3 + self.md.data["elements"]["generator"][gen]["lifetime"] = 3 + + self.md.data["elements"]["generator"][gen][ + "spinning_reserve_frac" + ] = 0.1 + self.md.data["elements"]["generator"][gen][ + "quickstart_reserve_frac" + ] = 0.1 + self.md.data["elements"]["generator"][gen]["capital_multiplier"] = 1 + self.md.data["elements"]["generator"][gen][ + "extension_multiplier" + ] = 0 + self.md.data["elements"]["generator"][gen][ + "max_operating_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen][ + "max_spinning_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen][ + "max_quickstart_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen]["ramp_up_rate"] = 0.1 + self.md.data["elements"]["generator"][gen]["ramp_down_rate"] = 0.1 + self.md.data["elements"]["generator"][gen]["emissions_factor"] = 1 + self.md.data["elements"]["generator"][gen]["start_fuel"] = 1 + self.md.data["elements"]["generator"][gen]["investment_cost"] = 1 + if "branch" in self.md.data["elements"]: + for branch in self.md.data["elements"]["branch"]: + self.md.data["elements"]["branch"][branch]["loss_rate"] = 0 + self.md.data["elements"]["branch"][branch]["distance"] = 1 + self.md.data["elements"]["branch"][branch][ + "capital_cost" + ] = 10000000 + if "system" in self.md.data: + self.md.data["system"]["min_operating_reserve"] = 0.1 + self.md.data["system"]["min_spinning_reserve"] = 0.1 def load_storage_csv(self, data_path): + """Imports storage data. + + :param data_path: filepath for storage data csv file + + """ try: storage_path = data_path + "/storage.csv" storage_df = pd.read_csv(storage_path) @@ -305,6 +349,12 @@ def load_storage_csv(self, data_path): self.md.data["elements"]["storage"] = {} def texas_case_study_updates(self, data_path): + """Imports generator data for texas case study. + + :param data_path: filepath for generator data csv file + + """ + # check that datapath is coming from a texas case study directory if "Texas" or "Coal" not in data_path: raise ValueError("The data path provided is not a Texas case study") diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 29f15fe1..b25dbaca 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -61,7 +61,15 @@ def read_debug_model( dataObject = ExpansionPlanningData( stages, num_reps, len_reps, num_commit, num_dispatch, duration_dispatch ) - dataObject.load_prescient(debug_data_path) + dataObject.load_prescient( + debug_data_path, + [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", + ], + ) return dataObject diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 0683cc7a..2385156c 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -39,7 +39,15 @@ def get_solution_object(): num_commit=6, num_dispatch=4, ) - data_object.load_prescient(input_data_source) + data_object.load_prescient( + input_data_source, + [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", + ], + ) mod_object = ExpansionPlanningModel(data=data_object) mod_object.create_model() From ee637cf1b198cd69a0acc648180e9ce5509924de Mon Sep 17 00:00:00 2001 From: Belle Date: Mon, 16 Mar 2026 12:13:28 -0600 Subject: [PATCH 10/31] update lifetime value assignment --- gtep/gtep_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 56232ff2..317b4519 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -268,7 +268,10 @@ def load_default_data_settings(self): if "elements" in self.md.data: if "generator" in self.md.data["elements"]: for gen in self.md.data["elements"]["generator"]: + # set lifetime value to default first + self.md.data["elements"]["generator"][gen]["lifetime"] = 3 if "fuel" in self.md.data["elements"]["generator"][gen]: + # update lifetime value if checks pass if self.md.data["elements"]["generator"][gen]["fuel"] == "C": if ( "in_service" @@ -287,9 +290,6 @@ def load_default_data_settings(self): self.md.data["elements"]["generator"][gen][ "lifetime" ] = 2 - else: - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 self.md.data["elements"]["generator"][gen][ "spinning_reserve_frac" From 83abe4f249f5fec3d12cfacd738511de004babb4 Mon Sep 17 00:00:00 2001 From: bstorm Date: Wed, 25 Mar 2026 14:47:08 -0600 Subject: [PATCH 11/31] move inputs to data instead of model object --- gtep/gtep_data.py | 17 +++++++-- gtep/gtep_model.py | 18 ++++------ gtep/tests/unit/test_gtep_model.py | 57 ++++++++++++++++++------------ gtep/tests/unit/test_validation.py | 11 +++--- 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 75baa4eb..3f6e7ef4 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -27,8 +27,21 @@ class ExpansionPlanningData: """Standard data storage class for the IDAES GTEP model.""" - def __init__(self): - pass + def __init__( + self, + stages=2, + num_reps=4, + len_reps=1, + num_commit=24, + num_dispatch=1, + duration_dispatch=60, + ): + self.stages = stages + self.num_reps = num_reps + self.len_reps = len_reps + self.num_commit = num_commit + self.num_dispatch = num_dispatch + self.duration_dispatch = duration_dispatch def load_prescient(self, data_path, options_dict=None): """Loads data structured via Prescient data loader. diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index c2c7ba57..3d8731f4 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -72,15 +72,9 @@ class ExpansionPlanningModel: def __init__( self, config=None, - stages=1, formulation=None, data=None, cost_data=None, - num_reps=3, - len_reps=24, - num_commit=24, - num_dispatch=4, - duration_dispatch=15, ): """Initialize generation & expansion planning model object. @@ -96,15 +90,15 @@ def __init__( :return: Pyomo model for full GTEP """ - self.stages = stages + self.stages = data.stages self.formulation = formulation self.data = data self.cost_data = cost_data - self.num_reps = num_reps - self.len_reps = len_reps - self.num_commit = num_commit - self.num_dispatch = num_dispatch - self.duration_dispatch = duration_dispatch + self.num_reps = data.num_reps + self.len_reps = data.len_reps + self.num_commit = data.num_commit + self.num_dispatch = data.num_dispatch + self.duration_dispatch = data.duration_dispatch self.config = _get_model_config() self.timer = TicTocTimer() diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 68796369..94832ec5 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -47,10 +47,24 @@ def patch_unit_handlers(): # Helper functions -def read_debug_model(): +def read_debug_model( + stages=2, + num_reps=4, + len_reps=16, + num_commit=12, + num_dispatch=12, + duration_dispatch=15, +): curr_dir = dirname(abspath(__file__)) debug_data_path = abspath(join(curr_dir, "..", "..", "data", "5bus")) - dataObject = ExpansionPlanningData() + dataObject = ExpansionPlanningData( + stages=stages, + num_reps=num_reps, + len_reps=len_reps, + num_commit=num_commit, + num_dispatch=num_dispatch, + duration_dispatch=duration_dispatch, + ) dataObject.load_prescient(debug_data_path) return dataObject @@ -77,11 +91,6 @@ def test_model_init(self): # properly with non-default values modObject = ExpansionPlanningModel( data=data_object, - stages=2, - num_reps=4, - len_reps=16, - num_commit=12, - num_dispatch=12, ) self.assertIsInstance(modObject, ExpansionPlanningModel) modObject.create_model() @@ -125,15 +134,16 @@ def test_model_init(self): def test_model_unit_consistency(self): # Test that the ExpansionPlanningModel has consistent units and spot check that # components have their expected units - data_object = read_debug_model() - modObject = ExpansionPlanningModel( - data=data_object, + data_object = read_debug_model( stages=2, num_reps=2, len_reps=2, num_commit=2, num_dispatch=2, ) + modObject = ExpansionPlanningModel( + data=data_object, + ) modObject.create_model() m = modObject.model @@ -187,10 +197,10 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is - data_object = read_debug_model() - modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + data_object = read_debug_model( + stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 ) + modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() # Check for consistent units @@ -214,9 +224,11 @@ def test_solve_bigm(self): def test_no_investment(self): # Solve the debug model with no investment - data_object = read_debug_model() + data_object = read_debug_model( + stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + ) modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + data=data_object, ) modObject.config["include_investment"] = False modObject.create_model() @@ -262,7 +274,14 @@ def test_with_cost_data(self): "Land-Based Wind", ] - data_processing_object = DataProcessing() + data_processing_object = DataProcessing( + stages=2, + num_reps=2, + len_reps=1, + num_commit=6, + num_dispatch=4, + duration_dispatch=15, + ) data_processing_object.load_gen_data( bus_data_path=bus_data_path, cost_data_path=cost_data_path, @@ -271,14 +290,8 @@ def test_with_cost_data(self): # Populate and create GTEP model mod_object = ExpansionPlanningModel( - stages=2, data=data_object, cost_data=data_processing_object, - num_reps=2, - len_reps=1, - num_commit=6, - num_dispatch=4, - duration_dispatch=15, ) mod_object.config["include_investment"] = True diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 5dc19b10..20ee6b97 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -32,17 +32,18 @@ def get_solution_object(): - data_object = ExpansionPlanningData() - data_object.load_prescient(input_data_source) - - mod_object = ExpansionPlanningModel( + data_object = ExpansionPlanningData( stages=2, - data=data_object, num_reps=2, len_reps=1, num_commit=6, num_dispatch=4, ) + data_object.load_prescient(input_data_source) + + mod_object = ExpansionPlanningModel( + data=data_object, + ) mod_object.create_model() TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) TransformationFactory("gdp.bigm").apply_to(mod_object.model) From ef30b038ab47c570763884e561eb56d8d80f02a5 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 09:53:07 -0600 Subject: [PATCH 12/31] correct inputs --- gtep/tests/unit/test_gtep_model.py | 50 ++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 94832ec5..b792d056 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -48,11 +48,11 @@ def patch_unit_handlers(): # Helper functions def read_debug_model( - stages=2, - num_reps=4, - len_reps=16, - num_commit=12, - num_dispatch=12, + stages=1, + num_reps=3, + len_reps=24, + num_commit=24, + num_dispatch=4, duration_dispatch=15, ): curr_dir = dirname(abspath(__file__)) @@ -74,7 +74,13 @@ class TestGTEP(unittest.TestCase): def test_model_init(self): # Test that the ExpansionPlanningModel object can read a default dataset and init # properly with default values, including building a Pyomo.ConcreteModel object - data_object = read_debug_model() + data_object = read_debug_model( + stages=1, + num_reps=3, + len_reps=24, + num_commit=24, + num_dispatch=4, + ) modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) modObject.create_model() @@ -89,6 +95,13 @@ def test_model_init(self): # Test that the ExpansionPlanningModel object can read a default dataset and init # properly with non-default values + data_object = read_debug_model( + stages=2, + num_reps=4, + len_reps=16, + num_commit=12, + num_dispatch=12, + ) modObject = ExpansionPlanningModel( data=data_object, ) @@ -198,7 +211,7 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is data_object = read_debug_model( - stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -225,7 +238,10 @@ def test_solve_bigm(self): def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model( - stages=1, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + num_reps=1, + len_reps=1, + num_commit=1, + num_dispatch=1, ) modObject = ExpansionPlanningModel( data=data_object, @@ -256,7 +272,14 @@ def test_no_investment(self): def test_with_cost_data(self): # Test ExpansionPlanningModel with cost data # This model originated from driver_esr.py - data_object = read_debug_model() + data_object = read_debug_model( + stages=2, + num_reps=2, + len_reps=1, + num_commit=6, + num_dispatch=4, + duration_dispatch=15, + ) curr_dir = dirname(abspath(__file__)) data_path = abspath(join(curr_dir, "..", "..", "data", "costs")) @@ -274,14 +297,7 @@ def test_with_cost_data(self): "Land-Based Wind", ] - data_processing_object = DataProcessing( - stages=2, - num_reps=2, - len_reps=1, - num_commit=6, - num_dispatch=4, - duration_dispatch=15, - ) + data_processing_object = DataProcessing() data_processing_object.load_gen_data( bus_data_path=bus_data_path, cost_data_path=cost_data_path, From 2714f2802ea8c90a881425b9b6f17e90430b9ffb Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 13:24:55 -0600 Subject: [PATCH 13/31] change weights calculation and make representative dates an input --- gtep/gtep_data.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 3f6e7ef4..daf61068 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -43,7 +43,18 @@ def __init__( self.num_dispatch = num_dispatch self.duration_dispatch = duration_dispatch - def load_prescient(self, data_path, options_dict=None): + def load_prescient( + self, + data_path, + representative_dates=[ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", ## Change the last date for whatever extreme day is needed based on the given run(s) + ], + representative_weights={}, + options_dict=None, + ): """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded @@ -102,26 +113,19 @@ def load_prescient(self, data_path, options_dict=None): ## of modelData objects, not just a single modelData object # Arbitrary time points and lengths picked for representative periods # default here allows up to 24 hours for periods + self.representative_dates = representative_dates - ## RMA: - ## Change the last date for whatever extreme day is needed based on the given run(s) + if not representative_weights: + # set the weight for each day to the total weight divided by number of days + total_weight = prescient_options.num_days * self.stages + weight_per_date = int(total_weight / (len(representative_dates))) + self.representative_weights = { + key: weight_per_date + for date, key in enumerate(self.representative_dates) + } time_keys = self.md.data["system"]["time_keys"] - self.representative_dates = [ - "2020-01-28 00:00", - "2020-04-23 00:00", - "2020-07-05 00:00", - "2020-10-14 00:00", - ] - ## FIXME: - ## RESIL WEEK ONLY - ## but we'll want something similar just less insane in the future - if len(self.representative_dates) == 5: - self.representative_weights = {1: 91, 2: 91, 3: 91, 4: 91, 5: 1} - else: - self.representative_weights = {1: 91, 2: 91, 3: 91, 4: 91} - # self.representative_weights = {1:1} for date in self.representative_dates: key_idx = time_keys.index(date) time_key_set = time_keys[key_idx : key_idx + 24] From 2101b2fefac996041cfdcc5975a748d0437cadba Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 13:54:55 -0600 Subject: [PATCH 14/31] correct options dictionary input --- gtep/gtep_data.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index daf61068..e205865a 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -61,17 +61,21 @@ def load_prescient( :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None """ self.data_type = "prescient" - options_dict = { - "data_path": data_path, - "input_format": "rts-gmlc", - "start_date": "01-01-2020", - "num_days": 365, - "sced_horizon": 1, - "sced_frequency_minutes": 60, - "ruc_horizon": 36, - } - + # create prescient config object with defaults prescient_options = PrescientConfig() + + if options_dict is None: + # set basic configurations that do not match prescient defaults + options_dict = { + "data_path": data_path, + "num_days": 365, + "ruc_horizon": 36, + } + + else: + # ensure data path is included in options dictionary + options_dict["data_path"] = data_path + prescient_options.set_value(options_dict) # Use prescient data provider to load in sequential data for representative periods data_list = [] From f63cf8c04f455b9c5e054002cb99570e69489f16 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 13:57:31 -0600 Subject: [PATCH 15/31] Add comment description for inputs --- gtep/gtep_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index e205865a..33e2ff49 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -36,6 +36,15 @@ def __init__( num_dispatch=1, duration_dispatch=60, ): + """Initialize generation & expansion planning data object. + + :param stages: integer number of investment periods + :param num_reps: integer number of representative periods per investment period + :param len_reps: (for now integer) length of each representative period (in hours) + :param num_commit: integer number of commitment periods per representative period + :param num_dispatch: integer number of dispatch periods per commitment period + :param duration_dispatch: (for now integer) duration of each dispatch period (in minutes) + """ self.stages = stages self.num_reps = num_reps self.len_reps = len_reps @@ -58,7 +67,10 @@ def load_prescient( """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded + :param representative_dates: List of time points to include Note: Change the last date for whatever extreme day is needed based on the given run(s) + :param representative_weights: dictionary of weights for each representative date, defaults to empty Dict :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None + """ self.data_type = "prescient" # create prescient config object with defaults From 54ae2587480f532b3bd68caa846b7a429cf4ecf5 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 14:42:57 -0600 Subject: [PATCH 16/31] comments added --- gtep/gtep_model.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 3d8731f4..a5b873dd 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -78,15 +78,10 @@ def __init__( ): """Initialize generation & expansion planning model object. - :param stages: integer number of investment periods :param formulation: Egret stuff, to be filled :param data: full set of model data :param cost_data: full set of cost data for all generators - :param num_reps: integer number of representative periods per investment period - :param len_reps: (for now integer) length of each representative period (in hours) - :param num_commit: integer number of commitment periods per representative period - :param num_dispatch: integer number of dispatch periods per commitment period - :param duration_dispatch: (for now integer) duration of each dispatch period (in minutes) + :return: Pyomo model for full GTEP """ From 69bcbb60136f3aadb25323f812a09ab620f94f84 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:11:29 -0600 Subject: [PATCH 17/31] pull data from simulations_objects.csv --- gtep/gtep_data.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 33e2ff49..cc6f9ee3 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -22,6 +22,7 @@ from prescient.data.providers import gmlc_data_provider import datetime import pandas as pd +import os class ExpansionPlanningData: @@ -88,16 +89,31 @@ def load_prescient( # ensure data path is included in options dictionary options_dict["data_path"] = data_path + # update configuration values based on options dictionary prescient_options.set_value(options_dict) # Use prescient data provider to load in sequential data for representative periods data_list = [] - x = datetime.datetime(2020, 1, 1) data_provider = gmlc_data_provider.GmlcDataProvider(options=prescient_options) + + # grab details from simulation objects file (data provider above throws error if no simulation_objects.csv exists) + metadata_path = os.path.join(data_path, "simulation_objects.csv") + metadata_df = pd.read_csv(metadata_path, index_col=0) + + # save to variable for easy calling + sced_freq_min = prescient_options.sced_frequency_minutes + + # This step is grabbing DAY_AHEAD information for now + # (in the future we may want to update to grab the "REAL_TIME" data if the data has reliable data since the actuals model is looking for real time data info) + period_per_step = int(metadata_df.loc["Periods_per_Step"]["DAY_AHEAD"]) + total_num_steps = prescient_options.num_days * period_per_step + + x = datetime.datetime(2020, 1, 1) # populate an egret model data with the basic stuff self.md = data_provider.get_initial_actuals_model( options=prescient_options, num_time_steps=24 * 365, minutes_per_timestep=60 ) + # fill in renewable actuals and maybe demand idk data_provider.populate_with_actuals( options=prescient_options, From 28f1bc4e54772a978181cb2afbc564a6c3ccaf74 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:13:57 -0600 Subject: [PATCH 18/31] update egred model population and actuals function with simulations_objects.csv info --- gtep/gtep_data.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index cc6f9ee3..82710d74 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -108,18 +108,19 @@ def load_prescient( period_per_step = int(metadata_df.loc["Periods_per_Step"]["DAY_AHEAD"]) total_num_steps = prescient_options.num_days * period_per_step - x = datetime.datetime(2020, 1, 1) # populate an egret model data with the basic stuff self.md = data_provider.get_initial_actuals_model( - options=prescient_options, num_time_steps=24 * 365, minutes_per_timestep=60 + options=prescient_options, + num_time_steps=total_num_steps, + minutes_per_timestep=sced_freq_min, ) - # fill in renewable actuals and maybe demand idk + # fill in renewable actuals data_provider.populate_with_actuals( options=prescient_options, - num_time_periods=24 * 365, - time_period_length_minutes=60, - start_time=x, + num_time_periods=total_num_steps, + time_period_length_minutes=sced_freq_min, + start_time=data_provider._start_time, model=self.md, ) From 096bb1e5059097e1a07b28769aa2e2dafac03fc7 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:19:51 -0600 Subject: [PATCH 19/31] pull time key set from simulation_object periods per step --- gtep/gtep_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 82710d74..863bfcf2 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -161,7 +161,7 @@ def load_prescient( for date in self.representative_dates: key_idx = time_keys.index(date) - time_key_set = time_keys[key_idx : key_idx + 24] + time_key_set = time_keys[key_idx : key_idx + period_per_step] data_list.append(self.md.clone_at_time_keys(time_key_set)) self.representative_data = data_list From 62f09d3afcdb64e9da875f11f696c69e17b7062a Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:22:46 -0600 Subject: [PATCH 20/31] make forecast years an input --- gtep/gtep_data.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 863bfcf2..f55f96f4 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -166,13 +166,29 @@ def load_prescient( self.representative_data = data_list - def import_load_scaling(self, load_file_name): + def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]): + """Imports load scaling data for forecast years. + + :param load_file_name: filepath for adjusted forecast excel file + :param forecast_years: list of years to forecast, defaults to [2025, 2030, 2035] + + """ adjusted_forecast = pd.read_excel(load_file_name) + + # check years are valid + if len(forecast_years) < self.stages: + raise ValueError( + "Not enough forecast years for the number of stages of investment" + ) + elif any(year < 2020 or year > 2050 for year in forecast_years): + raise ValueError( + "The list of years includes a year before 2020 or after 2050." + ) + adjusted_forecast_by_period = adjusted_forecast[ - (adjusted_forecast["year"] == 2025) - | (adjusted_forecast["year"] == 2030) - | (adjusted_forecast["year"] == 2035) + adjusted_forecast["year"].isin(forecast_years) ] + base_zones = [ "base_economic_coast", "base_economic_east", From 76e1770c364e95ce49fd253b7469378ad3aa5bbe Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:24:31 -0600 Subject: [PATCH 21/31] add comment and remove unused import --- gtep/gtep_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index f55f96f4..1826e30b 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -239,11 +239,15 @@ def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]) self.load_scaling = load_scaling_df def import_outage_data(self, load_file_name): + """Imports outage data. + + :param load_file_name: filepath for adjusted forecast excel file + + """ outage_list = pd.read_csv(load_file_name) percentile_threshold = 0.9 threshold_value = outage_list["case_4b_prob"].quantile(percentile_threshold) filtered_outages = outage_list[outage_list["case_4b_prob"] >= threshold_value] - import re filtered_outages["hour"] = filtered_outages["lim_timestamp"].str.extract( r" (\d+):" From 2990c9a4594059a88d8a3cdf25af452c332fcde6 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:25:49 -0600 Subject: [PATCH 22/31] add error for using a nonTexas data set for texas case study function --- gtep/gtep_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 1826e30b..2c3a5ccb 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -323,6 +323,14 @@ def load_storage_csv(self, data_path): self.md.data["elements"]["storage"] = {} def texas_case_study_updates(self, data_path): + """Imports generator data for texas case study. + + :param data_path: filepath for generator data csv file + """ + # check that datapath is coming from a texas case study directory + if "Texas" or "Coal" not in data_path: + raise ValueError("The data path provided is not a Texas case study") + generator_update_path = data_path + "/gen.csv" generator_df = pd.read_csv(generator_update_path) bonus_feature_list = [ From f35fb88f8dad34c83c0af131bc635fca7a57e854 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:27:37 -0600 Subject: [PATCH 23/31] add param comment --- gtep/gtep_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 2c3a5ccb..52f3bffc 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -306,6 +306,10 @@ def load_default_data_settings(self): self.md.data["system"]["min_spinning_reserve"] = 0.1 def load_storage_csv(self, data_path): + """Imports storage data. + + :param data_path: filepath for storage data csv file + """ try: storage_path = data_path + "/storage.csv" storage_df = pd.read_csv(storage_path) From d510ff8b57e86f5e47c0dd6ad76e7e57fd685500 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 15:40:48 -0600 Subject: [PATCH 24/31] add checks for existing keys in default settings --- gtep/gtep_data.py | 82 +++++++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 52f3bffc..97596272 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -275,35 +275,61 @@ def import_outage_data(self, load_file_name): self.bus_hours = self.bus_hours.astype(int) def load_default_data_settings(self): - ## TODO: too many of these are hard coded; everything should check if it exists too. + ##many of these are hard coded, but they are not set later in the process as of now """Fills in necessary but unspecified data information.""" - for gen in self.md.data["elements"]["generator"]: - if self.md.data["elements"]["generator"][gen]["fuel"] == "C": - if self.md.data["elements"]["generator"][gen]["in_service"] == False: - self.md.data["elements"]["generator"][gen]["lifetime"] = 1 - else: - self.md.data["elements"]["generator"][gen]["lifetime"] = 2 - else: - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 - self.md.data["elements"]["generator"][gen]["lifetime"] = 3 - self.md.data["elements"]["generator"][gen]["spinning_reserve_frac"] = 0.1 - self.md.data["elements"]["generator"][gen]["quickstart_reserve_frac"] = 0.1 - self.md.data["elements"]["generator"][gen]["capital_multiplier"] = 1 - self.md.data["elements"]["generator"][gen]["extension_multiplier"] = 0 - self.md.data["elements"]["generator"][gen]["max_operating_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["max_spinning_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["max_quickstart_reserve"] = 1 - self.md.data["elements"]["generator"][gen]["ramp_up_rate"] = 0.1 - self.md.data["elements"]["generator"][gen]["ramp_down_rate"] = 0.1 - self.md.data["elements"]["generator"][gen]["emissions_factor"] = 1 - self.md.data["elements"]["generator"][gen]["start_fuel"] = 1 - self.md.data["elements"]["generator"][gen]["investment_cost"] = 1 - for branch in self.md.data["elements"]["branch"]: - self.md.data["elements"]["branch"][branch]["loss_rate"] = 0 - self.md.data["elements"]["branch"][branch]["distance"] = 1 - self.md.data["elements"]["branch"][branch]["capital_cost"] = 10000000 - self.md.data["system"]["min_operating_reserve"] = 0.1 - self.md.data["system"]["min_spinning_reserve"] = 0.1 + if "elements" in self.md.data.keys(): + if "generator" in self.md.data["elements"].keys(): + for gen in self.md.data["elements"]["generator"]: + # set lifetime value to default first + self.md.data["elements"]["generator"][gen]["lifetime"] = 3 + if "fuel" in self.md.data["elements"]["generator"][gen].keys(): + if self.md.data["elements"]["generator"][gen]["fuel"] == "C": + if ( + self.md.data["elements"]["generator"][gen]["in_service"] + == False + ): + self.md.data["elements"]["generator"][gen][ + "lifetime" + ] = 1 + else: + self.md.data["elements"]["generator"][gen][ + "lifetime" + ] = 2 + + self.md.data["elements"]["generator"][gen][ + "spinning_reserve_frac" + ] = 0.1 + self.md.data["elements"]["generator"][gen][ + "quickstart_reserve_frac" + ] = 0.1 + self.md.data["elements"]["generator"][gen]["capital_multiplier"] = 1 + self.md.data["elements"]["generator"][gen][ + "extension_multiplier" + ] = 0 + self.md.data["elements"]["generator"][gen][ + "max_operating_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen][ + "max_spinning_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen][ + "max_quickstart_reserve" + ] = 1 + self.md.data["elements"]["generator"][gen]["ramp_up_rate"] = 0.1 + self.md.data["elements"]["generator"][gen]["ramp_down_rate"] = 0.1 + self.md.data["elements"]["generator"][gen]["emissions_factor"] = 1 + self.md.data["elements"]["generator"][gen]["start_fuel"] = 1 + self.md.data["elements"]["generator"][gen]["investment_cost"] = 1 + if "branch" in self.md.data["elements"].keys(): + for branch in self.md.data["elements"]["branch"]: + self.md.data["elements"]["branch"][branch]["loss_rate"] = 0 + self.md.data["elements"]["branch"][branch]["distance"] = 1 + self.md.data["elements"]["branch"][branch][ + "capital_cost" + ] = 10000000 + if "system" in self.md.data.keys(): + self.md.data["system"]["min_operating_reserve"] = 0.1 + self.md.data["system"]["min_spinning_reserve"] = 0.1 def load_storage_csv(self, data_path): """Imports storage data. From e91b3a5fb6af623bdcedabc301d66b35879adfc3 Mon Sep 17 00:00:00 2001 From: bstorm Date: Thu, 26 Mar 2026 16:38:59 -0600 Subject: [PATCH 25/31] fix defaults in test_gtep_model data input --- gtep/tests/unit/test_gtep_model.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index dc1d7f6d..f44d2745 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -80,18 +80,19 @@ def test_model_init(self): len_reps=24, num_commit=24, num_dispatch=4, + duration_dispatch=60, ) modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) modObject.create_model() self.assertIsInstance(modObject.model, ConcreteModel) - self.assertEqual(modObject.stages, 2) + self.assertEqual(modObject.stages, 1) self.assertEqual(modObject.formulation, None) self.assertIsInstance(modObject.model.md, ModelData) - self.assertEqual(modObject.num_reps, 4) - self.assertEqual(modObject.len_reps, 1) + self.assertEqual(modObject.num_reps, 3) + self.assertEqual(modObject.len_reps, 24) self.assertEqual(modObject.num_commit, 24) - self.assertEqual(modObject.num_dispatch, 1) + self.assertEqual(modObject.num_dispatch, 4) self.assertEqual(modObject.duration_dispatch, 60) # Test that the ExpansionPlanningModel object can read a default dataset and init From 3f71e84ebc29f88f223d354a7c8fcc3dd2c4d1f2 Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 23 Apr 2026 09:09:39 -0600 Subject: [PATCH 26/31] correct defaults --- gtep/gtep_data.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 61bb624b..daa352f6 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -55,12 +55,7 @@ def __init__( def load_prescient( self, data_path, - representative_dates=[ - "2020-01-28 00:00", - "2020-04-23 00:00", - "2020-07-05 00:00", - "2020-10-14 00:00", ## Change the last date for whatever extreme day is needed based on the given run(s) - ], + representative_dates=None, representative_weights={}, options_dict=None, ): @@ -146,6 +141,13 @@ def load_prescient( ## of modelData objects, not just a single modelData object # Arbitrary time points and lengths picked for representative periods # default here allows up to 24 hours for periods + if self.representative_dates is None: + representative_dates = [ + "2020-01-28 00:00", + "2020-04-23 00:00", + "2020-07-05 00:00", + "2020-10-14 00:00", ## Change the last date for whatever extreme day is needed based on the given run(s) + ] self.representative_dates = representative_dates if not representative_weights: @@ -166,7 +168,7 @@ def load_prescient( self.representative_data = data_list - def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]): + def import_load_scaling(self, load_file_name, forecast_years=None): """Imports load scaling data for forecast years. :param load_file_name: filepath for adjusted forecast excel file @@ -175,6 +177,9 @@ def import_load_scaling(self, load_file_name, forecast_years=[2025, 2030, 2035]) """ adjusted_forecast = pd.read_excel(load_file_name) + if forecast_years is None: + forecast_years = [2025, 2030, 2035] + # check years are valid if len(forecast_years) < self.stages: raise ValueError( From 4d62cacce78dfb5ca2e7f129651af02aee69aaf3 Mon Sep 17 00:00:00 2001 From: belle-storm Date: Thu, 23 Apr 2026 16:12:04 -0600 Subject: [PATCH 27/31] Update gtep/gtep_data.py Co-authored-by: Bethany Nicholson --- gtep/gtep_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index daa352f6..78e5c9af 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -62,7 +62,7 @@ def load_prescient( """Loads data structured via Prescient data loader. :param data_path: Folder containing the data to be loaded - :param representative_dates: List of time points to include Note: Change the last date for whatever extreme day is needed based on the given run(s) + :param representative_dates: List of time points to include. Note: Change the last date for whatever extreme day is needed based on the given run(s) :param representative_weights: dictionary of weights for each representative date, defaults to empty Dict :param options_dict: Options dictionary to pass to the Prescient data loader, defaults to None From 6997781d6d1c8aa67444d9072367fc2000d6439b Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 23 Apr 2026 16:28:26 -0600 Subject: [PATCH 28/31] add prescient workaround --- gtep/gtep_data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 78e5c9af..e165984c 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -22,6 +22,7 @@ from prescient.data.providers import gmlc_data_provider import pandas as pd import os +from pathlib import Path class ExpansionPlanningData: @@ -71,6 +72,10 @@ def load_prescient( # create prescient config object with defaults prescient_options = PrescientConfig() + # work around for prescient throwing an error with Path objects + if isinstance(data_path, Path): + data_path = str(data_path) + if options_dict is None: # set basic configurations that do not match prescient defaults options_dict = { From e511ae3997c4242f8c4ea172f5c6e9a0844c2a2a Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 23 Apr 2026 16:31:48 -0600 Subject: [PATCH 29/31] representative date assignment fix --- gtep/gtep_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index e165984c..251fcbc6 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -146,7 +146,7 @@ def load_prescient( ## of modelData objects, not just a single modelData object # Arbitrary time points and lengths picked for representative periods # default here allows up to 24 hours for periods - if self.representative_dates is None: + if representative_dates is None: representative_dates = [ "2020-01-28 00:00", "2020-04-23 00:00", From 8d11bfe9305fa4cf52537d40f487b3d90d00aa6e Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 23 Apr 2026 16:35:04 -0600 Subject: [PATCH 30/31] correct test_gtep_model after syncing fork --- gtep/tests/unit/test_gtep_model.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 9eddcda6..f4b78e86 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -69,9 +69,23 @@ def read_debug_model( return dataObject -def prepare_model_and_cost_data(): +def prepare_model_and_cost_data( + stages=1, + num_reps=3, + len_reps=24, + num_commit=24, + num_dispatch=4, + duration_dispatch=15, +): # Prepare model and cost data - dataObject = read_debug_model() + dataObject = read_debug_model( + stages, + num_reps, + len_reps, + num_commit, + num_dispatch, + duration_dispatch, + ) curr_dir = dirname(abspath(__file__)) data_path = abspath(join(curr_dir, "..", "..", "data", "costs")) bus_data_path = abspath(join(data_path, "Bus_data_gen_weights_mappings.csv")) @@ -306,7 +320,7 @@ def test_no_investment(self): def test_with_cost_data_and_commitment(self): # Test ExpansionPlanningModel with cost data # This model originated from driver_esr.py - data_object = read_debug_model( + dataObject, dataProcessingObject = prepare_model_and_cost_data( stages=2, num_reps=2, len_reps=1, @@ -357,9 +371,9 @@ def test_with_cost_data_and_no_commitment(self): dataObject, dataProcessingObject = prepare_model_and_cost_data() # Populate and create GTEP model - mod_object = ExpansionPlanningModel( - data=data_object, - cost_data=data_processing_object, + modObject = ExpansionPlanningModel( + data=dataObject, + cost_data=dataProcessingObject, ) modObject.config["include_investment"] = True From f734fa69ac18527aa248a0dc75a8236ea63763dc Mon Sep 17 00:00:00 2001 From: Belle Date: Thu, 23 Apr 2026 16:47:05 -0600 Subject: [PATCH 31/31] fix data inputs on test_gtep_model.py --- gtep/tests/unit/test_gtep_model.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index f4b78e86..58cf77d7 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -368,7 +368,14 @@ def test_with_cost_data_and_commitment(self): def test_with_cost_data_and_no_commitment(self): # Test ExpansionPlanningModel with cost data and no commitment # This model originated from driver_esr.py - dataObject, dataProcessingObject = prepare_model_and_cost_data() + dataObject, dataProcessingObject = prepare_model_and_cost_data( + stages=2, + num_reps=2, + len_reps=1, + num_commit=6, + num_dispatch=4, + duration_dispatch=15, + ) # Populate and create GTEP model modObject = ExpansionPlanningModel(