From 91f1223b201085a3c0bddaf80d1fe26efb4d4135 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 11:00:27 -0700 Subject: [PATCH 01/20] Remove period length from ramping constraints since ramp rates are dimensionless (do not need to be multiplied by any time length) --- gtep/model_library/gen.py | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/gtep/model_library/gen.py b/gtep/model_library/gen.py index ae0530c0..dbdde32a 100644 --- a/gtep/model_library/gen.py +++ b/gtep/model_library/gen.py @@ -281,9 +281,7 @@ def ramp_up_limits(disj, dispatchPeriod, generator): return ( b.dispatchPeriod[dispatchPeriod].thermalGeneration[generator] - b.dispatchPeriod[dispatchPeriod - 1].thermalGeneration[generator] - <= m.rampUpRates[generator] - * b.dispatchPeriod[dispatchPeriod].periodLength - * m.thermalCapacity[generator] + <= m.rampUpRates[generator] * m.thermalCapacity[generator] ) elif dispatchPeriod == 1 and commitment_period != 1: return ( @@ -291,9 +289,7 @@ def ramp_up_limits(disj, dispatchPeriod, generator): - r_p.commitmentPeriod[commitment_period - 1] .dispatchPeriod[b.dispatchPeriods.last()] .thermalGeneration[generator] - <= m.rampUpRates[generator] - * b.dispatchPeriod[dispatchPeriod].periodLength - * m.thermalCapacity[generator] + <= m.rampUpRates[generator] * m.thermalCapacity[generator] ) else: return pyo.Constraint.Skip @@ -309,7 +305,6 @@ def ramp_down_limits(disj, dispatchPeriod, generator): b.dispatchPeriod[dispatchPeriod - 1].thermalGeneration[generator] - b.dispatchPeriod[dispatchPeriod].thermalGeneration[generator] <= m.rampDownRates[generator] # in MW/min - * b.dispatchPeriod[dispatchPeriod].periodLength # in min * m.thermalCapacity[generator] # in MW ) elif dispatchPeriod == 1 and commitment_period != 1: @@ -319,7 +314,6 @@ def ramp_down_limits(disj, dispatchPeriod, generator): .thermalGeneration[generator] - b.dispatchPeriod[dispatchPeriod].thermalGeneration[generator] <= m.rampDownRates[generator] # in MW/min - * b.dispatchPeriod[dispatchPeriod].periodLength # in min * m.thermalCapacity[generator] # in MW ) else: @@ -369,12 +363,7 @@ def ramp_up_limits(disj, dispatchPeriod, generator): - b.dispatchPeriod[dispatchPeriod - 1].thermalGeneration[generator] <= max( pyo.value(m.thermalMin[generator]), - # [ESR: Make sure the time units are consistent - # here since we are only taking the value] pyo.value(m.rampUpRates[generator]) - * pyo.value( - b.dispatchPeriod[dispatchPeriod].periodLength - ) # in minutes * pyo.value(m.thermalCapacity[generator]), ) * u.MW @@ -387,12 +376,7 @@ def ramp_up_limits(disj, dispatchPeriod, generator): .thermalGeneration[generator] <= max( pyo.value(m.thermalMin[generator]), - # [ESR: Make sure the time units are consistent - # here since we are only taking the value] pyo.value(m.rampUpRates[generator]) - * pyo.value( - b.dispatchPeriod[dispatchPeriod].periodLength - ) # in minutes * pyo.value(m.thermalCapacity[generator]), ) * u.MW @@ -436,12 +420,7 @@ def ramp_down_limits(disj, dispatchPeriod, generator): - b.dispatchPeriod[dispatchPeriod].thermalGeneration[generator] <= max( pyo.value(m.thermalMin[generator]), - # [ESR: Make sure the time units are consistent - # here since we are taking the value only] pyo.value(m.rampDownRates[generator]) - * pyo.value( - b.dispatchPeriod[dispatchPeriod].periodLength - ) # in minutes * pyo.value(m.thermalCapacity[generator]), ) * u.MW @@ -454,12 +433,7 @@ def ramp_down_limits(disj, dispatchPeriod, generator): - b.dispatchPeriod[dispatchPeriod].thermalGeneration[generator] <= max( pyo.value(m.thermalMin[generator]), - # [ESR: Make sure the time units are consistent - # here since we are taking the value only] pyo.value(m.rampDownRates[generator]) - * pyo.value( - b.dispatchPeriod[dispatchPeriod].periodLength - ) # in minutes * pyo.value(m.thermalCapacity[generator]), ) * u.MW From 22047ddc92936951db5d3e57acefc049e879cabe Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 11:08:30 -0700 Subject: [PATCH 02/20] Replace periodLength parameters with new indexed parameters These indexed parameters take now a list of values for each stage. --- gtep/gtep_model.py | 91 +++++++++++++++++++++++------- gtep/model_library/commitment.py | 11 ++-- gtep/model_library/components.py | 14 +---- gtep/tests/unit/test_gtep_model.py | 47 +++++---------- 4 files changed, 89 insertions(+), 74 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 6292bc40..a888b05b 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -94,6 +94,8 @@ def __init__( len_reps=24, num_commit=24, num_dispatch=4, + duration_representative_period=24, + duration_commitment=1, duration_dispatch=15, ): """Initialize generation & expansion planning model object. @@ -118,13 +120,41 @@ def __init__( self.len_reps = len_reps self.num_commit = num_commit self.num_dispatch = num_dispatch - self.duration_dispatch = duration_dispatch self.config = _get_model_config() self.timer = TicTocTimer() + self.duration_representative_period = self._ensure_list_length( + "representative periods", duration_representative_period, num_reps, 24 + ) + self.duration_commitment = self._ensure_list_length( + "commitment", duration_commitment, num_commit, 1 + ) + self.duration_dispatch = self._ensure_list_length( + "dispatch", duration_dispatch, num_dispatch, 15 + ) _add_common_configs(self.config) _add_investment_configs(self.config) + @staticmethod + def _ensure_list_length(name, param, expected_length, default_value): + """This method ensures that any of the duration parameters is + a list of expected_length (equal to number of each + stage). If no value is given, a default value is used for + each time period. + + """ + if param is None: + return [default_value] * expected_length + elif isinstance(param, list): + if len(param) != expected_length: + print( + f"\n***WARNING: The list {param} for the duration_{name} parameter has a length ({len(param)}), which does not match the expected number of {name} periods ({expected_length}). We will use {default_value} as the default value.\n" + ) + return [default_value] * expected_length + return param + else: + return [param] * expected_length + def create_model(self): """Create concrete Pyomo model object associated with the ExpansionPlanningModel class @@ -166,10 +196,15 @@ def create_model(self): m, self.num_commit, self.num_dispatch, - self.duration_dispatch, ) - create_stages(m, self.stages) + create_stages( + m, + self.stages, + self.duration_representative_period, + self.duration_commitment, + self.duration_dispatch, + ) obj_comp.create_objective_function(m) @@ -214,7 +249,9 @@ def report_large_coefficients(self, outfile, magnitude_cutoff=1e5): json.dump(really_bad_var_coef_list, fil) -def create_stages(m, stages): +def create_stages( + m, stages, duration_representative_period, duration_commitment, duration_dispatch +): """This method constructs the block structure for the Generation and Transmission Expansion Planning (GTEP) model. It creates investment, representative period, and commitment blocks for each @@ -258,6 +295,13 @@ def create_stages(m, stages): # representative_period. for representative_period in b_inv.representativePeriods: b_rep = b_inv.representativePeriod[representative_period] + + b_rep.periodLength = pyo.Param( + initialize=duration_representative_period[representative_period - 1], + within=pyo.PositiveReals, + units=u.hr, + ) + b_rep.representative_date = m.data.representative_dates[ representative_period - 1 ] @@ -269,12 +313,10 @@ def create_stages(m, stages): # b_rep.day = int(broken_date[2]) b_rep.currentPeriod = representative_period - # [ESR NOTE: Include commitment blocks regardless of the - # value of include_commitment. When include_commitment is - # False, generators are assumed to be always online, and + # Include commitment blocks regardless of the value of + # include_commitment. When False, generators are on and # operational costs are determined solely by dispatch - # decisions. No redispatch for now.] - # if m.config["include_commitment"] or m.config["include_redispatch"]: + # decisions. b_rep.commitmentPeriods = pyo.RangeSet( m.numCommitmentPeriods[representative_period] ) @@ -285,6 +327,13 @@ def create_stages(m, stages): # constraints for commitment_period in b_rep.commitmentPeriods: b_comm = b_rep.commitmentPeriod[commitment_period] + + b_comm.periodLength = pyo.Param( + initialize=duration_commitment[commitment_period - 1], + within=pyo.PositiveReals, + units=u.hr, + ) + b_comm.commitmentPeriod = commitment_period if m.config["include_redispatch"]: @@ -310,21 +359,21 @@ def create_stages(m, stages): # [TODO: This feels REALLY inelegant and # bad. Check a better way of declaring these.] - for period in b_comm.dispatchPeriods: - b_comm.dispatchPeriod[period].periodLength = pyo.Param( - initialize=1, + for dispatch_period in b_comm.dispatchPeriods: + b_disp = b_comm.dispatchPeriod[dispatch_period] + + b_disp.periodLength = pyo.Param( + initialize=duration_dispatch[dispatch_period - 1], within=pyo.PositiveReals, - # units=u.minutes, + units=u.minutes, ) disp.add_dispatch_variables( - b_comm.dispatchPeriod[period], - period, - m.dispatchPeriodLength, - ) - disp.add_dispatch_constraints( - b_comm.dispatchPeriod[period], period + b_disp, + dispatch_period, + b_disp.periodLength, ) + disp.add_dispatch_constraints(b_disp, dispatch_period) # =.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=.=. @@ -349,8 +398,8 @@ def create_stages(m, stages): rep_period.add_representative_period_variables(b_rep, representative_period) if m.config["include_commitment"]: - # These logical constraint ensure the state - # disjuncts stay consistent. + # These logical constraints ensure the state disjuncts + # stay consistent. rep_period.add_representative_period_logical_constraints( b_rep, representative_period ) diff --git a/gtep/model_library/commitment.py b/gtep/model_library/commitment.py index a8be863b..6a24b5b8 100644 --- a/gtep/model_library/commitment.py +++ b/gtep/model_library/commitment.py @@ -28,9 +28,6 @@ def add_commitment_parameters(b, commitment_period, investmentStage): m = b.model() - b.commitmentPeriodLength = pyo.Param( - within=pyo.PositiveReals, default=1, units=u.hr - ) b.carbonTax = pyo.Param(default=0) # [ESR: Corrected to be in the commitment block "b", not in main @@ -117,7 +114,7 @@ def operatingCostCommitment(b): if m.config["include_commitment"]: op_cost_gen_state = sum( m.fixedCost[gen] - * b.commitmentPeriodLength + * b.periodLength * m.thermalCapacity[gen] * ( b.genOn[gen].indicator_var.get_associated_binary() @@ -136,7 +133,7 @@ def operatingCostCommitment(b): else: op_cost_gen_state = sum( m.fixedCost[gen] - * b.commitmentPeriodLength + * b.periodLength * m.thermalCapacity[gen] * b.genOn[gen].indicator_var.get_associated_binary() for gen in m.thermalGenerators @@ -174,7 +171,9 @@ def renewable_curtailment_cost(b): for com_per in b.representativePeriod[rep_per].commitmentPeriods: renewableCurtailmentRep += ( m.weights[rep_per] - * m.commitmentPeriodLength + * b.representativePeriod[rep_per] + .commitmentPeriod[com_per] + .periodLength * b.representativePeriod[rep_per] .commitmentPeriod[com_per] .renewableCurtailmentCommitment # in MW diff --git a/gtep/model_library/components.py b/gtep/model_library/components.py index 7d6bfdf1..d153d95a 100644 --- a/gtep/model_library/components.py +++ b/gtep/model_library/components.py @@ -106,7 +106,7 @@ def add_model_sets(m, stages, rep_per=["a", "b"], com_per=2, dis_per=2): ) -def add_model_parameters(m, num_commit, num_dispatch, duration_dispatch): +def add_model_parameters(m, num_commit, num_dispatch): """Creates and labels all the parameters in the GTEP model. This method ties input data directly to the model. @@ -120,9 +120,6 @@ def add_model_parameters(m, num_commit, num_dispatch, duration_dispatch): # Add parameters related to the representative periods for the # different stages - m.representativePeriodLength = pyo.Param( - m.representativePeriods, within=pyo.PositiveReals, default=24, units=u.hr - ) m.numCommitmentPeriods = pyo.Param( m.representativePeriods, within=pyo.PositiveIntegers, @@ -135,15 +132,6 @@ def add_model_parameters(m, num_commit, num_dispatch, duration_dispatch): default=2, initialize=num_dispatch, ) - m.commitmentPeriodLength = pyo.Param( - within=pyo.PositiveReals, default=1, units=u.hr - ) - - # [TODO: Index by dispatch period? Certainly index by - # commitment period.] - m.dispatchPeriodLength = pyo.Param( - within=pyo.PositiveReals, initialize=duration_dispatch, units=u.minutes - ) # Add power-related parameters m.thermalCapacity = pyo.Param( diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 0a9e4e4c..9cad8a55 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -169,47 +169,26 @@ def test_model_unit_consistency(self): assert_units_consistent(m) # Check that subset of model components have expected units + m_inv = m.investmentStage[1] + m_rep_period = m_inv.representativePeriod[1] + m_commit = m_rep_period.commitmentPeriod[1] + m_disp = m_commit.dispatchPeriod[1] + + assert_units_equivalent(m_rep_period.periodLength, u.h) + assert_units_equivalent(m_commit.periodLength, u.h) + assert_units_equivalent(m_disp.periodLength, u.min) assert_units_equivalent(m.renewable_capacity_enforcement[1, "10_PV"].expr, u.MW) - assert_units_equivalent( - m.investmentStage[1].renewable_curtailment_cost.expr, u.USD - ) - assert_units_equivalent( - m.investmentStage[1] - .representativePeriod[1] - .commitmentPeriod[1] - .dispatchPeriod[1] - .flow_balance["bus1"] - .expr, - u.MW, - ) - assert_units_equivalent(m.commitmentPeriodLength, u.h) - assert_units_equivalent(m.dispatchPeriodLength, u.min) + assert_units_equivalent(m_inv.renewable_curtailment_cost.expr, u.USD) + assert_units_equivalent(m_disp.flow_balance["bus1"].expr, u.MW) assert_units_equivalent(m.rampUpRates, u.dimensionless) assert_units_equivalent(m.varCost, u.USD / u.h / u.MW) + assert_units_equivalent(m_disp.spinningReserve, u.MW) assert_units_equivalent( - m.investmentStage[1] - .representativePeriod[1] - .commitmentPeriod[1] - .dispatchPeriod[1] - .spinningReserve, - u.MW, - ) - assert_units_equivalent( - m.investmentStage[1] - .representativePeriod[1] - .commitmentPeriod[1] - .genOn["3_CT"] - .operating_limit_min[1] - .expr, + m_commit.genOn["3_CT"].operating_limit_min[1].expr, u.MW, ) assert_units_equivalent( - m.investmentStage[1] - .representativePeriod[1] - .commitmentPeriod[1] - .genOn["4_STEAM"] - .max_spinning_reserve[1, "4_STEAM"] - .expr, + m_commit.genOn["4_STEAM"].max_spinning_reserve[1, "4_STEAM"].expr, u.MW, ) From e670b248acf9a3a3eaeba7feac4ef851bc0417fd Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 11:16:20 -0700 Subject: [PATCH 03/20] Run black --- gtep/tests/unit/test_gtep_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 9cad8a55..049e74cb 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -173,7 +173,7 @@ def test_model_unit_consistency(self): m_rep_period = m_inv.representativePeriod[1] m_commit = m_rep_period.commitmentPeriod[1] m_disp = m_commit.dispatchPeriod[1] - + assert_units_equivalent(m_rep_period.periodLength, u.h) assert_units_equivalent(m_commit.periodLength, u.h) assert_units_equivalent(m_disp.periodLength, u.min) From 98b19d7490b34e4de9ae0c13f539fd1f0ad32d49 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 11:24:43 -0700 Subject: [PATCH 04/20] Rename PeriodLength parameters in each stage to avoid confusion when called in the constraints --- gtep/gtep_model.py | 8 ++++---- gtep/model_library/commitment.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index a888b05b..89eeb500 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -296,7 +296,7 @@ def create_stages( for representative_period in b_inv.representativePeriods: b_rep = b_inv.representativePeriod[representative_period] - b_rep.periodLength = pyo.Param( + b_rep.representativePeriodLength = pyo.Param( initialize=duration_representative_period[representative_period - 1], within=pyo.PositiveReals, units=u.hr, @@ -328,7 +328,7 @@ def create_stages( for commitment_period in b_rep.commitmentPeriods: b_comm = b_rep.commitmentPeriod[commitment_period] - b_comm.periodLength = pyo.Param( + b_comm.commitmentPeriodLength = pyo.Param( initialize=duration_commitment[commitment_period - 1], within=pyo.PositiveReals, units=u.hr, @@ -362,7 +362,7 @@ def create_stages( for dispatch_period in b_comm.dispatchPeriods: b_disp = b_comm.dispatchPeriod[dispatch_period] - b_disp.periodLength = pyo.Param( + b_disp.dispatchPeriodLength = pyo.Param( initialize=duration_dispatch[dispatch_period - 1], within=pyo.PositiveReals, units=u.minutes, @@ -371,7 +371,7 @@ def create_stages( disp.add_dispatch_variables( b_disp, dispatch_period, - b_disp.periodLength, + b_disp.dispatchPeriodLength, ) disp.add_dispatch_constraints(b_disp, dispatch_period) diff --git a/gtep/model_library/commitment.py b/gtep/model_library/commitment.py index 6a24b5b8..7936f8c1 100644 --- a/gtep/model_library/commitment.py +++ b/gtep/model_library/commitment.py @@ -98,6 +98,7 @@ def renewableSurplusCommitment(b): for disp_per in b.dispatchPeriods ) + # [ESR TODO: Replace this constraint with expressions using bounds # transform and check if the costs considered need to be # re-assessed and account for missing data.] @@ -114,7 +115,7 @@ def operatingCostCommitment(b): if m.config["include_commitment"]: op_cost_gen_state = sum( m.fixedCost[gen] - * b.periodLength + * b.commitmentPeriodLength * m.thermalCapacity[gen] * ( b.genOn[gen].indicator_var.get_associated_binary() @@ -133,7 +134,7 @@ def operatingCostCommitment(b): else: op_cost_gen_state = sum( m.fixedCost[gen] - * b.periodLength + * b.commitmentPeriodLength * m.thermalCapacity[gen] * b.genOn[gen].indicator_var.get_associated_binary() for gen in m.thermalGenerators @@ -173,7 +174,7 @@ def renewable_curtailment_cost(b): m.weights[rep_per] * b.representativePeriod[rep_per] .commitmentPeriod[com_per] - .periodLength + .commitmentPeriodLength * b.representativePeriod[rep_per] .commitmentPeriod[com_per] .renewableCurtailmentCommitment # in MW From 44cfd156afdbcd0af9590ef3ddf278dbe6e80458 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 11:27:45 -0700 Subject: [PATCH 05/20] Update period length parameters names in test and run black --- gtep/model_library/commitment.py | 1 - gtep/tests/unit/test_gtep_model.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gtep/model_library/commitment.py b/gtep/model_library/commitment.py index 7936f8c1..fae75b41 100644 --- a/gtep/model_library/commitment.py +++ b/gtep/model_library/commitment.py @@ -98,7 +98,6 @@ def renewableSurplusCommitment(b): for disp_per in b.dispatchPeriods ) - # [ESR TODO: Replace this constraint with expressions using bounds # transform and check if the costs considered need to be # re-assessed and account for missing data.] diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 049e74cb..f0588dbd 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -174,9 +174,9 @@ def test_model_unit_consistency(self): m_commit = m_rep_period.commitmentPeriod[1] m_disp = m_commit.dispatchPeriod[1] - assert_units_equivalent(m_rep_period.periodLength, u.h) - assert_units_equivalent(m_commit.periodLength, u.h) - assert_units_equivalent(m_disp.periodLength, u.min) + assert_units_equivalent(m_rep_period.representativePeriodLength, u.h) + assert_units_equivalent(m_commit.commitmentPeriodLength, u.h) + assert_units_equivalent(m_disp.dispatchPeriodLength, u.min) assert_units_equivalent(m.renewable_capacity_enforcement[1, "10_PV"].expr, u.MW) assert_units_equivalent(m_inv.renewable_curtailment_cost.expr, u.USD) assert_units_equivalent(m_disp.flow_balance["bus1"].expr, u.MW) From 46afec6b27fab4a95080e4cf0bb03bcfc15533a8 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 16:35:01 -0700 Subject: [PATCH 06/20] Improve period structure handling to allow integer inputs for number and duration of periods while converting them to nested dictionaries for flexible period definitions In the model class, I included a function to expand scalars/lists to nested dictionaries for representative, commitment, and dispatch periods to support flexible period structures. --- gtep/driver_esr.py | 10 +- gtep/gtep_model.py | 166 ++++++++++++++++++++--------- gtep/model_library/components.py | 17 +-- gtep/tests/unit/test_gtep_model.py | 88 ++++++++++++--- 4 files changed, 198 insertions(+), 83 deletions(-) diff --git a/gtep/driver_esr.py b/gtep/driver_esr.py index 365117df..5e131467 100644 --- a/gtep/driver_esr.py +++ b/gtep/driver_esr.py @@ -55,11 +55,11 @@ stages=2, data=data_object, cost_data=data_processing_object, - num_reps=2, - len_reps=1, - num_commit=6, - num_dispatch=4, - # [ESR: in min by default, for now] + num_reps=4, + num_commit=12, + num_dispatch=12, + duration_representative_period=24, + duration_commitment=1, duration_dispatch=15, ) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 89eeb500..2a1457a9 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -91,8 +91,7 @@ def __init__( data=None, cost_data=None, num_reps=3, - len_reps=24, - num_commit=24, + num_commit=6, num_dispatch=4, duration_representative_period=24, duration_commitment=1, @@ -105,7 +104,6 @@ def __init__( :param data: full set of model data :param cost_data: full set of cost data for all generators :param num_reps: integer number of representative periods per investment period - :param len_reps: (for now integer) length of each representative period (in hours) :param num_commit: integer number of commitment periods per representative period :param num_dispatch: integer number of dispatch periods per commitment period :param duration_dispatch: (for now integer) duration of each dispatch period (in minutes) @@ -116,45 +114,38 @@ def __init__( self.formulation = formulation self.data = data self.cost_data = cost_data + self.config = _get_model_config() + self.timer = TicTocTimer() self.num_reps = num_reps - self.len_reps = len_reps self.num_commit = num_commit self.num_dispatch = num_dispatch - self.config = _get_model_config() - self.timer = TicTocTimer() - self.duration_representative_period = self._ensure_list_length( - "representative periods", duration_representative_period, num_reps, 24 - ) - self.duration_commitment = self._ensure_list_length( - "commitment", duration_commitment, num_commit, 1 - ) - self.duration_dispatch = self._ensure_list_length( - "dispatch", duration_dispatch, num_dispatch, 15 + self.duration_representative_period = duration_representative_period + self.duration_commitment = duration_commitment + self.duration_dispatch = duration_dispatch + + ( + self.num_commit, + self.num_dispatch, + self.duration_representative_period, + self.duration_commitment, + self.duration_dispatch, + ) = self._expand_period_structure_dict( + self.num_reps, + self.num_commit, + self.num_dispatch, + self.duration_representative_period, + self.duration_commitment, + self.duration_dispatch, ) + # print(self.num_commit) + # print(self.num_dispatch) + # print(self.duration_commitment) + # print(self.duration_dispatch) + _add_common_configs(self.config) _add_investment_configs(self.config) - @staticmethod - def _ensure_list_length(name, param, expected_length, default_value): - """This method ensures that any of the duration parameters is - a list of expected_length (equal to number of each - stage). If no value is given, a default value is used for - each time period. - - """ - if param is None: - return [default_value] * expected_length - elif isinstance(param, list): - if len(param) != expected_length: - print( - f"\n***WARNING: The list {param} for the duration_{name} parameter has a length ({len(param)}), which does not match the expected number of {name} periods ({expected_length}). We will use {default_value} as the default value.\n" - ) - return [default_value] * expected_length - return param - else: - return [param] * expected_length - def create_model(self): """Create concrete Pyomo model object associated with the ExpansionPlanningModel class @@ -192,15 +183,13 @@ def create_model(self): m, self.stages, rep_per=[i for i in range(1, self.num_reps + 1)] ) - comps.add_model_parameters( - m, - self.num_commit, - self.num_dispatch, - ) + comps.add_model_parameters(m) create_stages( m, self.stages, + self.num_commit, + self.num_dispatch, self.duration_representative_period, self.duration_commitment, self.duration_dispatch, @@ -248,9 +237,85 @@ def report_large_coefficients(self, outfile, magnitude_cutoff=1e5): with open(outfile, "w") as fil: json.dump(really_bad_var_coef_list, fil) + def _expand_period_structure_dict( + self, + num_reps, + num_commit, + num_dispatch, + duration_representative_period, + duration_commitment, + duration_dispatch, + ): + def get(val, *idx): + for i in idx: + if isinstance(val, (int, float)): + return val + elif isinstance(val, list): + val = val[i - 1] + elif isinstance(val, dict): + val = val[i] + else: + raise ValueError("Unsupported type in period structure") + return val + + duration_rep_dict = { + rep: get(duration_representative_period, rep) + for rep in range(1, num_reps + 1) + } + num_commit_dict = {rep: get(num_commit, rep) for rep in range(1, num_reps + 1)} + + duration_commit_dict = {} + num_dispatch_dict = {} + duration_dispatch_dict = {} + for rep in range(1, num_reps + 1): + ncom = num_commit_dict[rep] + duration_commit_dict[rep] = { + com: get(duration_commitment, rep, com) for com in range(1, ncom + 1) + } + num_dispatch_dict[rep] = { + com: get(num_dispatch, rep, com) for com in range(1, ncom + 1) + } + duration_dispatch_dict[rep] = {} + for com in range(1, ncom + 1): + ndisp = num_dispatch_dict[rep][com] + duration_dispatch_dict[rep][com] = { + disp: get(duration_dispatch, rep, com, disp) + for disp in range(1, ndisp + 1) + } + + # Consistency check: sum of dispatch durations should + # equal the commitment duration + dispatch_sum_hr = pyo.units.convert( + sum( + duration_dispatch_dict[rep][com][disp] + for disp in range(1, ndisp + 1) + ) + * u.minutes, + to_units=u.hours, + ) + commitment_hr = duration_commit_dict[rep][com] + if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: + print( + f"WARNING: The summation of dispatch durations is not the same as the commitment duration ({pyo.value(dispatch_sum_hr)} hr(s) vs {commitment_hr} hr(s)). Make sure these match!" + ) + + return ( + num_commit_dict, + num_dispatch_dict, + duration_rep_dict, + duration_commit_dict, + duration_dispatch_dict, + ) + def create_stages( - m, stages, duration_representative_period, duration_commitment, duration_dispatch + m, + stages, + num_commit, + num_dispatch, + duration_representative_period, + duration_commitment, + duration_dispatch, ): """This method constructs the block structure for the Generation and Transmission Expansion Planning (GTEP) model. It creates @@ -297,7 +362,7 @@ def create_stages( b_rep = b_inv.representativePeriod[representative_period] b_rep.representativePeriodLength = pyo.Param( - initialize=duration_representative_period[representative_period - 1], + initialize=duration_representative_period[representative_period], within=pyo.PositiveReals, units=u.hr, ) @@ -317,9 +382,8 @@ def create_stages( # include_commitment. When False, generators are on and # operational costs are determined solely by dispatch # decisions. - b_rep.commitmentPeriods = pyo.RangeSet( - m.numCommitmentPeriods[representative_period] - ) + n_commit = num_commit[representative_period] + b_rep.commitmentPeriods = pyo.RangeSet(n_commit) b_rep.commitmentPeriod = pyo.Block(b_rep.commitmentPeriods) # --.--.--.--.--.--.----.--.--.--.--.--.----.--.--.--.--.--.-- @@ -329,7 +393,9 @@ def create_stages( b_comm = b_rep.commitmentPeriod[commitment_period] b_comm.commitmentPeriodLength = pyo.Param( - initialize=duration_commitment[commitment_period - 1], + initialize=duration_commitment[representative_period][ + commitment_period + ], within=pyo.PositiveReals, units=u.hr, ) @@ -337,9 +403,8 @@ def create_stages( b_comm.commitmentPeriod = commitment_period if m.config["include_redispatch"]: - b_comm.dispatchPeriods = pyo.RangeSet( - m.numDispatchPeriods[b_rep.currentPeriod] - ) + n_dispatch = num_dispatch[representative_period][commitment_period] + b_comm.dispatchPeriods = pyo.RangeSet(n_dispatch) b_comm.dispatchPeriod = pyo.Block(b_comm.dispatchPeriods) # [TODO: update properties for this time period!] @@ -361,9 +426,10 @@ def create_stages( # bad. Check a better way of declaring these.] for dispatch_period in b_comm.dispatchPeriods: b_disp = b_comm.dispatchPeriod[dispatch_period] - b_disp.dispatchPeriodLength = pyo.Param( - initialize=duration_dispatch[dispatch_period - 1], + initialize=duration_dispatch[representative_period][ + commitment_period + ][dispatch_period], within=pyo.PositiveReals, units=u.minutes, ) diff --git a/gtep/model_library/components.py b/gtep/model_library/components.py index d153d95a..4e06fd7a 100644 --- a/gtep/model_library/components.py +++ b/gtep/model_library/components.py @@ -106,7 +106,7 @@ def add_model_sets(m, stages, rep_per=["a", "b"], com_per=2, dis_per=2): ) -def add_model_parameters(m, num_commit, num_dispatch): +def add_model_parameters(m): """Creates and labels all the parameters in the GTEP model. This method ties input data directly to the model. @@ -118,21 +118,6 @@ def add_model_parameters(m, num_commit, num_dispatch): # configuration arg and not hardcoded values.] m.years = [2025, 2030, 2035] - # Add parameters related to the representative periods for the - # different stages - m.numCommitmentPeriods = pyo.Param( - m.representativePeriods, - within=pyo.PositiveIntegers, - default=2, - initialize=num_commit, - ) - m.numDispatchPeriods = pyo.Param( - m.representativePeriods, - within=pyo.PositiveIntegers, - default=2, - initialize=num_dispatch, - ) - # Add power-related parameters m.thermalCapacity = pyo.Param( m.thermalGenerators, diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index f0588dbd..9a513b6f 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -97,9 +97,18 @@ def test_model_init(self): self.assertEqual(modObject.formulation, None) self.assertIsInstance(modObject.model.md, ModelData) self.assertEqual(modObject.num_reps, 3) - self.assertEqual(modObject.len_reps, 24) - self.assertEqual(modObject.num_commit, 24) - self.assertEqual(modObject.num_dispatch, 4) + self.assertEqual(modObject.num_commit, {1: 6, 2: 6, 3: 6}) + self.assertEqual( + modObject.num_dispatch, + { + 1: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, + 2: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, + 3: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, + }, + ) + self.assertEqual( + modObject.duration_representative_period, {1: 24, 2: 24, 3: 24} + ) # Test that the ExpansionPlanningModel object can read a default dataset and init # properly with non-default values @@ -107,7 +116,6 @@ def test_model_init(self): data=data_object, stages=2, num_reps=4, - len_reps=16, num_commit=12, num_dispatch=12, ) @@ -118,9 +126,68 @@ def test_model_init(self): self.assertEqual(modObject.formulation, None) self.assertIsInstance(modObject.model.md, ModelData) self.assertEqual(modObject.num_reps, 4) - self.assertEqual(modObject.len_reps, 16) - self.assertEqual(modObject.num_commit, 12) - self.assertEqual(modObject.num_dispatch, 12) + self.assertEqual(modObject.num_commit, {1: 12, 2: 12, 3: 12, 4: 12}) + self.assertEqual( + modObject.num_dispatch, + { + 1: { + 1: 12, + 2: 12, + 3: 12, + 4: 12, + 5: 12, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 12, + 12: 12, + }, + 2: { + 1: 12, + 2: 12, + 3: 12, + 4: 12, + 5: 12, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 12, + 12: 12, + }, + 3: { + 1: 12, + 2: 12, + 3: 12, + 4: 12, + 5: 12, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 12, + 12: 12, + }, + 4: { + 1: 12, + 2: 12, + 3: 12, + 4: 12, + 5: 12, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 12, + 12: 12, + }, + }, + ) # We have expansion blocks and they are where and what we think they are expansion_blocks = modObject.model.component("investmentStage") @@ -158,7 +225,6 @@ def test_model_unit_consistency(self): data=data_object, stages=2, num_reps=2, - len_reps=2, num_commit=2, num_dispatch=2, ) @@ -196,7 +262,7 @@ def test_solve_bigm(self): # Solve the debug model as is data_object = read_debug_model() modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + data=data_object, num_reps=1, num_commit=1, num_dispatch=1 ) modObject.create_model() @@ -223,7 +289,7 @@ def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model() modObject = ExpansionPlanningModel( - data=data_object, num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 + data=data_object, num_reps=1, num_commit=1, num_dispatch=1 ) modObject.config["include_investment"] = False modObject.create_model() @@ -259,7 +325,6 @@ def test_with_cost_data_and_commitment(self): data=dataObject, cost_data=dataProcessingObject, num_reps=2, - len_reps=1, num_commit=6, num_dispatch=4, duration_dispatch=15, @@ -308,7 +373,6 @@ def test_with_cost_data_and_no_commitment(self): data=dataObject, cost_data=dataProcessingObject, num_reps=2, - len_reps=1, num_commit=6, num_dispatch=4, duration_dispatch=15, From c27a645dfb19fc4ba2065deddacfc216ec2c83c1 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 16:42:17 -0700 Subject: [PATCH 07/20] Remove argument from class since it not needed anymore --- gtep/tests/unit/test_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 45fb458f..7f93108d 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -48,7 +48,6 @@ def get_solution_object(): stages=2, data=data_object, num_reps=2, - len_reps=1, num_commit=6, num_dispatch=4, ) From 13ec9725f09aa9dd82866aa8631d3f3191e6201c Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 16 Apr 2026 16:56:48 -0700 Subject: [PATCH 08/20] Remove len_reps since it is no longer needed or used --- gtep/gtep_solution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index d7e5f773..e22b164d 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -69,7 +69,6 @@ def load_from_model(self, gtep_model): self.formulation = gtep_model.formulation # None (???) self.data = gtep_model.data # ModelData object self.num_reps = gtep_model.num_reps # int - self.len_reps = gtep_model.len_reps # int self.num_commit = gtep_model.num_commit # int self.num_dispatch = gtep_model.num_dispatch # int From 47b397aced64dc8b05bd535af941b0559b512f9f Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 14:33:50 -0700 Subject: [PATCH 09/20] Add new file "utils" that includes functions to generate the period structure dictionary needed in the expansion model class --- gtep/utils.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 gtep/utils.py diff --git a/gtep/utils.py b/gtep/utils.py new file mode 100644 index 00000000..562e3045 --- /dev/null +++ b/gtep/utils.py @@ -0,0 +1,86 @@ +import json + + +def generate_period_structure_skeleton( + num_reps, + num_commit, + num_dispatch, + rep_duration=24, + com_duration=1, + disp_duration=15, +): + """This method generates a skeleton of the period structure + dictionary for user editing. The dictionary is nested as follows: + + { + "number_representative": , + "number_commitment": {rep: }, + "number_dispatch": {rep: {com: }}, + "duration_representative_period": {rep: }, + "duration_commitment": {rep: {com: }}, + "duration_dispatch": {rep: {com: {disp: }}} + } + + """ + + period_dict = { + "number_representative": num_reps, + "number_commitment": {rep: num_commit for rep in range(1, num_reps + 1)}, + "number_dispatch": { + rep: {com: num_dispatch for com in range(1, num_commit + 1)} + for rep in range(1, num_reps + 1) + }, + "duration_representative_period": { + rep: rep_duration for rep in range(1, num_reps + 1) + }, + "duration_commitment": { + rep: {com: com_duration for com in range(1, num_commit + 1)} + for rep in range(1, num_reps + 1) + }, + "duration_dispatch": { + rep: { + com: {disp: disp_duration for disp in range(1, num_dispatch + 1)} + for com in range(1, num_commit + 1) + } + for rep in range(1, num_reps + 1) + }, + } + + return period_dict + + +def save_period_structure_json(period_structure, filename): + """This method saves a period structure dictionary to a .json file + file. + + """ + + with open(filename, "w") as f: + json.dump(period_structure, f, indent=2) + + +def generate_period_structure_utils( + num_reps, + num_commit, + num_dispatch, + rep_duration=24, + com_duration=1, + disp_duration=15, + filename=None, +): + """This method generates a period structure skeleton and + optionally saves it as a .json file for user editing. + + :return: period structure dictionary; if a filename is provided, + also saves it as a .json file. + + """ + + period_dict = generate_period_structure_skeleton( + num_reps, num_commit, num_dispatch, rep_duration, com_duration, disp_duration + ) + + if filename: + save_period_structure_json(period_dict, filename) + + return period_dict From 807f4900ea40eed48acbeb640d13a4746f7d51ed Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 14:51:57 -0700 Subject: [PATCH 10/20] Improve period structure in class to read number of periods and their duration from scalars or json file. Also, add test for this. In the GTEP class, remove function to expand scalars to a period structure dictionary and add this to a new function in utils file. Also, include new function that allows both, scalars (to be expanded to a period structure dictionary) or a .json file with customed period structure. --- gtep/driver_esr.py | 9 +- gtep/gtep_model.py | 216 +++++++++++++++++------------ gtep/tests/unit/test_gtep_model.py | 71 ++++++++++ 3 files changed, 204 insertions(+), 92 deletions(-) diff --git a/gtep/driver_esr.py b/gtep/driver_esr.py index 5e131467..0cca0f36 100644 --- a/gtep/driver_esr.py +++ b/gtep/driver_esr.py @@ -55,12 +55,15 @@ stages=2, data=data_object, cost_data=data_processing_object, - num_reps=4, - num_commit=12, - num_dispatch=12, + num_reps=2, + num_commit=6, + num_dispatch=4, duration_representative_period=24, duration_commitment=1, duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, + # period_structure_json_file="period_structure_from_gtep.json", ) mod_object.config["include_investment"] = True diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 2a1457a9..91267c65 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -27,6 +27,7 @@ import json import numpy as np import re +import os import pyomo.environ as pyo from pyomo.environ import units as u @@ -52,6 +53,10 @@ import gtep.model_library.storage as stor import gtep.model_library.transmission as transm +from gtep.utils import generate_period_structure_utils + +curr_dir = os.path.dirname(os.path.abspath(__file__)) + # Define what a USD is for pyomo units purposes. This will be set to a # base year and we will do NPV calculations based on automatic Pyomo # unit transformations. @@ -96,17 +101,28 @@ def __init__( duration_representative_period=24, duration_commitment=1, duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, ): """Initialize generation & expansion planning model object. - :param stages: integer number of investment periods - :param formulation: Egret stuff, to be filled - :param data: full set of model data - :param cost_data: full set of cost data for all generators - :param num_reps: integer number of representative periods per investment period - :param num_commit: integer number of commitment periods per representative period - :param num_dispatch: integer number of dispatch periods per commitment period - :param duration_dispatch: (for now integer) duration of each dispatch period (in minutes) + :param: stages: integer number of investment periods + :param: formulation: Egret stuff, to be filled + :param: data: full set of model data + :param: cost_data: full set of cost data for all generators + :param: num_reps: integer number of representative periods per investment period + :param: num_commit: integer number of commitment periods per representative period + :param: num_dispatch: integer number of dispatch periods per commitment period + :param: duration_representative_period: duration of each representative period + (in hours) + :param: duration_commitment: duration of each commitment period (in hours) + :param: duration_dispatch: duration of each dispatch period (in minutes) + :param: save_period_structure_file: (optional) If True, saves the generated + period structure as a JSON file in the data directory. Default is False. + :param: period_structure_json_file: (optional) Path to a JSON file in the data + directory specifying the period structure. Overrides scalar/list arguments + if provided. Default is None. + :return: Pyomo model for full GTEP """ @@ -122,30 +138,114 @@ def __init__( self.duration_representative_period = duration_representative_period self.duration_commitment = duration_commitment self.duration_dispatch = duration_dispatch + self.save_period_structure_file = save_period_structure_file + self.period_structure_json_file = period_structure_json_file - ( - self.num_commit, - self.num_dispatch, - self.duration_representative_period, - self.duration_commitment, - self.duration_dispatch, - ) = self._expand_period_structure_dict( - self.num_reps, - self.num_commit, - self.num_dispatch, - self.duration_representative_period, - self.duration_commitment, - self.duration_dispatch, - ) - - # print(self.num_commit) - # print(self.num_dispatch) - # print(self.duration_commitment) - # print(self.duration_dispatch) + # Set and validate period structure attributes from .json file + # or provided scalars. This function also implements a + # consistency check on dispatch and commitment durations. + self._set_period_structure_dict() _add_common_configs(self.config) _add_investment_configs(self.config) + def _set_period_structure_dict(self): + """This method initializes and validates the period structure + attributes (number and duration of representative, commitment, + and dispatch periods) from either a user-provided .json file + or from provided scalar arguments. + + This method performs a consistency check to ensure that the + sum of dispatch durations matches each commitment period + duration. + + """ + + # If a .json file with period structure data is provided, use + # it, otherwise, expand from scalars. + + if self.period_structure_json_file is not None: + # Use provided .json file + json_path = os.path.abspath( + os.path.join(curr_dir, "data", self.period_structure_json_file) + ) + with open(json_path, "r") as f: + period_dict = json.load(f) + + # Helper function to recursively convert string keys to + # integers + def convert_keys_to_int(obj): + if isinstance(obj, dict): + return { + ( + int(k) if isinstance(k, str) and k.isdigit() else k + ): convert_keys_to_int(v) + for k, v in obj.items() + } + else: + return obj + + period_dict = convert_keys_to_int(period_dict) + + else: + # .json file not provided; expand period structure + # dictionary from scalar arguments. Optionally save the + # expanded dictionary as a .json file with a default name + # under the data directory. + filename = ( + os.path.abspath( + os.path.join(curr_dir, "data", "period_structure_from_gtep.json") + ) + if self.save_period_structure_file + else None + ) + period_dict = generate_period_structure_utils( + self.num_reps, + self.num_commit, + self.num_dispatch, + self.duration_representative_period, + self.duration_commitment, + self.duration_dispatch, + filename=filename, + ) + if self.save_period_structure_file: + print( + f"\nINFO: Period structure dictionary generated from scalar period arguments has been written to '{filename}'.\n" + ) + + # Assign period structure attributes from the dictionary + self.num_reps = period_dict.get("number_representative", self.num_reps) + self.num_commit = period_dict["number_commitment"] + self.num_dispatch = period_dict["number_dispatch"] + self.duration_representative_period = period_dict[ + "duration_representative_period" + ] + self.duration_commitment = period_dict["duration_commitment"] + self.duration_dispatch = period_dict["duration_dispatch"] + + # # Consistency check: the sum of dispatch durations should + # # equal the commitment duration + # for rep in range(1, self.num_reps + 1): + # for com in range(1, self.num_commit[rep] + 1): + + # # Sum dispatch durations (in minutes) and convert it + # # to hours to compare commitment and dispatch duration + # dispatch_sum_hr = pyo.units.convert( + # sum( + # self.duration_dispatch[rep][com][disp] + # for disp in range(1, self.num_dispatch[rep][com] + 1) + # ) * u.minutes, + # to_units=u.hours, + # ) + # commitment_hr = self.duration_commitment[rep][com] + # if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: + # raise ValueError( + # f"ERROR: The sum of dispatch period durations ({pyo.value(dispatch_sum_hr)} hr) " + # f"does not match the commitment period duration ({commitment_hr} hr) " + # f"for representative period {rep}, commitment period {com}. " + # "Please ensure these durations are consistent in your period structure file ({})." + # ) + def create_model(self): """Create concrete Pyomo model object associated with the ExpansionPlanningModel class @@ -237,68 +337,6 @@ def report_large_coefficients(self, outfile, magnitude_cutoff=1e5): with open(outfile, "w") as fil: json.dump(really_bad_var_coef_list, fil) - def _expand_period_structure_dict( - self, - num_reps, - num_commit, - num_dispatch, - duration_representative_period, - duration_commitment, - duration_dispatch, - ): - def get(val, *idx): - for i in idx: - if isinstance(val, (int, float)): - return val - elif isinstance(val, list): - val = val[i - 1] - elif isinstance(val, dict): - val = val[i] - else: - raise ValueError("Unsupported type in period structure") - return val - - duration_rep_dict = { - rep: get(duration_representative_period, rep) - for rep in range(1, num_reps + 1) - } - num_commit_dict = {rep: get(num_commit, rep) for rep in range(1, num_reps + 1)} - - duration_commit_dict = {} - num_dispatch_dict = {} - duration_dispatch_dict = {} - for rep in range(1, num_reps + 1): - ncom = num_commit_dict[rep] - duration_commit_dict[rep] = { - com: get(duration_commitment, rep, com) for com in range(1, ncom + 1) - } - num_dispatch_dict[rep] = { - com: get(num_dispatch, rep, com) for com in range(1, ncom + 1) - } - duration_dispatch_dict[rep] = {} - for com in range(1, ncom + 1): - ndisp = num_dispatch_dict[rep][com] - duration_dispatch_dict[rep][com] = { - disp: get(duration_dispatch, rep, com, disp) - for disp in range(1, ndisp + 1) - } - - # Consistency check: sum of dispatch durations should - # equal the commitment duration - dispatch_sum_hr = pyo.units.convert( - sum( - duration_dispatch_dict[rep][com][disp] - for disp in range(1, ndisp + 1) - ) - * u.minutes, - to_units=u.hours, - ) - commitment_hr = duration_commit_dict[rep][com] - if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: - print( - f"WARNING: The summation of dispatch durations is not the same as the commitment duration ({pyo.value(dispatch_sum_hr)} hr(s) vs {commitment_hr} hr(s)). Make sure these match!" - ) - return ( num_commit_dict, num_dispatch_dict, diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 882e42ca..8d91500c 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -11,6 +11,8 @@ # for full copyright and license information. ################################################################################# +import os +import json from os.path import abspath, join, dirname import pytest @@ -91,7 +93,9 @@ def test_model_init(self): data_object = read_debug_model() modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) + modObject.create_model() + self.assertIsInstance(modObject.model, ConcreteModel) self.assertEqual(modObject.stages, 1) self.assertEqual(modObject.formulation, None) @@ -404,3 +408,70 @@ def test_with_cost_data_and_no_commitment(self): ) assert_units_equivalent(modObject.model.total_cost_objective_rule.expr, u.USD) + + def test_period_structure_from_scalars_and_json(self): + # Test with scalar/list arguments (all periods same) + dataObject, dataProcessingObject = prepare_model_and_cost_data() + + modObject = ExpansionPlanningModel( + stages=1, + data=dataObject, + cost_data=dataProcessingObject, + num_reps=2, + num_commit=3, + num_dispatch=4, + duration_representative_period=24, + duration_commitment=1, + duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, + ) + + # Check that all values are as expected (all periods same) + self.assertEqual(modObject.num_commit[1], 3) + self.assertEqual(modObject.num_dispatch[2][3], 4) + self.assertEqual(modObject.duration_commitment[1][2], 1) + self.assertEqual(modObject.duration_dispatch[2][3][4], 15) + + # Test custom period structure with irregular values. This + # dictionary is saved as a .json file and then used to + # initialize the ExpansionPlanningModel class. + period_dict = { + "number_representative": 2, + "number_commitment": {1: 2, 2: 3}, + "number_dispatch": {1: {1: 3, 2: 2}, 2: {1: 2, 2: 3, 3: 2}}, + "duration_representative_period": {1: 24, 2: 18}, + "duration_commitment": {1: {1: 1, 2: 2}, 2: {1: 1, 2: 1.5, 3: 2}}, + "duration_dispatch": { + 1: {1: {1: 10, 2: 20, 3: 30}, 2: {1: 30, 2: 90}}, + 2: {1: {1: 30, 2: 30}, 2: {1: 20, 2: 20, 3: 50}, 3: {1: 60, 2: 60}}, + }, + } + curr_dir = os.path.dirname(os.path.abspath(__file__)) + json_path = os.path.join(curr_dir, "test_custom_period_structure.json") + with open(json_path, "w") as f: + json.dump(period_dict, f, indent=2) + + # Test that the model correctly reads and assigns the custom + # period structure values. Here we instantiate the model using + # the .json file. + modObject = ExpansionPlanningModel( + stages=1, + data=dataObject, + cost_data=dataProcessingObject, + period_structure_json_file=json_path, + save_period_structure_file=False, + ) + + # Assert that we have the correct reading of the structure + self.assertEqual(modObject.num_reps, 2) + self.assertEqual(modObject.num_commit[2], 3) + self.assertEqual(modObject.num_dispatch[2][2], 3) + self.assertEqual(modObject.duration_commitment[2][2], 1.5) + self.assertEqual(modObject.duration_dispatch[2][2][3], 50) + self.assertEqual(modObject.duration_representative_period[2], 18) + self.assertEqual(modObject.duration_dispatch[1][1][2], 20) + self.assertEqual(modObject.duration_commitment[1][2], 2) + + # Remove the .json file after the test + os.remove(json_path) From e894155b1d9c6ecb9a70c9e4537311350b04f382 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 15:32:44 -0700 Subject: [PATCH 11/20] Update test to call scalars from data file and not as an argument in the gtep class --- gtep/tests/unit/test_gtep_model.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 843850b3..9af974a2 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -52,7 +52,6 @@ def patch_unit_handlers(): def read_debug_model( stages=1, num_reps=3, - len_reps=24, num_commit=24, num_dispatch=4, duration_dispatch=15, @@ -62,7 +61,6 @@ def read_debug_model( dataObject = ExpansionPlanningData( stages=stages, num_reps=num_reps, - len_reps=len_reps, num_commit=num_commit, num_dispatch=num_dispatch, duration_dispatch=duration_dispatch, @@ -74,7 +72,6 @@ def read_debug_model( def prepare_model_and_cost_data( stages=1, num_reps=3, - len_reps=24, num_commit=24, num_dispatch=4, duration_dispatch=15, @@ -83,7 +80,6 @@ def prepare_model_and_cost_data( dataObject = read_debug_model( stages, num_reps, - len_reps, num_commit, num_dispatch, duration_dispatch, @@ -121,8 +117,7 @@ def test_model_init(self): data_object = read_debug_model( stages=1, num_reps=3, - len_reps=24, - num_commit=24, + num_commit=6, num_dispatch=4, duration_dispatch=60, ) @@ -303,9 +298,7 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is - data_object = read_debug_model( - num_reps=1, len_reps=1, num_commit=1, num_dispatch=1 - ) + data_object = read_debug_model(num_reps=1, num_commit=1, num_dispatch=1) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -332,7 +325,6 @@ def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model( num_reps=1, - len_reps=1, num_commit=1, num_dispatch=1, ) @@ -459,18 +451,18 @@ def test_with_cost_data_and_no_commitment(self): def test_period_structure_from_scalars_and_json(self): # Test with scalar/list arguments (all periods same) - dataObject, dataProcessingObject = prepare_model_and_cost_data() + dataObject, dataProcessingObject = prepare_model_and_cost_data( + num_reps=2, + num_commit=3, + num_dispatch=4, + duration_dispatch=15, + ) modObject = ExpansionPlanningModel( - stages=1, data=dataObject, cost_data=dataProcessingObject, - num_reps=2, - num_commit=3, - num_dispatch=4, duration_representative_period=24, duration_commitment=1, - duration_dispatch=15, save_period_structure_file=False, period_structure_json_file=None, ) @@ -504,7 +496,6 @@ def test_period_structure_from_scalars_and_json(self): # period structure values. Here we instantiate the model using # the .json file. modObject = ExpansionPlanningModel( - stages=1, data=dataObject, cost_data=dataProcessingObject, period_structure_json_file=json_path, From e7877a1fac8b8ad31fc0b125a3449ec4b0976fa7 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 16:12:52 -0700 Subject: [PATCH 12/20] Move new arguments from expansion class to data class and update driver and test to reflect these changes --- gtep/driver_esr.py | 23 ++++++++++---------- gtep/gtep_data.py | 30 +++++++++++++++++++------ gtep/gtep_model.py | 23 +++++--------------- gtep/tests/unit/test_gtep_model.py | 35 ++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/gtep/driver_esr.py b/gtep/driver_esr.py index 0cca0f36..4db3d045 100644 --- a/gtep/driver_esr.py +++ b/gtep/driver_esr.py @@ -22,7 +22,18 @@ # Add data data_path = "./data/5bus" -data_object = ExpansionPlanningData() +data_object = ExpansionPlanningData( + stages=2, + num_reps=2, + num_commit=6, + num_dispatch=4, + duration_representative_period=24, + duration_commitment=1, + duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, + # period_structure_json_file="period_structure_from_gtep.json", +) data_object.load_prescient(data_path) # [ESR WIP: Add bus and cost data files to be used on the @@ -52,18 +63,8 @@ # Populate and create GTEP model mod_object = ExpansionPlanningModel( - stages=2, data=data_object, cost_data=data_processing_object, - num_reps=2, - num_commit=6, - num_dispatch=4, - duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, - save_period_structure_file=False, - period_structure_json_file=None, - # period_structure_json_file="period_structure_from_gtep.json", ) mod_object.config["include_investment"] = True diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index 251fcbc6..cfcc0241 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -35,23 +35,39 @@ def __init__( len_reps=1, num_commit=24, num_dispatch=1, - duration_dispatch=60, + duration_representative_period=24, + duration_commitment=1, + duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, ): """Initialize generation & expansion planning data object. - :param stages: integer number of investment periods - :param num_reps: integer number of representative periods per investment period - :param len_reps: (for now integer) length of each representative period (in hours) - :param num_commit: integer number of commitment periods per representative period - :param num_dispatch: integer number of dispatch periods per commitment period - :param duration_dispatch: (for now integer) duration of each dispatch period (in minutes) + :param: stages: integer number of investment periods + :param: num_reps: integer number of representative periods per investment period + :param: num_commit: integer number of commitment periods per representative period + :param: num_dispatch: integer number of dispatch periods per commitment period + :param: duration_representative_period: duration of each representative period + (in hours) + :param: duration_commitment: duration of each commitment period (in hours) + :param: duration_dispatch: duration of each dispatch period (in minutes) + :param: save_period_structure_file: (optional) If True, saves the generated + period structure as a JSON file in the data directory. Default is False. + :param: period_structure_json_file: (optional) Path to a JSON file in the data + directory specifying the period structure. Overrides scalar/list arguments + if provided. Default is None. + """ self.stages = stages self.num_reps = num_reps self.len_reps = len_reps self.num_commit = num_commit self.num_dispatch = num_dispatch + self.duration_representative_period = duration_representative_period + self.duration_commitment = duration_commitment self.duration_dispatch = duration_dispatch + self.save_period_structure_file = save_period_structure_file + self.period_structure_json_file = period_structure_json_file def load_prescient( self, diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 2c13e32e..9fc74dc0 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -94,26 +94,12 @@ def __init__( formulation=None, data=None, cost_data=None, - duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, - save_period_structure_file=False, - period_structure_json_file=None, ): """Initialize generation & expansion planning model object. :param formulation: Egret stuff, to be filled :param data: full set of model data :param cost_data: full set of cost data for all generators - :param: duration_representative_period: duration of each representative period - (in hours) - :param: duration_commitment: duration of each commitment period (in hours) - :param: duration_dispatch: duration of each dispatch period (in minutes) - :param: save_period_structure_file: (optional) If True, saves the generated - period structure as a JSON file in the data directory. Default is False. - :param: period_structure_json_file: (optional) Path to a JSON file in the data - directory specifying the period structure. Overrides scalar/list arguments - if provided. Default is None. :return: Pyomo model for full GTEP """ @@ -126,12 +112,13 @@ def __init__( self.num_commit = data.num_commit self.num_dispatch = data.num_dispatch self.duration_dispatch = data.duration_dispatch + self.duration_representative_period = data.duration_representative_period + self.duration_commitment = data.duration_commitment + self.save_period_structure_file = data.save_period_structure_file + self.period_structure_json_file = data.period_structure_json_file + self.config = _get_model_config() self.timer = TicTocTimer() - self.duration_representative_period = duration_representative_period - self.duration_commitment = duration_commitment - self.save_period_structure_file = save_period_structure_file - self.period_structure_json_file = period_structure_json_file # Set and validate period structure attributes from .json file # or provided scalars. This function also implements a diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 9af974a2..0c15aa6f 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -54,7 +54,11 @@ def read_debug_model( num_reps=3, num_commit=24, num_dispatch=4, + duration_representative_period=24, + duration_commitment=1, duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, ): curr_dir = dirname(abspath(__file__)) debug_data_path = abspath(join(curr_dir, "..", "..", "data", "5bus")) @@ -63,7 +67,11 @@ def read_debug_model( num_reps=num_reps, num_commit=num_commit, num_dispatch=num_dispatch, + duration_representative_period=duration_representative_period, + duration_commitment=duration_commitment, duration_dispatch=duration_dispatch, + save_period_structure_file=save_period_structure_file, + period_structure_json_file=period_structure_json_file, ) dataObject.load_prescient(debug_data_path) return dataObject @@ -74,7 +82,11 @@ def prepare_model_and_cost_data( num_reps=3, num_commit=24, num_dispatch=4, + duration_representative_period=24, + duration_commitment=1, duration_dispatch=15, + save_period_structure_file=False, + period_structure_json_file=None, ): # Prepare model and cost data dataObject = read_debug_model( @@ -82,7 +94,11 @@ def prepare_model_and_cost_data( num_reps, num_commit, num_dispatch, + duration_representative_period, + duration_commitment, duration_dispatch, + save_period_structure_file, + period_structure_json_file, ) curr_dir = dirname(abspath(__file__)) data_path = abspath(join(curr_dir, "..", "..", "data", "costs")) @@ -449,7 +465,7 @@ def test_with_cost_data_and_no_commitment(self): assert_units_equivalent(modObject.model.total_cost_objective_rule.expr, u.USD) - def test_period_structure_from_scalars_and_json(self): + def test_period_structure_from_scalars(self): # Test with scalar/list arguments (all periods same) dataObject, dataProcessingObject = prepare_model_and_cost_data( num_reps=2, @@ -459,12 +475,7 @@ def test_period_structure_from_scalars_and_json(self): ) modObject = ExpansionPlanningModel( - data=dataObject, - cost_data=dataProcessingObject, - duration_representative_period=24, - duration_commitment=1, - save_period_structure_file=False, - period_structure_json_file=None, + data=dataObject, cost_data=dataProcessingObject ) # Check that all values are as expected (all periods same) @@ -473,6 +484,7 @@ def test_period_structure_from_scalars_and_json(self): self.assertEqual(modObject.duration_commitment[1][2], 1) self.assertEqual(modObject.duration_dispatch[2][3][4], 15) + def test_period_structure_from_json(self): # Test custom period structure with irregular values. This # dictionary is saved as a .json file and then used to # initialize the ExpansionPlanningModel class. @@ -495,11 +507,12 @@ def test_period_structure_from_scalars_and_json(self): # Test that the model correctly reads and assigns the custom # period structure values. Here we instantiate the model using # the .json file. - modObject = ExpansionPlanningModel( - data=dataObject, - cost_data=dataProcessingObject, + dataObject, dataProcessingObject = prepare_model_and_cost_data( period_structure_json_file=json_path, - save_period_structure_file=False, + ) + + modObject = ExpansionPlanningModel( + data=dataObject, cost_data=dataProcessingObject ) # Assert that we have the correct reading of the structure From 0b0baf013255374218b45c0a5b2654b9e92b2188 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 16:35:46 -0700 Subject: [PATCH 13/20] Add consistency check for the duration of dispatch periods and update tests to avoid touching the error --- gtep/gtep_model.py | 45 +++++++++++++++--------------- gtep/tests/unit/test_gtep_model.py | 21 +++++++------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 9fc74dc0..74ba5ec8 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -202,28 +202,29 @@ def convert_keys_to_int(obj): self.duration_commitment = period_dict["duration_commitment"] self.duration_dispatch = period_dict["duration_dispatch"] - # # Consistency check: the sum of dispatch durations should - # # equal the commitment duration - # for rep in range(1, self.num_reps + 1): - # for com in range(1, self.num_commit[rep] + 1): - - # # Sum dispatch durations (in minutes) and convert it - # # to hours to compare commitment and dispatch duration - # dispatch_sum_hr = pyo.units.convert( - # sum( - # self.duration_dispatch[rep][com][disp] - # for disp in range(1, self.num_dispatch[rep][com] + 1) - # ) * u.minutes, - # to_units=u.hours, - # ) - # commitment_hr = self.duration_commitment[rep][com] - # if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: - # raise ValueError( - # f"ERROR: The sum of dispatch period durations ({pyo.value(dispatch_sum_hr)} hr) " - # f"does not match the commitment period duration ({commitment_hr} hr) " - # f"for representative period {rep}, commitment period {com}. " - # "Please ensure these durations are consistent in your period structure file ({})." - # ) + # Consistency check: the sum of dispatch durations should + # equal the commitment duration + for rep in range(1, self.num_reps + 1): + for com in range(1, self.num_commit[rep] + 1): + + # Sum dispatch durations (in minutes) and convert it + # to hours to compare commitment and dispatch duration + dispatch_sum_hr = pyo.units.convert( + sum( + self.duration_dispatch[rep][com][disp] + for disp in range(1, self.num_dispatch[rep][com] + 1) + ) + * u.minutes, + to_units=u.hours, + ) + commitment_hr = self.duration_commitment[rep][com] + if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: + raise ValueError( + f"ERROR: The sum of dispatch period durations ({pyo.value(dispatch_sum_hr)} hr) " + f"does not match the commitment period duration ({commitment_hr} hr) " + f"for representative period {rep}, commitment period {com}. " + "Please ensure these durations are consistent in your period structure file ({})." + ) def create_model(self): """Create concrete Pyomo model object associated with the diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 0c15aa6f..79bf6cd5 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -135,7 +135,6 @@ def test_model_init(self): num_reps=3, num_commit=6, num_dispatch=4, - duration_dispatch=60, ) modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) @@ -167,7 +166,7 @@ def test_model_init(self): num_reps=4, num_commit=12, num_dispatch=12, - duration_dispatch=30, + duration_dispatch=5, ) modObject = ExpansionPlanningModel( data=data_object, @@ -278,6 +277,7 @@ def test_model_unit_consistency(self): num_reps=2, num_commit=2, num_dispatch=2, + duration_dispatch=30, ) modObject = ExpansionPlanningModel( data=data_object, @@ -314,7 +314,9 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is - data_object = read_debug_model(num_reps=1, num_commit=1, num_dispatch=1) + data_object = read_debug_model( + num_reps=1, num_commit=1, num_dispatch=1, duration_dispatch=60 + ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -331,18 +333,17 @@ def test_solve_bigm(self): modObject.results = opt.solve(modObject.model) - # previous successful objective values: 9207.95, 6078.86, 531860.15, 531883.43 + # Previous successful objective values: 9207.95, 6078.86, + # 531860.15, 531883.43, 2127462.53 self.assertAlmostEqual( - value(modObject.model.total_cost_objective_rule), 531883.43, places=1 + value(modObject.model.total_cost_objective_rule), 2127462.53, places=1 ) assert_units_equivalent(modObject.model.total_cost_objective_rule.expr, u.USD) def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model( - num_reps=1, - num_commit=1, - num_dispatch=1, + num_reps=1, num_commit=1, num_dispatch=1, duration_dispatch=60 ) modObject = ExpansionPlanningModel( data=data_object, @@ -365,9 +366,9 @@ def test_no_investment(self): modObject.results = opt.solve(modObject.model) - # previous successful objective values: 531860.15, 531883.43 + # previous successful objective values: 531860.15, 531883.43, 2127462.53 self.assertAlmostEqual( - value(modObject.model.total_cost_objective_rule), 531883.43, places=1 + value(modObject.model.total_cost_objective_rule), 2127462.53, places=1 ) assert_units_equivalent(modObject.model.total_cost_objective_rule.expr, u.USD) From 570b10dda7e7b3cb99f7be169eb360f4de2a9e7e Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Wed, 29 Apr 2026 16:44:29 -0700 Subject: [PATCH 14/20] Add test for dispatch duration consistency check --- gtep/tests/unit/test_gtep_model.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 79bf6cd5..fdd66875 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -528,3 +528,24 @@ def test_period_structure_from_json(self): # Remove the .json file after the test os.remove(json_path) + + def test_period_structure_consistency_error_with_scalars(self): + # Prepare model and cost data as usual + dataObject, dataProcessingObject = prepare_model_and_cost_data( + stages=1, + num_reps=1, + num_commit=1, + num_dispatch=2, + duration_representative_period=24, + duration_commitment=1, + duration_dispatch=60, + ) + # The sum of dispatch durations (2*60min = 120 min = 2hr) does + # not match the commitment duration (1hr) + with self.assertRaises(ValueError) as cm: + ExpansionPlanningModel(data=dataObject, cost_data=dataProcessingObject) + + self.assertIn( + "ERROR: The sum of dispatch period durations (2.0 hr) does not match the commitment period duration (1 hr)", + str(cm.exception), + ) From 1db32766b19d32e137b4deb9240b75af06235efe Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 30 Apr 2026 08:54:49 -0700 Subject: [PATCH 15/20] Add consistency check for sum of commitment duration and update test to avoid touching that error --- gtep/driver_esr.py | 2 +- gtep/gtep_model.py | 28 ++++- gtep/tests/unit/test_gtep_model.py | 192 ++++++++++++++++------------- 3 files changed, 131 insertions(+), 91 deletions(-) diff --git a/gtep/driver_esr.py b/gtep/driver_esr.py index 4db3d045..992ff03e 100644 --- a/gtep/driver_esr.py +++ b/gtep/driver_esr.py @@ -25,7 +25,7 @@ data_object = ExpansionPlanningData( stages=2, num_reps=2, - num_commit=6, + num_commit=24, num_dispatch=4, duration_representative_period=24, duration_commitment=1, diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 74ba5ec8..23341405 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -202,13 +202,29 @@ def convert_keys_to_int(obj): self.duration_commitment = period_dict["duration_commitment"] self.duration_dispatch = period_dict["duration_dispatch"] - # Consistency check: the sum of dispatch durations should - # equal the commitment duration + # Consistency checks: (1) the sum of commitment durations + # should equal the representative period duration and (2) the + # sum of dispatch durations should equal the commitment + # duration for rep in range(1, self.num_reps + 1): - for com in range(1, self.num_commit[rep] + 1): + # Consistency check (1): Sum commitment durations (in + # hours) + commitment_sum_hr = sum( + self.duration_commitment[rep][com] + for com in range(1, self.num_commit[rep] + 1) + ) + rep_period_hr = self.duration_representative_period[rep] + if abs(commitment_sum_hr - rep_period_hr) > 1e-6: + raise ValueError( + f"ERROR: The sum of commitment period durations ({commitment_sum_hr} hr) " + f"does not match the representative period duration ({rep_period_hr} hr) " + f"for representative period {rep}. " + "Please ensure these durations are consistent in your period structure data." + ) - # Sum dispatch durations (in minutes) and convert it - # to hours to compare commitment and dispatch duration + for com in range(1, self.num_commit[rep] + 1): + # Consistency check (2): Sum dispatch durations (in + # minutes) and convert it to hours dispatch_sum_hr = pyo.units.convert( sum( self.duration_dispatch[rep][com][disp] @@ -223,7 +239,7 @@ def convert_keys_to_int(obj): f"ERROR: The sum of dispatch period durations ({pyo.value(dispatch_sum_hr)} hr) " f"does not match the commitment period duration ({commitment_hr} hr) " f"for representative period {rep}, commitment period {com}. " - "Please ensure these durations are consistent in your period structure file ({})." + "Please ensure these durations are consistent in your period structure data." ) def create_model(self): diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index fdd66875..559ff8bb 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -128,14 +128,10 @@ def prepare_model_and_cost_data( @pytest.mark.usefixtures("patch_unit_handlers") class TestGTEP(unittest.TestCase): def test_model_init(self): - # Test that the ExpansionPlanningModel object can read a default dataset and init - # properly with default values, including building a Pyomo.ConcreteModel object - data_object = read_debug_model( - stages=1, - num_reps=3, - num_commit=6, - num_dispatch=4, - ) + # Test that the ExpansionPlanningModel object can read a + # default dataset and init properly with default values, + # including building a Pyomo.ConcreteModel object + data_object = read_debug_model() modObject = ExpansionPlanningModel(data=data_object) self.assertIsInstance(modObject, ExpansionPlanningModel) @@ -146,26 +142,43 @@ def test_model_init(self): self.assertEqual(modObject.formulation, None) self.assertIsInstance(modObject.model.md, ModelData) self.assertEqual(modObject.num_reps, 3) - self.assertEqual(modObject.num_commit, {1: 6, 2: 6, 3: 6}) + self.assertEqual(modObject.num_commit, {1: 24, 2: 24, 3: 24}) self.assertEqual( modObject.num_dispatch, { - 1: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, - 2: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, - 3: {1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}, + 1: {i: 4 for i in range(1, 25)}, + 2: {i: 4 for i in range(1, 25)}, + 3: {i: 4 for i in range(1, 25)}, }, ) self.assertEqual( modObject.duration_representative_period, {1: 24, 2: 24, 3: 24} ) + self.assertEqual( + modObject.duration_commitment, + { + 1: {i: 1 for i in range(1, 25)}, + 2: {i: 1 for i in range(1, 25)}, + 3: {i: 1 for i in range(1, 25)}, + }, + ) + self.assertEqual( + modObject.duration_dispatch, + { + 1: {com: {disp: 15 for disp in range(1, 5)} for com in range(1, 25)}, + 2: {com: {disp: 15 for disp in range(1, 5)} for com in range(1, 25)}, + 3: {com: {disp: 15 for disp in range(1, 5)} for com in range(1, 25)}, + }, + ) - # Test that the ExpansionPlanningModel object can read a default dataset and init - # properly with non-default values + # Test that the ExpansionPlanningModel object can read a + # default dataset and init properly with non-default values data_object = read_debug_model( stages=2, num_reps=4, num_commit=12, - num_dispatch=12, + num_dispatch=24, + duration_commitment=2, duration_dispatch=5, ) modObject = ExpansionPlanningModel( @@ -182,62 +195,31 @@ def test_model_init(self): self.assertEqual( modObject.num_dispatch, { - 1: { - 1: 12, - 2: 12, - 3: 12, - 4: 12, - 5: 12, - 6: 12, - 7: 12, - 8: 12, - 9: 12, - 10: 12, - 11: 12, - 12: 12, - }, - 2: { - 1: 12, - 2: 12, - 3: 12, - 4: 12, - 5: 12, - 6: 12, - 7: 12, - 8: 12, - 9: 12, - 10: 12, - 11: 12, - 12: 12, - }, - 3: { - 1: 12, - 2: 12, - 3: 12, - 4: 12, - 5: 12, - 6: 12, - 7: 12, - 8: 12, - 9: 12, - 10: 12, - 11: 12, - 12: 12, - }, - 4: { - 1: 12, - 2: 12, - 3: 12, - 4: 12, - 5: 12, - 6: 12, - 7: 12, - 8: 12, - 9: 12, - 10: 12, - 11: 12, - 12: 12, - }, + 1: {i: 24 for i in range(1, 13)}, + 2: {i: 24 for i in range(1, 13)}, + 3: {i: 24 for i in range(1, 13)}, + 4: {i: 24 for i in range(1, 13)}, + }, + ) + self.assertEqual( + modObject.duration_representative_period, {1: 24, 2: 24, 3: 24, 4: 24} + ) + self.assertEqual( + modObject.duration_commitment, + { + 1: {i: 2 for i in range(1, 13)}, + 2: {i: 2 for i in range(1, 13)}, + 3: {i: 2 for i in range(1, 13)}, + 4: {i: 2 for i in range(1, 13)}, + }, + ) + self.assertEqual( + modObject.duration_dispatch, + { + 1: {com: {disp: 5 for disp in range(1, 25)} for com in range(1, 13)}, + 2: {com: {disp: 5 for disp in range(1, 25)} for com in range(1, 13)}, + 3: {com: {disp: 5 for disp in range(1, 25)} for com in range(1, 13)}, + 4: {com: {disp: 5 for disp in range(1, 25)} for com in range(1, 13)}, }, ) @@ -277,6 +259,7 @@ def test_model_unit_consistency(self): num_reps=2, num_commit=2, num_dispatch=2, + duration_representative_period=2, duration_dispatch=30, ) modObject = ExpansionPlanningModel( @@ -315,7 +298,11 @@ def test_model_unit_consistency(self): def test_solve_bigm(self): # Solve the debug model as is data_object = read_debug_model( - num_reps=1, num_commit=1, num_dispatch=1, duration_dispatch=60 + num_reps=1, + num_commit=1, + num_dispatch=1, + duration_representative_period=1, + duration_dispatch=60, ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -334,7 +321,7 @@ def test_solve_bigm(self): modObject.results = opt.solve(modObject.model) # Previous successful objective values: 9207.95, 6078.86, - # 531860.15, 531883.43, 2127462.53 + # 531860.15, 531883.43 self.assertAlmostEqual( value(modObject.model.total_cost_objective_rule), 2127462.53, places=1 ) @@ -343,7 +330,11 @@ def test_solve_bigm(self): def test_no_investment(self): # Solve the debug model with no investment data_object = read_debug_model( - num_reps=1, num_commit=1, num_dispatch=1, duration_dispatch=60 + num_reps=1, + num_commit=1, + num_dispatch=1, + duration_representative_period=1, + duration_dispatch=60, ) modObject = ExpansionPlanningModel( data=data_object, @@ -366,7 +357,7 @@ def test_no_investment(self): modObject.results = opt.solve(modObject.model) - # previous successful objective values: 531860.15, 531883.43, 2127462.53 + # previous successful objective values: 531860.15, 531883.43 self.assertAlmostEqual( value(modObject.model.total_cost_objective_rule), 2127462.53, places=1 ) @@ -381,6 +372,7 @@ def test_with_cost_data_and_commitment(self): num_reps=2, num_commit=6, num_dispatch=4, + duration_representative_period=6, duration_dispatch=15, ) @@ -428,6 +420,7 @@ def test_with_cost_data_and_no_commitment(self): num_reps=2, num_commit=6, num_dispatch=4, + duration_representative_period=6, duration_dispatch=15, ) @@ -472,6 +465,7 @@ def test_period_structure_from_scalars(self): num_reps=2, num_commit=3, num_dispatch=4, + duration_representative_period=3, duration_dispatch=15, ) @@ -482,6 +476,7 @@ def test_period_structure_from_scalars(self): # Check that all values are as expected (all periods same) self.assertEqual(modObject.num_commit[1], 3) self.assertEqual(modObject.num_dispatch[2][3], 4) + self.assertEqual(modObject.duration_representative_period[1], 3) self.assertEqual(modObject.duration_commitment[1][2], 1) self.assertEqual(modObject.duration_dispatch[2][3][4], 15) @@ -494,12 +489,17 @@ def test_period_structure_from_json(self): "number_commitment": {1: 2, 2: 3}, "number_dispatch": {1: {1: 3, 2: 2}, 2: {1: 2, 2: 3, 3: 2}}, "duration_representative_period": {1: 24, 2: 18}, - "duration_commitment": {1: {1: 1, 2: 2}, 2: {1: 1, 2: 1.5, 3: 2}}, + "duration_commitment": {1: {1: 12, 2: 12}, 2: {1: 6, 2: 6, 3: 6}}, "duration_dispatch": { - 1: {1: {1: 10, 2: 20, 3: 30}, 2: {1: 30, 2: 90}}, - 2: {1: {1: 30, 2: 30}, 2: {1: 20, 2: 20, 3: 50}, 3: {1: 60, 2: 60}}, + 1: {1: {1: 360, 2: 180, 3: 180}, 2: {1: 360, 2: 360}}, + 2: { + 1: {1: 180, 2: 180}, + 2: {1: 120, 2: 120, 3: 120}, + 3: {1: 180, 2: 180}, + }, }, } + curr_dir = os.path.dirname(os.path.abspath(__file__)) json_path = os.path.join(curr_dir, "test_custom_period_structure.json") with open(json_path, "w") as f: @@ -520,17 +520,19 @@ def test_period_structure_from_json(self): self.assertEqual(modObject.num_reps, 2) self.assertEqual(modObject.num_commit[2], 3) self.assertEqual(modObject.num_dispatch[2][2], 3) - self.assertEqual(modObject.duration_commitment[2][2], 1.5) - self.assertEqual(modObject.duration_dispatch[2][2][3], 50) + self.assertEqual(modObject.duration_commitment[2][2], 6) + self.assertEqual(modObject.duration_dispatch[2][2][3], 120) self.assertEqual(modObject.duration_representative_period[2], 18) - self.assertEqual(modObject.duration_dispatch[1][1][2], 20) - self.assertEqual(modObject.duration_commitment[1][2], 2) + self.assertEqual(modObject.duration_dispatch[1][1][2], 180) + self.assertEqual(modObject.duration_commitment[1][2], 12) # Remove the .json file after the test os.remove(json_path) - def test_period_structure_consistency_error_with_scalars(self): - # Prepare model and cost data as usual + def test_period_structure_consistency_errors_with_scalars(self): + + # Prepare model and cost data to touch commitment consistency + # error dataObject, dataProcessingObject = prepare_model_and_cost_data( stages=1, num_reps=1, @@ -540,6 +542,28 @@ def test_period_structure_consistency_error_with_scalars(self): duration_commitment=1, duration_dispatch=60, ) + # The sum of commitment durations (1hr) does not match the + # representative period duration (24hr) + with self.assertRaises(ValueError) as cm: + ExpansionPlanningModel(data=dataObject, cost_data=dataProcessingObject) + + self.assertIn( + "ERROR: The sum of commitment period durations (1 hr) does not match the representative period duration (24 hr) for representative period 1", + str(cm.exception), + ) + + # Prepare model and cost data to touch dispatch consistency + # error + dataObject, dataProcessingObject = prepare_model_and_cost_data( + stages=1, + num_reps=1, + num_commit=1, + num_dispatch=2, + duration_representative_period=1, + duration_commitment=1, + duration_dispatch=60, + ) + # The sum of dispatch durations (2*60min = 120 min = 2hr) does # not match the commitment duration (1hr) with self.assertRaises(ValueError) as cm: From 730ba25928dcb2c4b4595f48d15939ec7bc86d41 Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 30 Apr 2026 08:57:02 -0700 Subject: [PATCH 16/20] Add new duration for representative period to avoid touching consistency error --- gtep/tests/unit/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index a654b8ba..339e5d26 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -46,6 +46,7 @@ def get_solution_object(): num_reps=2, num_commit=6, num_dispatch=4, + duration_representative_period=6, ) data_object.load_prescient(str(input_data_source)) From 7017099948ce5cceb133adfde3a5c26b42e81e2f Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Thu, 30 Apr 2026 14:56:29 -0700 Subject: [PATCH 17/20] Add header --- gtep/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gtep/utils.py b/gtep/utils.py index 562e3045..c61b0694 100644 --- a/gtep/utils.py +++ b/gtep/utils.py @@ -1,3 +1,16 @@ +################################################################################# +# The Institute for the Design of Advanced Energy Systems Integrated Platform +# Framework (IDAES IP) was produced under the DOE Institute for the +# Design of Advanced Energy Systems (IDAES). +# +# Copyright (c) 2018-2026 by the software owners: The Regents of the +# University of California, through Lawrence Berkeley National Laboratory, +# National Technology & Engineering Solutions of Sandia, LLC, Carnegie Mellon +# University, West Virginia University Research Corporation, et al. +# All rights reserved. Please see the files COPYRIGHT.md and LICENSE.md +# for full copyright and license information. +################################################################################# + import json From 2e6632fa5baa47a53ac47eeb21886a36e7d6a17d Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Fri, 1 May 2026 08:19:11 -0700 Subject: [PATCH 18/20] =?UTF-8?q?Use=20Pyomo=E2=80=99s=20TempfileManager?= =?UTF-8?q?=20to=20create=20a=20temporary=20directory=20and=20.json=20file?= =?UTF-8?q?=20to=20use=20in=20period=20structure=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gtep/tests/unit/test_gtep_model.py | 58 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 559ff8bb..6d2bf0ff 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -11,8 +11,8 @@ # for full copyright and license information. ################################################################################# -import os import json +from pathlib import Path from os.path import abspath, join, dirname import pytest @@ -25,6 +25,8 @@ _component_data_handlers, assert_units_equivalent, ) +from pyomo.common.tempfiles import TempfileManager + from gtep.gtep_model import ExpansionPlanningModel from gtep.gtep_data import ExpansionPlanningData from gtep.gtep_data_processing import DataProcessing @@ -481,9 +483,7 @@ def test_period_structure_from_scalars(self): self.assertEqual(modObject.duration_dispatch[2][3][4], 15) def test_period_structure_from_json(self): - # Test custom period structure with irregular values. This - # dictionary is saved as a .json file and then used to - # initialize the ExpansionPlanningModel class. + # Test custom period structure with irregular values. period_dict = { "number_representative": 2, "number_commitment": {1: 2, 2: 3}, @@ -500,34 +500,32 @@ def test_period_structure_from_json(self): }, } - curr_dir = os.path.dirname(os.path.abspath(__file__)) - json_path = os.path.join(curr_dir, "test_custom_period_structure.json") - with open(json_path, "w") as f: - json.dump(period_dict, f, indent=2) - - # Test that the model correctly reads and assigns the custom - # period structure values. Here we instantiate the model using - # the .json file. - dataObject, dataProcessingObject = prepare_model_and_cost_data( - period_structure_json_file=json_path, - ) - - modObject = ExpansionPlanningModel( - data=dataObject, cost_data=dataProcessingObject - ) + with TempfileManager.new_context() as tempfile: + temp_dir = Path(tempfile.mkdtemp()) + json_path = temp_dir / "test_custom_period_structure.json" + with open(json_path, "w") as f: + json.dump(period_dict, f, indent=2) + + # Test that the model correctly reads and assigns the + # custom period structure values. For this, we instantiate + # the model using the temp .json file. + dataObject, dataProcessingObject = prepare_model_and_cost_data( + period_structure_json_file=str(json_path), + ) - # Assert that we have the correct reading of the structure - self.assertEqual(modObject.num_reps, 2) - self.assertEqual(modObject.num_commit[2], 3) - self.assertEqual(modObject.num_dispatch[2][2], 3) - self.assertEqual(modObject.duration_commitment[2][2], 6) - self.assertEqual(modObject.duration_dispatch[2][2][3], 120) - self.assertEqual(modObject.duration_representative_period[2], 18) - self.assertEqual(modObject.duration_dispatch[1][1][2], 180) - self.assertEqual(modObject.duration_commitment[1][2], 12) + modObject = ExpansionPlanningModel( + data=dataObject, cost_data=dataProcessingObject + ) - # Remove the .json file after the test - os.remove(json_path) + # Assert that we have the correct reading of the structure + self.assertEqual(modObject.num_reps, 2) + self.assertEqual(modObject.num_commit[2], 3) + self.assertEqual(modObject.num_dispatch[2][2], 3) + self.assertEqual(modObject.duration_commitment[2][2], 6) + self.assertEqual(modObject.duration_dispatch[2][2][3], 120) + self.assertEqual(modObject.duration_representative_period[2], 18) + self.assertEqual(modObject.duration_dispatch[1][1][2], 180) + self.assertEqual(modObject.duration_commitment[1][2], 12) def test_period_structure_consistency_errors_with_scalars(self): From 34a7a292268e74b4268fafbf919dbc8842ea71eb Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Fri, 1 May 2026 13:13:20 -0700 Subject: [PATCH 19/20] Add functions that expand the period structure to a dictionary and the consistency check --- gtep/utils.py | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/gtep/utils.py b/gtep/utils.py index c61b0694..586ac989 100644 --- a/gtep/utils.py +++ b/gtep/utils.py @@ -11,8 +11,14 @@ # for full copyright and license information. ################################################################################# +import os import json +import pyomo.environ as pyo +from pyomo.environ import units as u + +curr_dir = os.path.dirname(os.path.abspath(__file__)) + def generate_period_structure_skeleton( num_reps, @@ -97,3 +103,146 @@ def generate_period_structure_utils( save_period_structure_json(period_dict, filename) return period_dict + + +def _set_period_structure_dict( + num_reps, + num_commit, + num_dispatch, + duration_representative_period, + duration_commitment, + duration_dispatch, + save_period_structure_file, + period_structure_json_file, +): + """This method returns a period structure dictionary with keys for + the number and duration of representative, commitment, and + dispatch periods. + + If a JSON file is specified, it loads the period structure from + that file. Otherwise, generates the period structure from the + provided scalar arguments. Optionally saves the generated + structure to a JSON file. + + Returns: + + :dict: period structure dictionary + + """ + + # If a .json file with period structure data is provided, use + # it, otherwise, expand to a dictionary using the provided + # scalars. + + if period_structure_json_file is not None: + # Use provided .json file + json_path = os.path.abspath( + os.path.join(curr_dir, "data", period_structure_json_file) + ) + with open(json_path, "r") as f: + period_dict = json.load(f) + + # Helper function to recursively convert string keys to + # integers + def convert_keys_to_int(obj): + if isinstance(obj, dict): + return { + ( + int(k) if isinstance(k, str) and k.isdigit() else k + ): convert_keys_to_int(v) + for k, v in obj.items() + } + else: + return obj + + period_dict = convert_keys_to_int(period_dict) + + else: + # .json file not provided; expand period structure + # dictionary from scalar arguments. Optionally save the + # expanded dictionary as a .json file with a default name + # under the data directory. + filename = ( + os.path.abspath( + os.path.join(curr_dir, "data", "period_structure_from_gtep.json") + ) + if save_period_structure_file + else None + ) + + period_dict = generate_period_structure_utils( + num_reps, + num_commit, + num_dispatch, + duration_representative_period, + duration_commitment, + duration_dispatch, + filename=filename, + ) + if save_period_structure_file: + print( + f"\nINFO: Period structure dictionary generated from scalar period arguments has been written to '{filename}'.\n" + ) + + return period_dict + + +def check_period_structure_consistency( + num_reps, + num_commit, + num_dispatch, + duration_representative_period, + duration_commitment, + duration_dispatch, +): + """This method checks that the sum of commitment and dispatch + durations equals the representative and commitment period + duration. It raises ValueError with details if mismatches are + found. + + """ + + commitment_errors = [] + dispatch_errors = [] + for rep in range(1, num_reps + 1): + # Consistency check (1): Sum commitment durations (in hours) + commitment_sum_hr = sum( + duration_commitment[rep][com] for com in range(1, num_commit[rep] + 1) + ) + rep_period_hr = duration_representative_period[rep] + if abs(commitment_sum_hr - rep_period_hr) > 1e-6: + commitment_errors.append( + f" - Representative period {rep}: sum of commitment durations ({commitment_sum_hr} hr) != representative period duration ({rep_period_hr} hr)" + ) + + for com in range(1, num_commit[rep] + 1): + # Consistency check (2): Sum dispatch durations (in + # minutes) and convert to hours + dispatch_sum_hr = pyo.units.convert( + sum( + duration_dispatch[rep][com][disp] + for disp in range(1, num_dispatch[rep][com] + 1) + ) + * u.minutes, + to_units=u.hours, + ) + commitment_hr = duration_commitment[rep][com] + if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: + dispatch_errors.append( + f" - Representative period {rep}, commitment period {com}: sum of dispatch durations ({pyo.value(dispatch_sum_hr)} hr) != commitment period duration ({commitment_hr} hr)" + ) + + # Raise an error if any mismatches were found + if commitment_errors or dispatch_errors: + msg = ["Period structure consistency check failed:\n"] + if commitment_errors: + msg.append( + f"ERROR: Found ({len(commitment_errors)}) mismatches for commitment period duration:\n" + + "\n".join(commitment_errors) + ) + if dispatch_errors: + msg.append( + f"ERROR: Found ({len(dispatch_errors)}) mismatches for dispatch period duration:\n" + + "\n".join(dispatch_errors) + ) + raise ValueError("\n".join(msg)) From 069625faed2f25890737a4608f21ca0fe9ceba1b Mon Sep 17 00:00:00 2001 From: Edna Soraya Rawlings Date: Fri, 1 May 2026 13:14:49 -0700 Subject: [PATCH 20/20] remove duration for commitment and dispatch as an argument in the class and calculate them based on the provided scalars. Also, reorganize functions and update tests to reflect these changes. --- gtep/driver_esr.py | 6 +- gtep/gtep_data.py | 6 -- gtep/gtep_model.py | 163 ++++++++--------------------- gtep/tests/unit/test_gtep_model.py | 121 ++++++++++----------- 4 files changed, 108 insertions(+), 188 deletions(-) diff --git a/gtep/driver_esr.py b/gtep/driver_esr.py index 992ff03e..e3c34496 100644 --- a/gtep/driver_esr.py +++ b/gtep/driver_esr.py @@ -25,11 +25,9 @@ data_object = ExpansionPlanningData( stages=2, num_reps=2, - num_commit=24, + num_commit=6, num_dispatch=4, - duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, + duration_representative_period=6, save_period_structure_file=False, period_structure_json_file=None, # period_structure_json_file="period_structure_from_gtep.json", diff --git a/gtep/gtep_data.py b/gtep/gtep_data.py index cfcc0241..d6a1ea70 100644 --- a/gtep/gtep_data.py +++ b/gtep/gtep_data.py @@ -36,8 +36,6 @@ def __init__( num_commit=24, num_dispatch=1, duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, save_period_structure_file=False, period_structure_json_file=None, ): @@ -49,8 +47,6 @@ def __init__( :param: num_dispatch: integer number of dispatch periods per commitment period :param: duration_representative_period: duration of each representative period (in hours) - :param: duration_commitment: duration of each commitment period (in hours) - :param: duration_dispatch: duration of each dispatch period (in minutes) :param: save_period_structure_file: (optional) If True, saves the generated period structure as a JSON file in the data directory. Default is False. :param: period_structure_json_file: (optional) Path to a JSON file in the data @@ -64,8 +60,6 @@ def __init__( self.num_commit = num_commit self.num_dispatch = num_dispatch self.duration_representative_period = duration_representative_period - self.duration_commitment = duration_commitment - self.duration_dispatch = duration_dispatch self.save_period_structure_file = save_period_structure_file self.period_structure_json_file = period_structure_json_file diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 23341405..88ecd61d 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -53,7 +53,10 @@ import gtep.model_library.storage as stor import gtep.model_library.transmission as transm -from gtep.utils import generate_period_structure_utils +from gtep.utils import ( + _set_period_structure_dict, + check_period_structure_consistency, +) curr_dir = os.path.dirname(os.path.abspath(__file__)) @@ -104,96 +107,46 @@ def __init__( :return: Pyomo model for full GTEP """ + self.config = _get_model_config() + self.timer = TicTocTimer() + self.stages = data.stages self.formulation = formulation self.data = data self.cost_data = cost_data - self.num_reps = data.num_reps - self.num_commit = data.num_commit - self.num_dispatch = data.num_dispatch - self.duration_dispatch = data.duration_dispatch - self.duration_representative_period = data.duration_representative_period - self.duration_commitment = data.duration_commitment self.save_period_structure_file = data.save_period_structure_file self.period_structure_json_file = data.period_structure_json_file - self.config = _get_model_config() - self.timer = TicTocTimer() - - # Set and validate period structure attributes from .json file - # or provided scalars. This function also implements a - # consistency check on dispatch and commitment durations. - self._set_period_structure_dict() - - _add_common_configs(self.config) - _add_investment_configs(self.config) - - def _set_period_structure_dict(self): - """This method initializes and validates the period structure - attributes (number and duration of representative, commitment, - and dispatch periods) from either a user-provided .json file - or from provided scalar arguments. - - This method performs a consistency check to ensure that the - sum of dispatch durations matches each commitment period - duration. - - """ - - # If a .json file with period structure data is provided, use - # it, otherwise, expand from scalars. - - if self.period_structure_json_file is not None: - # Use provided .json file - json_path = os.path.abspath( - os.path.join(curr_dir, "data", self.period_structure_json_file) + # Calculate commitment and dispatch period durations using + # provided scalars for the duration of representative period + # and number of commitment and dispatch periods. Note: We are + # assuming that all periods within their parent are of equal + # length. + duration_commitment = data.duration_representative_period / data.num_commit + duration_dispatch = pyo.value( + pyo.units.convert( + (duration_commitment / data.num_dispatch) * u.hours, + to_units=u.minutes, ) - with open(json_path, "r") as f: - period_dict = json.load(f) - - # Helper function to recursively convert string keys to - # integers - def convert_keys_to_int(obj): - if isinstance(obj, dict): - return { - ( - int(k) if isinstance(k, str) and k.isdigit() else k - ): convert_keys_to_int(v) - for k, v in obj.items() - } - else: - return obj - - period_dict = convert_keys_to_int(period_dict) + ) - else: - # .json file not provided; expand period structure - # dictionary from scalar arguments. Optionally save the - # expanded dictionary as a .json file with a default name - # under the data directory. - filename = ( - os.path.abspath( - os.path.join(curr_dir, "data", "period_structure_from_gtep.json") - ) - if self.save_period_structure_file - else None - ) - period_dict = generate_period_structure_utils( - self.num_reps, - self.num_commit, - self.num_dispatch, - self.duration_representative_period, - self.duration_commitment, - self.duration_dispatch, - filename=filename, - ) - if self.save_period_structure_file: - print( - f"\nINFO: Period structure dictionary generated from scalar period arguments has been written to '{filename}'.\n" - ) + # Generate the period structure dictionary from provided + # scalars, or load it from a .json file if specified. In + # either case, the period structure is returned as a + # dictionary. + period_dict = _set_period_structure_dict( + data.num_reps, + data.num_commit, + data.num_dispatch, + data.duration_representative_period, + duration_commitment, + duration_dispatch, + self.save_period_structure_file, + self.period_structure_json_file, + ) # Assign period structure attributes from the dictionary - self.num_reps = period_dict.get("number_representative", self.num_reps) + self.num_reps = period_dict.get("number_representative", data.num_reps) self.num_commit = period_dict["number_commitment"] self.num_dispatch = period_dict["number_dispatch"] self.duration_representative_period = period_dict[ @@ -202,45 +155,19 @@ def convert_keys_to_int(obj): self.duration_commitment = period_dict["duration_commitment"] self.duration_dispatch = period_dict["duration_dispatch"] - # Consistency checks: (1) the sum of commitment durations - # should equal the representative period duration and (2) the - # sum of dispatch durations should equal the commitment - # duration - for rep in range(1, self.num_reps + 1): - # Consistency check (1): Sum commitment durations (in - # hours) - commitment_sum_hr = sum( - self.duration_commitment[rep][com] - for com in range(1, self.num_commit[rep] + 1) - ) - rep_period_hr = self.duration_representative_period[rep] - if abs(commitment_sum_hr - rep_period_hr) > 1e-6: - raise ValueError( - f"ERROR: The sum of commitment period durations ({commitment_sum_hr} hr) " - f"does not match the representative period duration ({rep_period_hr} hr) " - f"for representative period {rep}. " - "Please ensure these durations are consistent in your period structure data." - ) + # Run a consistency check on commitment and dispatch + # durations. + check_period_structure_consistency( + self.num_reps, + self.num_commit, + self.num_dispatch, + self.duration_representative_period, + self.duration_commitment, + self.duration_dispatch, + ) - for com in range(1, self.num_commit[rep] + 1): - # Consistency check (2): Sum dispatch durations (in - # minutes) and convert it to hours - dispatch_sum_hr = pyo.units.convert( - sum( - self.duration_dispatch[rep][com][disp] - for disp in range(1, self.num_dispatch[rep][com] + 1) - ) - * u.minutes, - to_units=u.hours, - ) - commitment_hr = self.duration_commitment[rep][com] - if abs(pyo.value(dispatch_sum_hr) - commitment_hr) > 1e-6: - raise ValueError( - f"ERROR: The sum of dispatch period durations ({pyo.value(dispatch_sum_hr)} hr) " - f"does not match the commitment period duration ({commitment_hr} hr) " - f"for representative period {rep}, commitment period {com}. " - "Please ensure these durations are consistent in your period structure data." - ) + _add_common_configs(self.config) + _add_investment_configs(self.config) def create_model(self): """Create concrete Pyomo model object associated with the diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 6d2bf0ff..e255506b 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -57,8 +57,6 @@ def read_debug_model( num_commit=24, num_dispatch=4, duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, save_period_structure_file=False, period_structure_json_file=None, ): @@ -70,8 +68,6 @@ def read_debug_model( num_commit=num_commit, num_dispatch=num_dispatch, duration_representative_period=duration_representative_period, - duration_commitment=duration_commitment, - duration_dispatch=duration_dispatch, save_period_structure_file=save_period_structure_file, period_structure_json_file=period_structure_json_file, ) @@ -85,8 +81,6 @@ def prepare_model_and_cost_data( num_commit=24, num_dispatch=4, duration_representative_period=24, - duration_commitment=1, - duration_dispatch=15, save_period_structure_file=False, period_structure_json_file=None, ): @@ -97,8 +91,6 @@ def prepare_model_and_cost_data( num_commit, num_dispatch, duration_representative_period, - duration_commitment, - duration_dispatch, save_period_structure_file, period_structure_json_file, ) @@ -180,8 +172,7 @@ def test_model_init(self): num_reps=4, num_commit=12, num_dispatch=24, - duration_commitment=2, - duration_dispatch=5, + duration_representative_period=24, ) modObject = ExpansionPlanningModel( data=data_object, @@ -262,7 +253,6 @@ def test_model_unit_consistency(self): num_commit=2, num_dispatch=2, duration_representative_period=2, - duration_dispatch=30, ) modObject = ExpansionPlanningModel( data=data_object, @@ -304,7 +294,6 @@ def test_solve_bigm(self): num_commit=1, num_dispatch=1, duration_representative_period=1, - duration_dispatch=60, ) modObject = ExpansionPlanningModel(data=data_object) modObject.create_model() @@ -336,7 +325,6 @@ def test_no_investment(self): num_commit=1, num_dispatch=1, duration_representative_period=1, - duration_dispatch=60, ) modObject = ExpansionPlanningModel( data=data_object, @@ -375,7 +363,6 @@ def test_with_cost_data_and_commitment(self): num_commit=6, num_dispatch=4, duration_representative_period=6, - duration_dispatch=15, ) # Populate and create GTEP model @@ -423,7 +410,6 @@ def test_with_cost_data_and_no_commitment(self): num_commit=6, num_dispatch=4, duration_representative_period=6, - duration_dispatch=15, ) # Populate and create GTEP model @@ -462,28 +448,28 @@ def test_with_cost_data_and_no_commitment(self): assert_units_equivalent(modObject.model.total_cost_objective_rule.expr, u.USD) def test_period_structure_from_scalars(self): - # Test with scalar/list arguments (all periods same) + # Test period structure dictionary created using the provided + # scalars dataObject, dataProcessingObject = prepare_model_and_cost_data( num_reps=2, num_commit=3, num_dispatch=4, duration_representative_period=3, - duration_dispatch=15, ) modObject = ExpansionPlanningModel( data=dataObject, cost_data=dataProcessingObject ) - # Check that all values are as expected (all periods same) + # Assert that all values are as expected self.assertEqual(modObject.num_commit[1], 3) self.assertEqual(modObject.num_dispatch[2][3], 4) self.assertEqual(modObject.duration_representative_period[1], 3) self.assertEqual(modObject.duration_commitment[1][2], 1) self.assertEqual(modObject.duration_dispatch[2][3][4], 15) - def test_period_structure_from_json(self): - # Test custom period structure with irregular values. + # Test custom period structure dictionary with irregular + # values. period_dict = { "number_representative": 2, "number_commitment": {1: 2, 2: 3}, @@ -527,47 +513,62 @@ def test_period_structure_from_json(self): self.assertEqual(modObject.duration_dispatch[1][1][2], 180) self.assertEqual(modObject.duration_commitment[1][2], 12) - def test_period_structure_consistency_errors_with_scalars(self): + def test_period_structure_consistency_errors(self): - # Prepare model and cost data to touch commitment consistency - # error - dataObject, dataProcessingObject = prepare_model_and_cost_data( - stages=1, - num_reps=1, - num_commit=1, - num_dispatch=2, - duration_representative_period=24, - duration_commitment=1, - duration_dispatch=60, - ) - # The sum of commitment durations (1hr) does not match the - # representative period duration (24hr) - with self.assertRaises(ValueError) as cm: - ExpansionPlanningModel(data=dataObject, cost_data=dataProcessingObject) - - self.assertIn( - "ERROR: The sum of commitment period durations (1 hr) does not match the representative period duration (24 hr) for representative period 1", - str(cm.exception), - ) + # Check that a consistency error is raised if + # commitment/dispatch period durations do not sum to the + # representative/commitment period duration when loading from + # a .json file. The test first creates and writes the .json + # file, then checks for the error. + period_dict = { + "number_representative": 2, + "number_commitment": {"1": 6, "2": 6}, + "number_dispatch": { + "1": {"1": 4, "2": 4, "3": 4, "4": 4, "5": 4, "6": 4}, + "2": {"1": 4, "2": 4, "3": 4, "4": 4, "5": 4, "6": 4}, + }, + "duration_representative_period": {"1": 6, "2": 6}, + "duration_commitment": { + "1": {"1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 3.0}, + "2": {"1": 1.0, "2": 1.0, "3": 2.0, "4": 1.0, "5": 1.0, "6": 1.0}, + }, + "duration_dispatch": { + "1": { + "1": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "2": {"1": 25.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "3": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "4": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "5": {"1": 15.0, "2": 15.0, "3": 5.0, "4": 15.0}, + "6": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + }, + "2": { + "1": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "2": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "3": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "4": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "5": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + "6": {"1": 15.0, "2": 15.0, "3": 15.0, "4": 15.0}, + }, + }, + } - # Prepare model and cost data to touch dispatch consistency - # error - dataObject, dataProcessingObject = prepare_model_and_cost_data( - stages=1, - num_reps=1, - num_commit=1, - num_dispatch=2, - duration_representative_period=1, - duration_commitment=1, - duration_dispatch=60, - ) + with TempfileManager.new_context() as tempfile: + temp_dir = Path(tempfile.mkdtemp()) + json_path = temp_dir / "test_consistency_errors_period_structure.json" + with open(json_path, "w") as f: + json.dump(period_dict, f, indent=2) + + # Instantiate the model using the temp .json file. + dataObject, dataProcessingObject = prepare_model_and_cost_data( + period_structure_json_file=str(json_path), + ) - # The sum of dispatch durations (2*60min = 120 min = 2hr) does - # not match the commitment duration (1hr) - with self.assertRaises(ValueError) as cm: - ExpansionPlanningModel(data=dataObject, cost_data=dataProcessingObject) + # Assert that the sum of commitment durations (20hr) does + # not match the representative period duration (18hr) + with self.assertRaises(ValueError) as cm: + ExpansionPlanningModel(data=dataObject, cost_data=dataProcessingObject) - self.assertIn( - "ERROR: The sum of dispatch period durations (2.0 hr) does not match the commitment period duration (1 hr)", - str(cm.exception), - ) + self.assertIn( + "Period structure consistency check failed:\n\nERROR: Found (2) mismatches for commitment period duration:", + str(cm.exception), + )