Skip to content

Iteration yaml doesn't generate while populating DLCModelTraining (V1) #1636

Description

@rpswenson

Describe the bug
A yaml file seems like it needs to be generated in the dlc-model folder of a project at some point before populating DLCModelTraining, and that step is being skipped. My guess is this is due to a discrepancy in my DLC V1 environment, since I recently had to remake it with Chris's help.

To Reproduce

model_training_key = 
{'project_name': 'RS6420260613_run_V1',
 'dlc_training_params_name': 'tutorial_gpu2',
 'training_id': 1}

sgp.DLCModelTraining.populate(model_training_key)
Error Stack
FileNotFoundError                         Traceback (most recent call last)
Cell In[3], line 63
     46 sgp.DLCModelTrainingSelection().insert1(
     47    {
     48        **project_key,
   (...)
     53     skip_duplicates=True,
     54 )
     55 model_training_key = (
     56     sgp.DLCModelTrainingSelection
     57     & {
   (...)
     60     }
     61 ).fetch1("KEY")
---> 63 sgp.DLCModelTraining.populate(model_training_key)

File [~/spyglass/src/spyglass/utils/mixins/populate.py:98](http://localhost:6503/home/rswenson/spyglass/src/spyglass/utils/mixins/populate.py#line=97), in PopulateMixin.populate(self, *restrictions, **kwargs)
     96 if use_transact:  # Pass single-process populate to super
     97     kwargs["processes"] = processes
---> 98     return super().populate(*restrictions, **kwargs)
     99 else:  # No transaction protection, use bare make
    100     for key in keys:

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py:321](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py#line=320), in AutoPopulate.populate(self, keys, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions)
    315 if processes == 1:
    316     for key in (
    317         tqdm(keys, desc=self.__class__.__name__)
    318         if display_progress
    319         else keys
    320     ):
--> 321         status = self._populate1(key, jobs, **populate_kwargs)
    322         if status is True:
    323             success_list.append(1)

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py:407](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py#line=406), in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs)
    403 fetched_data = next(gen)
    404 fetch_hash = deepdiff.DeepHash(
    405     fetched_data, ignore_iterable_order=False
    406 )[fetched_data]
--> 407 computed_result = next(gen)  # perform the computation
    408 # fetch and insert inside a transaction
    409 self.connection.start_transaction()

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py:164](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/datajoint/autopopulate.py#line=163), in AutoPopulate.make(self, key, **kwargs)
    161 # Step 2: If computed result is not passed in, compute the result
    162 if computed_result is None:
    163     # this is only executed in the first invocation
--> 164     computed_result = self.make_compute(key, *fetched_data)
    165     yield computed_result  # this is passed to the second invocation of make
    167 # Step 3: Insert the computed result into the current table.

File [~/spyglass/src/spyglass/position/utils/general.py:174](http://localhost:6503/home/rswenson/spyglass/src/spyglass/position/utils/general.py#line=173), in file_log.<locals>.decorator.<locals>.wrapper(self, *args, **kwargs)
    172     logger.removeHandler(logger.handlers[0])
    173 try:
--> 174     return func(self, *args, **kwargs)
    175 finally:
    176     if not console:

File [~/spyglass/src/spyglass/position/v1/position_dlc_training.py:249](http://localhost:6503/home/rswenson/spyglass/src/spyglass/position/v1/position_dlc_training.py#line=248), in DLCModelTraining.make_compute(self, key, model_prefix, config_path, project_name, params, training_filelist)
    247 try:
    248     with suppress_print_from_package():
--> 249         train_network(dlc_cfg_filepath, **train_network_kwargs)
    250 except KeyboardInterrupt:  # pragma: no cover
    251     self._info_msg("DLC training stopped via Keyboard Interrupt")

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/training.py:286](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/training.py#line=285), in train_network(config, shuffle, trainingsetindex, max_snapshots_to_keep, displayiters, saveiters, maxiters, allow_growth, gputouse, autotune, keepdeconvweights, modelprefix, superanimal_name, superanimal_transfer_learning)
    275         train(
    276             str(poseconfigfile),
    277             displayiters,
   (...)
    282             allow_growth=allow_growth,
    283         )  # pass on path and file name for pose_cfg.yaml!
    285 except BaseException as e:
--> 286     raise e
    287 finally:
    288     os.chdir(str(start_path))

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/training.py:201](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/pose_estimation_tensorflow/training.py#line=200), in train_network(config, shuffle, trainingsetindex, max_snapshots_to_keep, displayiters, saveiters, maxiters, allow_growth, gputouse, autotune, keepdeconvweights, modelprefix, superanimal_name, superanimal_transfer_learning)
    199         os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
    200 try:
--> 201     cfg_dlc = auxiliaryfunctions.read_plainconfig(poseconfigfile)
    203     if superanimal_name != "":
    204         from deeplabcut.modelzoo.utils import parse_available_supermodels

File [~/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/utils/auxiliaryfunctions.py:292](http://localhost:6503/home/rswenson/miniforge3/envs/spyglass-pos1/lib/python3.10/site-packages/deeplabcut/utils/auxiliaryfunctions.py#line=291), in read_plainconfig(configname)
    290 def read_plainconfig(configname):
    291     if not os.path.exists(configname):
--> 292         raise FileNotFoundError(
    293             f"Config {configname} is not found. Please make sure that the file exists."
    294         )
    295     with open(configname) as file:
    296         return YAML().load(file)

FileNotFoundError: Config /nimbus/deeplabcut/projects/RS6420260613_run_V1-mcoulter_section-2026-07-06/dlc-models/iteration-0/RS6420260613_run_V1Jul6-trainset95shuffle1/train/pose_cfg.yaml is not found. Please make sure that the file exists.

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Fields

No fields configured for issues without a type.

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions