When I was running the training script, the following error occurred. Do you know what happened?:
Error executing job with overrides: ['+experiment=re10k']
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 436, in wrapped_fn_impl
param_fn(*args, **kwargs)
File "<@beartype(src.visualization.drawing.lines.draw_lines) at 0x7fc369252f80>", line 90, in draw_lines
beartype.roar.BeartypeCallHintParamViolation: Function src.visualization.drawing.lines.draw_lines() parameter start="tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')" violates type hint typing.Union[float, int, typing.Iterable[typing.Union[float, int]], jaxtyping.Shaped[Tensor, '3'], jaxtyping.Shaped[Tensor, 'batch 3']], as <protocol "torch.Tensor"> "tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')":
- Not int, float, <class "jaxtyping.Shaped[Tensor, 'batch 3']">, or <class "jaxtyping.Shaped[Tensor, '3']">.
- <protocol "torch.Tensor"> index 0 item <protocol "torch.Tensor"> "tensor([-0.0700, 0.1242], device='cuda:0')" not int or float.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 753, in _get_problem_arg
fn(*args, **kwargs)
File "<@beartype(src.visualization.drawing.lines.check_single_arg) at 0x7fc3f971d7e0>", line 66, in check_single_arg
beartype.roar.BeartypeCallHintParamViolation: Function src.visualization.drawing.lines.check_single_arg() parameter start="tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')" violates type hint typing.Union[float, int, typing.Iterable[typing.Union[float, int]], jaxtyping.Shaped[Tensor, '3'], jaxtyping.Shaped[Tensor, 'batch 3']], as <protocol "torch.Tensor"> "tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')":
- Not int, float, <class "jaxtyping.Shaped[Tensor, 'batch 3']">, or <class "jaxtyping.Shaped[Tensor, '3']">.
- <protocol "torch.Tensor"> index 0 item <protocol "torch.Tensor"> "tensor([-0.0700, 0.1242], device='cuda:0')" not int or float.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 441, in wrapped_fn_impl
argmsg = _get_problem_arg(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 756, in _get_problem_arg
raise TypeCheckError(
jaxtyping.TypeCheckError:
The problem arose whilst typechecking parameter 'start'.
Actual value: f3224,2
Expected type: typing.Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']].
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/data1/gjx/NoPoSplat/src/main.py", line 157, in train
trainer.fit(model_wrapper, datamodule=data_module, ckpt_path=checkpoint_path)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 561, in fit
call._call_and_handle_interrupt(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 47, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
return function(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 599, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1012, in _run
results = self._run_stage()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1054, in _run_stage
self._run_sanity_check()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1083, in _run_sanity_check
val_loop.run()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/utilities.py", line 179, in _decorator
return loop_run(self, *args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 145, in run
self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 437, in _evaluation_step
output = call._call_strategy_hook(trainer, hook_name, *step_args)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 328, in _call_strategy_hook
output = fn(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 411, in validation_step
return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 641, in call
wrapper_output = wrapper_module(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
else self._run_ddp_forward(*inputs, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
return self.module(*inputs, **kwargs) # type: ignore[index]
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 634, in wrapped_forward
out = method(_args, **_kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 41, in wrapped_fn
return fn(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/model/model_wrapper.py", line 460, in validation_step
cameras = hcat(*render_cameras(batch, 256))
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/visualization/validation_in_3d.py", line 104, in render_cameras
return draw_cameras(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/visualization/drawing/cameras.py", line 64, in draw_cameras
image = draw_lines(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 470, in wrapped_fn_impl
raise TypeCheckError(msg) from e
jaxtyping.TypeCheckError: Type-check error whilst checking the parameters of src.visualization.drawing.lines.draw_lines.
The problem arose whilst typechecking parameter 'start'.
Actual value: f3224,2
Expected type: typing.Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']].
Called with parameters: {
'image': f323,256,256,
'start': f3224,2,
'end': f3224,2,
'color': 0.25,
'width': 2,
'cap': 'round',
'num_msaa_passes': 1,
'x_range': f322,
'y_range': f322
}
Parameter annotations: (image: Float[Tensor, '3 height width'], start: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], end: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], color: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], width: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, ''], Shaped[Tensor, 'batch']], cap: Literal['butt', 'round', 'square'] = 'round', num_msaa_passes: int = 1, x_range: Union[Iterable[Union[float, int]], Shaped[Tensor, '2'], NoneType] = None, y_range: Union[Iterable[Union[float, int]], Shaped[Tensor, '2'], NoneType] = None) -> Any.
The current values for each jaxtyping axis annotation are as follows.
height=256
width=256
batch=24
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
When I was running the training script, the following error occurred. Do you know what happened?:
Error executing job with overrides: ['+experiment=re10k']
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 436, in wrapped_fn_impl
param_fn(*args, **kwargs)
File "<@beartype(src.visualization.drawing.lines.draw_lines) at 0x7fc369252f80>", line 90, in draw_lines
beartype.roar.BeartypeCallHintParamViolation: Function src.visualization.drawing.lines.draw_lines() parameter start="tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')" violates type hint typing.Union[float, int, typing.Iterable[typing.Union[float, int]], jaxtyping.Shaped[Tensor, '3'], jaxtyping.Shaped[Tensor, 'batch 3']], as <protocol "torch.Tensor"> "tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')":
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 753, in _get_problem_arg
fn(*args, **kwargs)
File "<@beartype(src.visualization.drawing.lines.check_single_arg) at 0x7fc3f971d7e0>", line 66, in check_single_arg
beartype.roar.BeartypeCallHintParamViolation: Function src.visualization.drawing.lines.check_single_arg() parameter start="tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')" violates type hint typing.Union[float, int, typing.Iterable[typing.Union[float, int]], jaxtyping.Shaped[Tensor, '3'], jaxtyping.Shaped[Tensor, 'batch 3']], as <protocol "torch.Tensor"> "tensor([[-0.0700, 0.1242],
[-0.0700, 0.1242],
[ 0.0700, 0.1242],
...')":
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 441, in wrapped_fn_impl
argmsg = _get_problem_arg(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 756, in _get_problem_arg
raise TypeCheckError(
jaxtyping.TypeCheckError:
The problem arose whilst typechecking parameter 'start'.
Actual value: f3224,2
Expected type: typing.Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']].
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/data1/gjx/NoPoSplat/src/main.py", line 157, in train
trainer.fit(model_wrapper, datamodule=data_module, ckpt_path=checkpoint_path)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 561, in fit
call._call_and_handle_interrupt(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 47, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
return function(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 599, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1012, in _run
results = self._run_stage()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1054, in _run_stage
self._run_sanity_check()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1083, in _run_sanity_check
val_loop.run()
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/utilities.py", line 179, in _decorator
return loop_run(self, *args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 145, in run
self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 437, in _evaluation_step
output = call._call_strategy_hook(trainer, hook_name, *step_args)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 328, in _call_strategy_hook
output = fn(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 411, in validation_step
return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 641, in call
wrapper_output = wrapper_module(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
else self._run_ddp_forward(*inputs, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
return self.module(*inputs, **kwargs) # type: ignore[index]
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 634, in wrapped_forward
out = method(_args, **_kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 41, in wrapped_fn
return fn(*args, **kwargs)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/model/model_wrapper.py", line 460, in validation_step
cameras = hcat(*render_cameras(batch, 256))
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/visualization/validation_in_3d.py", line 104, in render_cameras
return draw_cameras(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 473, in wrapped_fn_impl
out = fn(*args, **kwargs)
File "/data1/gjx/NoPoSplat/src/visualization/drawing/cameras.py", line 64, in draw_cameras
image = draw_lines(
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 549, in wrapped_fn
return wrapped_fn_impl(args, kwargs, bound, memos)
File "/mnt/lab/gjx/data1/miniconda/envs/noposplat/lib/python3.10/site-packages/jaxtyping/_decorator.py", line 470, in wrapped_fn_impl
raise TypeCheckError(msg) from e
jaxtyping.TypeCheckError: Type-check error whilst checking the parameters of src.visualization.drawing.lines.draw_lines.
The problem arose whilst typechecking parameter 'start'.
Actual value: f3224,2
Expected type: typing.Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']].
Called with parameters: {
'image': f323,256,256,
'start': f3224,2,
'end': f3224,2,
'color': 0.25,
'width': 2,
'cap': 'round',
'num_msaa_passes': 1,
'x_range': f322,
'y_range': f322
}
Parameter annotations: (image: Float[Tensor, '3 height width'], start: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], end: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], color: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, '3'], Shaped[Tensor, 'batch 3']], width: Union[float, int, Iterable[Union[float, int]], Shaped[Tensor, ''], Shaped[Tensor, 'batch']], cap: Literal['butt', 'round', 'square'] = 'round', num_msaa_passes: int = 1, x_range: Union[Iterable[Union[float, int]], Shaped[Tensor, '2'], NoneType] = None, y_range: Union[Iterable[Union[float, int]], Shaped[Tensor, '2'], NoneType] = None) -> Any.
The current values for each jaxtyping axis annotation are as follows.
height=256
width=256
batch=24
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.