Fix: Default disable auto recovery on motor loss comm (#62)

* feat(dm_driver): make motor auto-recovery opt-in, fail-fast by default

- i2rt/motor_drivers/dm_driver.py: added enable_auto_recovery constructor flag (default False, read live each control-loop iteration); gated both control-loop recovery paths (the RuntimeError "Motor error detected" catch and the feedback error_code check) behind it, so a loss-comm error (0xD) now sets running=False and re-raises instead of silently clearing and re-enabling the motor
- i2rt/robots/get_robot.py: threaded enable_auto_recovery through get_yam_robot and _get_gripper_only_robot into the DMChainCanInterface constructor (default False)
- i2rt/robots/motor_chain_robot.py: added enable_auto_recovery override on MotorChainRobot (None = inherit the chain's setting) and exposed the effective value in the robot info dict

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(flow_base): explicitly disable motor auto-recovery on the mobile base

- i2rt/flow_base/flow_base_controller.py: pass enable_auto_recovery=False at the
  holonomic_base and unified_motor_chain DMChainCanInterface constructions so the
  base's fail-fast intent is explicit at the call sites (ROB-1449); functional no-op
  since the constructor default is already False.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: nu-jliu

i2rt/flow_base/flow_base_controller.py

@@ -160,6 +160,7 @@ def _initialize_motor_chain(
             channel=channel,
             motor_chain_name="holonomic_base",
             control_mode=ControlMode.VEL,
+            enable_auto_recovery=False,  # fail-fast on motor error (ROB-1449); base does not self-heal
         )
         return motor_interface
 
@@ -676,6 +677,7 @@ def __init__(
             motor_chain_name="linear_rail_vehicle" if enable_linear_rail else "holonomic_base",
             control_mode=ControlMode.VEL,
             control_freq=control_freq,
+            enable_auto_recovery=False,  # fail-fast on motor error (ROB-1449); base does not self-heal
         )
 
         # Initialize brake GPIO only if linear rail is enabled

i2rt/motor_drivers/dm_driver.py

@@ -384,6 +384,7 @@ def __init__(
         use_buffered_reader: bool = False,  # buffered reader is not very stable, the latest encoder fix allows us to use the non-buffered reader
         report_interval: float = REPORT_INTERVAL,
         control_freq: float = CONTROL_FREQ,  # Control loop frequency (Hz), used for the CAN bandwidth check
+        enable_auto_recovery: bool = False,  # if True, try to clean+re-enable errored motors in the control loop instead of failing fast
     ):
         assert not use_buffered_reader, (
             "buffered reader is not very stable, the latest encoder fix allows us to use the non-buffered reader"
@@ -401,6 +402,9 @@ def __init__(
         self.motor_offset = np.array(motor_offset, dtype=float)
         self.motor_direction = np.array(motor_direction)
         self.channel = channel
+        # Read live each control-loop iteration; must be set before _motor_on()/start_thread() since
+        # some callers (e.g. _get_gripper_only_robot) start the thread inside this constructor.
+        self.enable_auto_recovery = enable_auto_recovery
         logging.info(f"Channel: {channel}, Bitrate: {bitrate}")
         if "can" in channel:
             self.motor_interface = DMSingleMotorCanInterface(
@@ -578,29 +582,25 @@ def _set_torques_and_update_state(self) -> None:
                         try:
                             motor_feedback = self._set_commands(self.commands)
                         except RuntimeError as e:
-                            if "Motor error detected" in str(e):
+                            if self.enable_auto_recovery and "Motor error detected" in str(e):
                                 logging.warning(f"Motor error in control loop, attempting recovery: {e}")
-                                recovered = self._try_recover_motors()
-                                if recovered:
+                                if self._try_recover_motors():
                                     logging.warning("Motor recovery successful, continuing control loop")
                                     continue
-                                else:
-                                    self.running = False
-                                    raise
+                                self.running = False
+                                raise
                             raise
 
                         errors = np.array([motor_feedback[i].error_code != "0x1" for i in range(len(motor_feedback))])
                         if np.any(errors):
-                            logging.warning(f"Motor errors detected in feedback: {errors}")
-                            recovered = self._try_recover_motors(motor_feedback)
-                            if recovered:
-                                logging.warning("Motor recovery successful, continuing control loop")
-                                continue
+                            if self.enable_auto_recovery:
+                                logging.warning(f"Motor errors detected in feedback: {errors}, attempting recovery")
+                                if self._try_recover_motors(motor_feedback):
+                                    logging.warning("Motor recovery successful, continuing control loop")
+                                    continue
                             self.running = False
                             logging.error(f"motor errors: {errors}")
-                            raise Exception(
-                                "motors have unrecoverable errors after recovery attempts, stopping control loop"
-                            )
+                            raise Exception(f"motor errors detected: {errors}, stopping control loop")
 
                     with self.state_lock:
                         self.state = motor_feedback

i2rt/robots/get_robot.py

@@ -62,13 +62,16 @@ def _get_gripper_only_robot(
     channel: str = "can0",
     gripper_type: GripperType = GripperType.LINEAR_4310,
     sim: bool = False,
+    enable_auto_recovery: bool = False,
 ) -> "Robot":
     """Create a gripper-only robot (no arm).
 
     Args:
         channel: CAN interface name (e.g. "can0"). Ignored in sim mode.
         gripper_type: Which gripper to load. Must not be NO_GRIPPER.
         sim: If True, return a SimRobot instead of connecting to real hardware.
+        enable_auto_recovery: If True, the motor chain tries to clean+re-enable errored motors in its
+            control loop instead of failing fast. Defaults to False (fail-fast).
     """
     if gripper_type == GripperType.NO_GRIPPER:
         raise ValueError("gripper_type cannot be NO_GRIPPER when arm_type is NO_ARM")
@@ -108,6 +111,7 @@ def _get_gripper_only_robot(
         motor_chain_name="gripper_only",
         receive_mode=ReceiveMode.p16,
         start_thread=True,
+        enable_auto_recovery=enable_auto_recovery,
     )
 
     return MotorChainRobot(
@@ -140,6 +144,7 @@ def get_yam_robot(
     sim: bool = False,
     joint_state_saver_factory: Optional[Callable[[], Any]] = None,
     set_realtime_and_pin_callback: Optional[Callable[[int], None]] = None,
+    enable_auto_recovery: bool = False,
 ) -> "Robot":
     """Create a YAM-family robot (real or sim).
 
@@ -156,10 +161,14 @@ def get_yam_robot(
         gripper_kp: Optional gripper kp override. Defaults to gripper_type's default.
         gripper_kd: Optional gripper kd override. Defaults to gripper_type's default.
         sim: If True, return a SimRobot instead of connecting to real hardware.
+        enable_auto_recovery: If True, the motor chain tries to clean+re-enable errored motors in its
+            control loop instead of failing fast. Defaults to False (fail-fast).
     """
     # --- Gripper-only path (no arm) -------------------------------------------
     if arm_type == ArmType.NO_ARM:
-        return _get_gripper_only_robot(channel=channel, gripper_type=gripper_type, sim=sim)
+        return _get_gripper_only_robot(
+            channel=channel, gripper_type=gripper_type, sim=sim, enable_auto_recovery=enable_auto_recovery
+        )
 
     with_gripper = gripper_type not in (GripperType.YAM_TEACHING_HANDLE, GripperType.NO_GRIPPER)
     with_teaching_handle = gripper_type == GripperType.YAM_TEACHING_HANDLE
@@ -245,6 +254,7 @@ def get_yam_robot(
         start_thread=False,
         get_same_bus_device_driver=get_encoder_chain if with_teaching_handle else None,
         use_buffered_reader=False,
+        enable_auto_recovery=enable_auto_recovery,
     )
     motor_states = motor_chain.read_states()
     logging.debug(f"motor_states: {motor_states}")

i2rt/robots/motor_chain_robot.py

@@ -93,6 +93,7 @@ def __init__(
         pinned_cpu: int | None = None,
         joint_state_saver_factory: Optional[Callable[[], Any]] = None,
         set_realtime_and_pin_callback: Optional[Callable[[int], None]] = None,
+        enable_auto_recovery: Optional[bool] = None,  # None: inherit motor_chain's setting; True/False: override it
     ) -> None:
         # Set up CPU pinning and real-time scheduling if requested
         if pinned_cpu is not None and set_realtime_and_pin_callback is not None:
@@ -138,6 +139,10 @@ def __init__(
         assert clip_motor_torque >= 0.0
         self._clip_motor_torque = clip_motor_torque
         self.motor_chain = motor_chain
+        # None means inherit whatever the chain was constructed with; an explicit value overrides it.
+        # The chain reads this flag live each control-loop iteration, so a late set is safe.
+        if enable_auto_recovery is not None:
+            self.motor_chain.enable_auto_recovery = enable_auto_recovery
         self.use_gravity_comp = use_gravity_comp
         self.gravity_comp_factor = (
             gravity_comp_factor if gravity_comp_factor is not None else np.ones(len(motor_chain))
@@ -303,6 +308,7 @@ def get_robot_info(self) -> Dict[str, Any]:
             "gripper_limits": self._gripper_limits,
             "gravity_comp_factor": self.gravity_comp_factor,
             "gripper_index": self._gripper_index,
+            "enable_auto_recovery": getattr(self.motor_chain, "enable_auto_recovery", False),
         }
         if self._gripper_index is not None:
             info["limit_gripper_effort"] = self._limit_gripper_force