From f7452943ba4fdbc277d95b64a08c055395118044 Mon Sep 17 00:00:00 2001
From: ronaldnetawat <ronnetawat@gmail.com>
Date: Wed, 29 Jul 2026 12:38:47 -0700
Subject: [PATCH] Fix: teleop freeze from portal signal-pipe leak in
 minimum_gello

The leader's foreground io loop drove portal RPC unpaced (~7 kHz) and
fired command_joint_pos fire-and-forget immediately followed by
get_joint_pos. portal's client wakes its socket thread by writing one
byte to a signal pipe per request but drains it at most once per
send-queue-emptying, so two requests merged into one send burst leak
one pipe byte per cycle. After ~2-3 minutes of synced teleop the 64 KiB
pipe fills and send() blocks forever in os.write: the leader silently
stops forwarding commands while its control worker keeps running, so
the follower freezes rigid at its last commanded pose and the desync
button appears dead. Confirmed twice with py-spy (MainThread stuck at
portal client_socket.py send/os.write), reproducibly minutes after
enabling sync and never while idle (one request per cycle happens to be
drain-balanced).

Fixes:
- Pace _run_leader_io_loop and _rpc_polling_worker with
  _WORKER_LOOP_PERIOD_S like every other loop in this file (these two
  were missed; 500 Hz is well above the ~270 Hz hardware rate).
- Call .result() on ClientRobot.command_joint_pos/command_joint_state.
  The wait is load-bearing: completing each request before issuing the
  next keeps every request in its own send burst, so pipe writes and
  drains stay exactly 1:1 balanced indefinitely.

Verified on real hardware (YAM leader + YAM/linear_4310 follower over
CAN): 10+ minutes of continuous synced teleop with zero io stalls,
versus a reliable freeze within ~3 minutes before the fix.
---
 examples/minimum_gello/minimum_gello.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/examples/minimum_gello/minimum_gello.py b/examples/minimum_gello/minimum_gello.py
index ead0d6cd..1c7fe8d6 100644
--- a/examples/minimum_gello/minimum_gello.py
+++ b/examples/minimum_gello/minimum_gello.py
@@ -47,10 +47,18 @@ def get_joint_pos(self) -> np.ndarray:
         return self._client.get_joint_pos().result()
 
     def command_joint_pos(self, joint_pos: np.ndarray) -> None:
-        self._client.command_joint_pos(joint_pos)
+        # .result() is load-bearing, not just error propagation. portal's client wakes its socket
+        # thread by writing one byte to a signal pipe per request, but drains that pipe only once
+        # per send-queue-emptying (portal/client_socket.py). A fire-and-forget command followed
+        # immediately by another call lands both requests in one send burst: two bytes written, one
+        # drained — a leak that fills the 64 KiB pipe in minutes of teleop, after which send()
+        # blocks forever in os.write and the leader silently stops forwarding commands (confirmed
+        # twice via py-spy: MainThread stuck at client_socket.py send). Waiting for the reply keeps
+        # every request in its own burst, so writes and drains stay exactly balanced.
+        self._client.command_joint_pos(joint_pos).result()
 
     def command_joint_state(self, joint_state: Dict[str, np.ndarray]) -> None:
-        self._client.command_joint_state(joint_state)
+        self._client.command_joint_state(joint_state).result()  # see command_joint_pos for why
 
     def get_observations(self) -> Dict[str, np.ndarray]:
         return self._client.get_observations().result()
@@ -207,6 +215,9 @@ def _rpc_polling_worker(
         except Exception as e:
             logging.error(f"[{rate_name}] error: {e}")
             time.sleep(0.1)
+        # Pace the loop: unbounded RPC deadlocks portal's client the same way as in
+        # _run_leader_io_loop (see the comment there).
+        time.sleep(_WORKER_LOOP_PERIOD_S)
 
 
 def _leader_control_worker(
@@ -399,6 +410,13 @@ def _run_leader_io_loop(
             # leader arm is energized in bilateral PD — an abrupt loss of control of a powered arm.
             logging.error(f"[yam-leader web-port io] error: {e}")
             time.sleep(0.1)
+        # Pace the loop. portal's client wakes its socket thread by writing one byte to a pipe per
+        # request, but that pipe is drained at most one byte per socket-loop iteration and only while
+        # the send queue is empty (portal/client_socket.py: the `if not writing:` guard). Unpaced this
+        # loop issues ~7 kHz of RPCs, the send queue never empties, the 64 KiB pipe fills, and send()
+        # blocks forever in os.write with no timeout — the leader silently stops forwarding commands
+        # while its control worker keeps running. 500 Hz is well clear of the ~250 Hz hardware rate.
+        time.sleep(_WORKER_LOOP_PERIOD_S)
 
 
 def _run_viewer_loop(